@rowstile/cli-win32-x64 0.1.0-alpha.2

package/python/Lib/multiprocessing/queues.py

@@ -0,0 +1,399 @@
+#
+# Module implementing queues
+#
+# multiprocessing/queues.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue']
+
+import sys
+import os
+import threading
+import collections
+import time
+import types
+import weakref
+import errno
+
+from queue import Empty, Full
+
+from . import connection
+from . import context
+_ForkingPickler = context.reduction.ForkingPickler
+
+from .util import debug, info, Finalize, register_after_fork, is_exiting
+
+#
+# Queue type using a pipe, buffer and thread
+#
+
+class Queue(object):
+
+    def __init__(self, maxsize=0, *, ctx):
+        if maxsize <= 0:
+            # Can raise ImportError (see issues #3770 and #23400)
+            from .synchronize import SEM_VALUE_MAX as maxsize
+        self._maxsize = maxsize
+        self._reader, self._writer = connection.Pipe(duplex=False)
+        self._rlock = ctx.Lock()
+        self._opid = os.getpid()
+        if sys.platform == 'win32':
+            self._wlock = None
+        else:
+            self._wlock = ctx.Lock()
+        self._sem = ctx.BoundedSemaphore(maxsize)
+        # For use by concurrent.futures
+        self._ignore_epipe = False
+        self._reset()
+
+        if sys.platform != 'win32':
+            register_after_fork(self, Queue._after_fork)
+
+    def __getstate__(self):
+        context.assert_spawning(self)
+        return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
+                self._rlock, self._wlock, self._sem, self._opid)
+
+    def __setstate__(self, state):
+        (self._ignore_epipe, self._maxsize, self._reader, self._writer,
+         self._rlock, self._wlock, self._sem, self._opid) = state
+        self._reset()
+
+    def _after_fork(self):
+        debug('Queue._after_fork()')
+        self._reset(after_fork=True)
+
+    def _reset(self, after_fork=False):
+        if after_fork:
+            self._notempty._at_fork_reinit()
+        else:
+            self._notempty = threading.Condition(threading.Lock())
+        self._buffer = collections.deque()
+        self._thread = None
+        self._jointhread = None
+        self._joincancelled = False
+        self._closed = False
+        self._close = None
+        self._send_bytes = self._writer.send_bytes
+        self._recv_bytes = self._reader.recv_bytes
+        self._poll = self._reader.poll
+
+    def put(self, obj, block=True, timeout=None):
+        if self._closed:
+            raise ValueError(f"Queue {self!r} is closed")
+        if not self._sem.acquire(block, timeout):
+            raise Full
+
+        with self._notempty:
+            if self._thread is None:
+                self._start_thread()
+            self._buffer.append(obj)
+            self._notempty.notify()
+
+    def get(self, block=True, timeout=None):
+        if self._closed:
+            raise ValueError(f"Queue {self!r} is closed")
+        if block and timeout is None:
+            with self._rlock:
+                res = self._recv_bytes()
+            self._sem.release()
+        else:
+            if block:
+                deadline = time.monotonic() + timeout
+            if not self._rlock.acquire(block, timeout):
+                raise Empty
+            try:
+                if block:
+                    timeout = deadline - time.monotonic()
+                    if not self._poll(timeout):
+                        raise Empty
+                elif not self._poll():
+                    raise Empty
+                res = self._recv_bytes()
+                self._sem.release()
+            finally:
+                self._rlock.release()
+        # unserialize the data after having released the lock
+        return _ForkingPickler.loads(res)
+
+    def qsize(self):
+        # Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
+        return self._maxsize - self._sem._semlock._get_value()
+
+    def empty(self):
+        return not self._poll()
+
+    def full(self):
+        return self._sem._semlock._is_zero()
+
+    def get_nowait(self):
+        return self.get(False)
+
+    def put_nowait(self, obj):
+        return self.put(obj, False)
+
+    def close(self):
+        self._closed = True
+        close = self._close
+        if close:
+            self._close = None
+            close()
+
+    def join_thread(self):
+        debug('Queue.join_thread()')
+        assert self._closed, "Queue {0!r} not closed".format(self)
+        if self._jointhread:
+            self._jointhread()
+
+    def cancel_join_thread(self):
+        debug('Queue.cancel_join_thread()')
+        self._joincancelled = True
+        try:
+            self._jointhread.cancel()
+        except AttributeError:
+            pass
+
+    def _terminate_broken(self):
+        # Close a Queue on error.
+
+        # gh-94777: Prevent queue writing to a pipe which is no longer read.
+        self._reader.close()
+
+        # gh-107219: Close the connection writer which can unblock
+        # Queue._feed() if it was stuck in send_bytes().
+        if sys.platform == 'win32':
+            self._writer.close()
+
+        self.close()
+        self.join_thread()
+
+    def _start_thread(self):
+        debug('Queue._start_thread()')
+
+        # Start thread which transfers data from buffer to pipe
+        self._buffer.clear()
+        self._thread = threading.Thread(
+            target=Queue._feed,
+            args=(self._buffer, self._notempty, self._send_bytes,
+                  self._wlock, self._reader.close, self._writer.close,
+                  self._ignore_epipe, self._on_queue_feeder_error,
+                  self._sem),
+            name='QueueFeederThread',
+            daemon=True,
+        )
+
+        try:
+            debug('doing self._thread.start()')
+            self._thread.start()
+            debug('... done self._thread.start()')
+        except:
+            # gh-109047: During Python finalization, creating a thread
+            # can fail with RuntimeError.
+            self._thread = None
+            raise
+
+        if not self._joincancelled:
+            self._jointhread = Finalize(
+                self._thread, Queue._finalize_join,
+                [weakref.ref(self._thread)],
+                exitpriority=-5
+                )
+
+        # Send sentinel to the thread queue object when garbage collected
+        self._close = Finalize(
+            self, Queue._finalize_close,
+            [self._buffer, self._notempty],
+            exitpriority=10
+            )
+
+    @staticmethod
+    def _finalize_join(twr):
+        debug('joining queue thread')
+        thread = twr()
+        if thread is not None:
+            thread.join()
+            debug('... queue thread joined')
+        else:
+            debug('... queue thread already dead')
+
+    @staticmethod
+    def _finalize_close(buffer, notempty):
+        debug('telling queue thread to quit')
+        with notempty:
+            buffer.append(_sentinel)
+            notempty.notify()
+
+    @staticmethod
+    def _feed(buffer, notempty, send_bytes, writelock, reader_close,
+              writer_close, ignore_epipe, onerror, queue_sem):
+        debug('starting thread to feed data to pipe')
+        nacquire = notempty.acquire
+        nrelease = notempty.release
+        nwait = notempty.wait
+        bpopleft = buffer.popleft
+        sentinel = _sentinel
+        if sys.platform != 'win32':
+            wacquire = writelock.acquire
+            wrelease = writelock.release
+        else:
+            wacquire = None
+
+        while 1:
+            try:
+                nacquire()
+                try:
+                    if not buffer:
+                        nwait()
+                finally:
+                    nrelease()
+                try:
+                    while 1:
+                        obj = bpopleft()
+                        if obj is sentinel:
+                            debug('feeder thread got sentinel -- exiting')
+                            reader_close()
+                            writer_close()
+                            return
+
+                        # serialize the data before acquiring the lock
+                        obj = _ForkingPickler.dumps(obj)
+                        if wacquire is None:
+                            send_bytes(obj)
+                        else:
+                            wacquire()
+                            try:
+                                send_bytes(obj)
+                            finally:
+                                wrelease()
+                except IndexError:
+                    pass
+            except Exception as e:
+                if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
+                    return
+                # Since this runs in a daemon thread the resources it uses
+                # may be become unusable while the process is cleaning up.
+                # We ignore errors which happen after the process has
+                # started to cleanup.
+                if is_exiting():
+                    info('error in queue thread: %s', e)
+                    return
+                else:
+                    # Since the object has not been sent in the queue, we need
+                    # to decrease the size of the queue. The error acts as
+                    # if the object had been silently removed from the queue
+                    # and this step is necessary to have a properly working
+                    # queue.
+                    queue_sem.release()
+                    onerror(e, obj)
+
+    @staticmethod
+    def _on_queue_feeder_error(e, obj):
+        """
+        Private API hook called when feeding data in the background thread
+        raises an exception.  For overriding by concurrent.futures.
+        """
+        import traceback
+        traceback.print_exc()
+
+    __class_getitem__ = classmethod(types.GenericAlias)
+
+
+_sentinel = object()
+
+#
+# A queue type which also supports join() and task_done() methods
+#
+# Note that if you do not call task_done() for each finished task then
+# eventually the counter's semaphore may overflow causing Bad Things
+# to happen.
+#
+
+class JoinableQueue(Queue):
+
+    def __init__(self, maxsize=0, *, ctx):
+        Queue.__init__(self, maxsize, ctx=ctx)
+        self._unfinished_tasks = ctx.Semaphore(0)
+        self._cond = ctx.Condition()
+
+    def __getstate__(self):
+        return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
+
+    def __setstate__(self, state):
+        Queue.__setstate__(self, state[:-2])
+        self._cond, self._unfinished_tasks = state[-2:]
+
+    def put(self, obj, block=True, timeout=None):
+        if self._closed:
+            raise ValueError(f"Queue {self!r} is closed")
+        if not self._sem.acquire(block, timeout):
+            raise Full
+
+        with self._notempty, self._cond:
+            if self._thread is None:
+                self._start_thread()
+            self._buffer.append(obj)
+            self._unfinished_tasks.release()
+            self._notempty.notify()
+
+    def task_done(self):
+        with self._cond:
+            if not self._unfinished_tasks.acquire(False):
+                raise ValueError('task_done() called too many times')
+            if self._unfinished_tasks._semlock._is_zero():
+                self._cond.notify_all()
+
+    def join(self):
+        with self._cond:
+            if not self._unfinished_tasks._semlock._is_zero():
+                self._cond.wait()
+
+#
+# Simplified Queue type -- really just a locked pipe
+#
+
+class SimpleQueue(object):
+
+    def __init__(self, *, ctx):
+        self._reader, self._writer = connection.Pipe(duplex=False)
+        self._rlock = ctx.Lock()
+        self._poll = self._reader.poll
+        if sys.platform == 'win32':
+            self._wlock = None
+        else:
+            self._wlock = ctx.Lock()
+
+    def close(self):
+        self._reader.close()
+        self._writer.close()
+
+    def empty(self):
+        return not self._poll()
+
+    def __getstate__(self):
+        context.assert_spawning(self)
+        return (self._reader, self._writer, self._rlock, self._wlock)
+
+    def __setstate__(self, state):
+        (self._reader, self._writer, self._rlock, self._wlock) = state
+        self._poll = self._reader.poll
+
+    def get(self):
+        with self._rlock:
+            res = self._reader.recv_bytes()
+        # unserialize the data after having released the lock
+        return _ForkingPickler.loads(res)
+
+    def put(self, obj):
+        # serialize the data before acquiring the lock
+        obj = _ForkingPickler.dumps(obj)
+        if self._wlock is None:
+            # writes to a message oriented win32 pipe are atomic
+            self._writer.send_bytes(obj)
+        else:
+            with self._wlock:
+                self._writer.send_bytes(obj)
+
+    __class_getitem__ = classmethod(types.GenericAlias)

package/python/Lib/multiprocessing/reduction.py

@@ -0,0 +1,281 @@
+#
+# Module which deals with pickling of objects.
+#
+# multiprocessing/reduction.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+from abc import ABCMeta
+import copyreg
+import functools
+import io
+import os
+import pickle
+import socket
+import sys
+
+from . import context
+
+__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump']
+
+
+HAVE_SEND_HANDLE = (sys.platform == 'win32' or
+                    (hasattr(socket, 'CMSG_LEN') and
+                     hasattr(socket, 'SCM_RIGHTS') and
+                     hasattr(socket.socket, 'sendmsg')))
+
+#
+# Pickler subclass
+#
+
+class ForkingPickler(pickle.Pickler):
+    '''Pickler subclass used by multiprocessing.'''
+    _extra_reducers = {}
+    _copyreg_dispatch_table = copyreg.dispatch_table
+
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.dispatch_table = self._copyreg_dispatch_table.copy()
+        self.dispatch_table.update(self._extra_reducers)
+
+    @classmethod
+    def register(cls, type, reduce):
+        '''Register a reduce function for a type.'''
+        cls._extra_reducers[type] = reduce
+
+    @classmethod
+    def dumps(cls, obj, protocol=None):
+        buf = io.BytesIO()
+        cls(buf, protocol).dump(obj)
+        return buf.getbuffer()
+
+    loads = pickle.loads
+
+register = ForkingPickler.register
+
+def dump(obj, file, protocol=None):
+    '''Replacement for pickle.dump() using ForkingPickler.'''
+    ForkingPickler(file, protocol).dump(obj)
+
+#
+# Platform specific definitions
+#
+
+if sys.platform == 'win32':
+    # Windows
+    __all__ += ['DupHandle', 'duplicate', 'steal_handle']
+    import _winapi
+
+    def duplicate(handle, target_process=None, inheritable=False,
+                  *, source_process=None):
+        '''Duplicate a handle.  (target_process is a handle not a pid!)'''
+        current_process = _winapi.GetCurrentProcess()
+        if source_process is None:
+            source_process = current_process
+        if target_process is None:
+            target_process = current_process
+        return _winapi.DuplicateHandle(
+            source_process, handle, target_process,
+            0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
+
+    def steal_handle(source_pid, handle):
+        '''Steal a handle from process identified by source_pid.'''
+        source_process_handle = _winapi.OpenProcess(
+            _winapi.PROCESS_DUP_HANDLE, False, source_pid)
+        try:
+            return _winapi.DuplicateHandle(
+                source_process_handle, handle,
+                _winapi.GetCurrentProcess(), 0, False,
+                _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE)
+        finally:
+            _winapi.CloseHandle(source_process_handle)
+
+    def send_handle(conn, handle, destination_pid):
+        '''Send a handle over a local connection.'''
+        dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
+        conn.send(dh)
+
+    def recv_handle(conn):
+        '''Receive a handle over a local connection.'''
+        return conn.recv().detach()
+
+    class DupHandle(object):
+        '''Picklable wrapper for a handle.'''
+        def __init__(self, handle, access, pid=None):
+            if pid is None:
+                # We just duplicate the handle in the current process and
+                # let the receiving process steal the handle.
+                pid = os.getpid()
+            proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
+            try:
+                self._handle = _winapi.DuplicateHandle(
+                    _winapi.GetCurrentProcess(),
+                    handle, proc, access, False, 0)
+            finally:
+                _winapi.CloseHandle(proc)
+            self._access = access
+            self._pid = pid
+
+        def detach(self):
+            '''Get the handle.  This should only be called once.'''
+            # retrieve handle from process which currently owns it
+            if self._pid == os.getpid():
+                # The handle has already been duplicated for this process.
+                return self._handle
+            # We must steal the handle from the process whose pid is self._pid.
+            proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
+                                       self._pid)
+            try:
+                return _winapi.DuplicateHandle(
+                    proc, self._handle, _winapi.GetCurrentProcess(),
+                    self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE)
+            finally:
+                _winapi.CloseHandle(proc)
+
+else:
+    # Unix
+    __all__ += ['DupFd', 'sendfds', 'recvfds']
+    import array
+
+    # On MacOSX we should acknowledge receipt of fds -- see Issue14669
+    ACKNOWLEDGE = sys.platform == 'darwin'
+
+    def sendfds(sock, fds):
+        '''Send an array of fds over an AF_UNIX socket.'''
+        fds = array.array('i', fds)
+        msg = bytes([len(fds) % 256])
+        sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
+        if ACKNOWLEDGE and sock.recv(1) != b'A':
+            raise RuntimeError('did not receive acknowledgement of fd')
+
+    def recvfds(sock, size):
+        '''Receive an array of fds over an AF_UNIX socket.'''
+        a = array.array('i')
+        bytes_size = a.itemsize * size
+        msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size))
+        if not msg and not ancdata:
+            raise EOFError
+        try:
+            if ACKNOWLEDGE:
+                sock.send(b'A')
+            if len(ancdata) != 1:
+                raise RuntimeError('received %d items of ancdata' %
+                                   len(ancdata))
+            cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+            if (cmsg_level == socket.SOL_SOCKET and
+                cmsg_type == socket.SCM_RIGHTS):
+                if len(cmsg_data) % a.itemsize != 0:
+                    raise ValueError
+                a.frombytes(cmsg_data)
+                if len(a) % 256 != msg[0]:
+                    raise AssertionError(
+                        "Len is {0:n} but msg[0] is {1!r}".format(
+                            len(a), msg[0]))
+                return list(a)
+        except (ValueError, IndexError):
+            pass
+        raise RuntimeError('Invalid data received')
+
+    def send_handle(conn, handle, destination_pid):
+        '''Send a handle over a local connection.'''
+        with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
+            sendfds(s, [handle])
+
+    def recv_handle(conn):
+        '''Receive a handle over a local connection.'''
+        with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
+            return recvfds(s, 1)[0]
+
+    def DupFd(fd):
+        '''Return a wrapper for an fd.'''
+        popen_obj = context.get_spawning_popen()
+        if popen_obj is not None:
+            return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
+        elif HAVE_SEND_HANDLE:
+            from . import resource_sharer
+            return resource_sharer.DupFd(fd)
+        else:
+            raise ValueError('SCM_RIGHTS appears not to be available')
+
+#
+# Try making some callable types picklable
+#
+
+def _reduce_method(m):
+    if m.__self__ is None:
+        return getattr, (m.__class__, m.__func__.__name__)
+    else:
+        return getattr, (m.__self__, m.__func__.__name__)
+class _C:
+    def f(self):
+        pass
+register(type(_C().f), _reduce_method)
+
+
+def _reduce_method_descriptor(m):
+    return getattr, (m.__objclass__, m.__name__)
+register(type(list.append), _reduce_method_descriptor)
+register(type(int.__add__), _reduce_method_descriptor)
+
+
+def _reduce_partial(p):
+    return _rebuild_partial, (p.func, p.args, p.keywords or {})
+def _rebuild_partial(func, args, keywords):
+    return functools.partial(func, *args, **keywords)
+register(functools.partial, _reduce_partial)
+
+#
+# Make sockets picklable
+#
+
+if sys.platform == 'win32':
+    def _reduce_socket(s):
+        from .resource_sharer import DupSocket
+        return _rebuild_socket, (DupSocket(s),)
+    def _rebuild_socket(ds):
+        return ds.detach()
+    register(socket.socket, _reduce_socket)
+
+else:
+    def _reduce_socket(s):
+        df = DupFd(s.fileno())
+        return _rebuild_socket, (df, s.family, s.type, s.proto)
+    def _rebuild_socket(df, family, type, proto):
+        fd = df.detach()
+        return socket.socket(family, type, proto, fileno=fd)
+    register(socket.socket, _reduce_socket)
+
+
+class AbstractReducer(metaclass=ABCMeta):
+    '''Abstract base class for use in implementing a Reduction class
+    suitable for use in replacing the standard reduction mechanism
+    used in multiprocessing.'''
+    ForkingPickler = ForkingPickler
+    register = register
+    dump = dump
+    send_handle = send_handle
+    recv_handle = recv_handle
+
+    if sys.platform == 'win32':
+        steal_handle = steal_handle
+        duplicate = duplicate
+        DupHandle = DupHandle
+    else:
+        sendfds = sendfds
+        recvfds = recvfds
+        DupFd = DupFd
+
+    _reduce_method = _reduce_method
+    _reduce_method_descriptor = _reduce_method_descriptor
+    _rebuild_partial = _rebuild_partial
+    _reduce_socket = _reduce_socket
+    _rebuild_socket = _rebuild_socket
+
+    def __init__(self, *args):
+        register(type(_C().f), _reduce_method)
+        register(type(list.append), _reduce_method_descriptor)
+        register(type(int.__add__), _reduce_method_descriptor)
+        register(functools.partial, _reduce_partial)
+        register(socket.socket, _reduce_socket)