@rowstile/cli-win32-x64 0.1.0-alpha.2

package/python/Lib/multiprocessing/sharedctypes.py

@@ -0,0 +1,240 @@
+#
+# Module which supports allocation of ctypes objects from shared memory
+#
+# multiprocessing/sharedctypes.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import ctypes
+import weakref
+
+from . import heap
+from . import get_context
+
+from .context import reduction, assert_spawning
+_ForkingPickler = reduction.ForkingPickler
+
+__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
+
+#
+#
+#
+
+typecode_to_type = {
+    'c': ctypes.c_char,     'u': ctypes.c_wchar,
+    'b': ctypes.c_byte,     'B': ctypes.c_ubyte,
+    'h': ctypes.c_short,    'H': ctypes.c_ushort,
+    'i': ctypes.c_int,      'I': ctypes.c_uint,
+    'l': ctypes.c_long,     'L': ctypes.c_ulong,
+    'q': ctypes.c_longlong, 'Q': ctypes.c_ulonglong,
+    'f': ctypes.c_float,    'd': ctypes.c_double
+    }
+
+#
+#
+#
+
+def _new_value(type_):
+    size = ctypes.sizeof(type_)
+    wrapper = heap.BufferWrapper(size)
+    return rebuild_ctype(type_, wrapper, None)
+
+def RawValue(typecode_or_type, *args):
+    '''
+    Returns a ctypes object allocated from shared memory
+    '''
+    type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
+    obj = _new_value(type_)
+    ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+    obj.__init__(*args)
+    return obj
+
+def RawArray(typecode_or_type, size_or_initializer):
+    '''
+    Returns a ctypes array allocated from shared memory
+    '''
+    type_ = typecode_to_type.get(typecode_or_type, typecode_or_type)
+    if isinstance(size_or_initializer, int):
+        type_ = type_ * size_or_initializer
+        obj = _new_value(type_)
+        ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj))
+        return obj
+    else:
+        type_ = type_ * len(size_or_initializer)
+        result = _new_value(type_)
+        result.__init__(*size_or_initializer)
+        return result
+
+def Value(typecode_or_type, *args, lock=True, ctx=None):
+    '''
+    Return a synchronization wrapper for a Value
+    '''
+    obj = RawValue(typecode_or_type, *args)
+    if lock is False:
+        return obj
+    if lock in (True, None):
+        ctx = ctx or get_context()
+        lock = ctx.RLock()
+    if not hasattr(lock, 'acquire'):
+        raise AttributeError("%r has no method 'acquire'" % lock)
+    return synchronized(obj, lock, ctx=ctx)
+
+def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None):
+    '''
+    Return a synchronization wrapper for a RawArray
+    '''
+    obj = RawArray(typecode_or_type, size_or_initializer)
+    if lock is False:
+        return obj
+    if lock in (True, None):
+        ctx = ctx or get_context()
+        lock = ctx.RLock()
+    if not hasattr(lock, 'acquire'):
+        raise AttributeError("%r has no method 'acquire'" % lock)
+    return synchronized(obj, lock, ctx=ctx)
+
+def copy(obj):
+    new_obj = _new_value(type(obj))
+    ctypes.pointer(new_obj)[0] = obj
+    return new_obj
+
+def synchronized(obj, lock=None, ctx=None):
+    assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
+    ctx = ctx or get_context()
+
+    if isinstance(obj, ctypes._SimpleCData):
+        return Synchronized(obj, lock, ctx)
+    elif isinstance(obj, ctypes.Array):
+        if obj._type_ is ctypes.c_char:
+            return SynchronizedString(obj, lock, ctx)
+        return SynchronizedArray(obj, lock, ctx)
+    else:
+        cls = type(obj)
+        try:
+            scls = class_cache[cls]
+        except KeyError:
+            names = [field[0] for field in cls._fields_]
+            d = {name: make_property(name) for name in names}
+            classname = 'Synchronized' + cls.__name__
+            scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
+        return scls(obj, lock, ctx)
+
+#
+# Functions for pickling/unpickling
+#
+
+def reduce_ctype(obj):
+    assert_spawning(obj)
+    if isinstance(obj, ctypes.Array):
+        return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_)
+    else:
+        return rebuild_ctype, (type(obj), obj._wrapper, None)
+
+def rebuild_ctype(type_, wrapper, length):
+    if length is not None:
+        type_ = type_ * length
+    _ForkingPickler.register(type_, reduce_ctype)
+    buf = wrapper.create_memoryview()
+    obj = type_.from_buffer(buf)
+    obj._wrapper = wrapper
+    return obj
+
+#
+# Function to create properties
+#
+
+def make_property(name):
+    try:
+        return prop_cache[name]
+    except KeyError:
+        d = {}
+        exec(template % ((name,)*7), d)
+        prop_cache[name] = d[name]
+        return d[name]
+
+template = '''
+def get%s(self):
+    self.acquire()
+    try:
+        return self._obj.%s
+    finally:
+        self.release()
+def set%s(self, value):
+    self.acquire()
+    try:
+        self._obj.%s = value
+    finally:
+        self.release()
+%s = property(get%s, set%s)
+'''
+
+prop_cache = {}
+class_cache = weakref.WeakKeyDictionary()
+
+#
+# Synchronized wrappers
+#
+
+class SynchronizedBase(object):
+
+    def __init__(self, obj, lock=None, ctx=None):
+        self._obj = obj
+        if lock:
+            self._lock = lock
+        else:
+            ctx = ctx or get_context(force=True)
+            self._lock = ctx.RLock()
+        self.acquire = self._lock.acquire
+        self.release = self._lock.release
+
+    def __enter__(self):
+        return self._lock.__enter__()
+
+    def __exit__(self, *args):
+        return self._lock.__exit__(*args)
+
+    def __reduce__(self):
+        assert_spawning(self)
+        return synchronized, (self._obj, self._lock)
+
+    def get_obj(self):
+        return self._obj
+
+    def get_lock(self):
+        return self._lock
+
+    def __repr__(self):
+        return '<%s wrapper for %s>' % (type(self).__name__, self._obj)
+
+
+class Synchronized(SynchronizedBase):
+    value = make_property('value')
+
+
+class SynchronizedArray(SynchronizedBase):
+
+    def __len__(self):
+        return len(self._obj)
+
+    def __getitem__(self, i):
+        with self:
+            return self._obj[i]
+
+    def __setitem__(self, i, value):
+        with self:
+            self._obj[i] = value
+
+    def __getslice__(self, start, stop):
+        with self:
+            return self._obj[start:stop]
+
+    def __setslice__(self, start, stop, values):
+        with self:
+            self._obj[start:stop] = values
+
+
+class SynchronizedString(SynchronizedArray):
+    value = make_property('value')
+    raw = make_property('raw')

package/python/Lib/multiprocessing/spawn.py

@@ -0,0 +1,307 @@
+#
+# Code used to start processes when using the spawn or forkserver
+# start methods.
+#
+# multiprocessing/spawn.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import os
+import sys
+import runpy
+import types
+
+from . import get_start_method, set_start_method
+from . import process
+from .context import reduction
+from . import util
+
+__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
+           'get_preparation_data', 'get_command_line', 'import_main_path']
+
+#
+# _python_exe is the assumed path to the python executable.
+# People embedding Python want to modify it.
+#
+
+if sys.platform != 'win32':
+    WINEXE = False
+    WINSERVICE = False
+else:
+    WINEXE = getattr(sys, 'frozen', False)
+    WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe")
+
+def set_executable(exe):
+    global _python_exe
+    if exe is None:
+        _python_exe = exe
+    elif sys.platform == 'win32':
+        _python_exe = os.fsdecode(exe)
+    else:
+        _python_exe = os.fsencode(exe)
+
+def get_executable():
+    return _python_exe
+
+if WINSERVICE:
+    set_executable(os.path.join(sys.exec_prefix, 'python.exe'))
+else:
+    set_executable(sys.executable)
+
+#
+#
+#
+
+def is_forking(argv):
+    '''
+    Return whether commandline indicates we are forking
+    '''
+    if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
+        return True
+    else:
+        return False
+
+
+def freeze_support():
+    '''
+    Run code for process object if this in not the main process
+    '''
+    if is_forking(sys.argv):
+        kwds = {}
+        for arg in sys.argv[2:]:
+            name, value = arg.split('=')
+            if value == 'None':
+                kwds[name] = None
+            else:
+                kwds[name] = int(value)
+        spawn_main(**kwds)
+        sys.exit()
+
+
+def get_command_line(**kwds):
+    '''
+    Returns prefix of command line used for spawning a child process
+    '''
+    if getattr(sys, 'frozen', False):
+        return ([sys.executable, '--multiprocessing-fork'] +
+                ['%s=%r' % item for item in kwds.items()])
+    else:
+        prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
+        prog %= ', '.join('%s=%r' % item for item in kwds.items())
+        opts = util._args_from_interpreter_flags()
+        exe = get_executable()
+        return [exe] + opts + ['-c', prog, '--multiprocessing-fork']
+
+
+def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
+    '''
+    Run code specified by data received over pipe
+    '''
+    assert is_forking(sys.argv), "Not forking"
+    if sys.platform == 'win32':
+        import msvcrt
+        import _winapi
+
+        if parent_pid is not None:
+            source_process = _winapi.OpenProcess(
+                _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE,
+                False, parent_pid)
+        else:
+            source_process = None
+        new_handle = reduction.duplicate(pipe_handle,
+                                         source_process=source_process)
+        fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
+        parent_sentinel = source_process
+    else:
+        from . import resource_tracker
+        resource_tracker._resource_tracker._fd = tracker_fd
+        fd = pipe_handle
+        parent_sentinel = os.dup(pipe_handle)
+    exitcode = _main(fd, parent_sentinel)
+    sys.exit(exitcode)
+
+
+def _main(fd, parent_sentinel):
+    with os.fdopen(fd, 'rb', closefd=True) as from_parent:
+        process.current_process()._inheriting = True
+        try:
+            preparation_data = reduction.pickle.load(from_parent)
+            prepare(preparation_data)
+            self = reduction.pickle.load(from_parent)
+        finally:
+            del process.current_process()._inheriting
+    return self._bootstrap(parent_sentinel)
+
+
+def _check_not_importing_main():
+    if getattr(process.current_process(), '_inheriting', False):
+        raise RuntimeError('''
+        An attempt has been made to start a new process before the
+        current process has finished its bootstrapping phase.
+
+        This probably means that you are not using fork to start your
+        child processes and you have forgotten to use the proper idiom
+        in the main module:
+
+            if __name__ == '__main__':
+                freeze_support()
+                ...
+
+        The "freeze_support()" line can be omitted if the program
+        is not going to be frozen to produce an executable.
+
+        To fix this issue, refer to the "Safe importing of main module"
+        section in https://docs.python.org/3/library/multiprocessing.html
+        ''')
+
+
+def get_preparation_data(name):
+    '''
+    Return info about parent needed by child to unpickle process object
+    '''
+    _check_not_importing_main()
+    d = dict(
+        log_to_stderr=util._log_to_stderr,
+        authkey=process.current_process().authkey,
+        )
+
+    if util._logger is not None:
+        d['log_level'] = util._logger.getEffectiveLevel()
+
+    sys_path=sys.path.copy()
+    try:
+        i = sys_path.index('')
+    except ValueError:
+        pass
+    else:
+        sys_path[i] = process.ORIGINAL_DIR
+
+    d.update(
+        name=name,
+        sys_path=sys_path,
+        sys_argv=sys.argv,
+        orig_dir=process.ORIGINAL_DIR,
+        dir=os.getcwd(),
+        start_method=get_start_method(allow_none=True),
+        )
+
+    # Figure out whether to initialise main in the subprocess as a module
+    # or through direct execution (or to leave it alone entirely)
+    main_module = sys.modules['__main__']
+    main_mod_name = getattr(main_module.__spec__, "name", None)
+    if main_mod_name is not None:
+        d['init_main_from_name'] = main_mod_name
+    elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
+        main_path = getattr(main_module, '__file__', None)
+        if main_path is not None:
+            if (not os.path.isabs(main_path) and
+                        process.ORIGINAL_DIR is not None):
+                main_path = os.path.join(process.ORIGINAL_DIR, main_path)
+            d['init_main_from_path'] = os.path.normpath(main_path)
+
+    return d
+
+#
+# Prepare current process
+#
+
+old_main_modules = []
+
+def prepare(data):
+    '''
+    Try to get current process ready to unpickle process object
+    '''
+    if 'name' in data:
+        process.current_process().name = data['name']
+
+    if 'authkey' in data:
+        process.current_process().authkey = data['authkey']
+
+    if 'log_to_stderr' in data and data['log_to_stderr']:
+        util.log_to_stderr()
+
+    if 'log_level' in data:
+        util.get_logger().setLevel(data['log_level'])
+
+    if 'sys_path' in data:
+        sys.path = data['sys_path']
+
+    if 'sys_argv' in data:
+        sys.argv = data['sys_argv']
+
+    if 'dir' in data:
+        os.chdir(data['dir'])
+
+    if 'orig_dir' in data:
+        process.ORIGINAL_DIR = data['orig_dir']
+
+    if 'start_method' in data:
+        set_start_method(data['start_method'], force=True)
+
+    if 'init_main_from_name' in data:
+        _fixup_main_from_name(data['init_main_from_name'])
+    elif 'init_main_from_path' in data:
+        _fixup_main_from_path(data['init_main_from_path'])
+
+# Multiprocessing module helpers to fix up the main module in
+# spawned subprocesses
+def _fixup_main_from_name(mod_name):
+    # __main__.py files for packages, directories, zip archives, etc, run
+    # their "main only" code unconditionally, so we don't even try to
+    # populate anything in __main__, nor do we make any changes to
+    # __main__ attributes
+    current_main = sys.modules['__main__']
+    if mod_name == "__main__" or mod_name.endswith(".__main__"):
+        return
+
+    # If this process was forked, __main__ may already be populated
+    if getattr(current_main.__spec__, "name", None) == mod_name:
+        return
+
+    # Otherwise, __main__ may contain some non-main code where we need to
+    # support unpickling it properly. We rerun it as __mp_main__ and make
+    # the normal __main__ an alias to that
+    old_main_modules.append(current_main)
+    main_module = types.ModuleType("__mp_main__")
+    main_content = runpy.run_module(mod_name,
+                                    run_name="__mp_main__",
+                                    alter_sys=True)
+    main_module.__dict__.update(main_content)
+    sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
+
+
+def _fixup_main_from_path(main_path):
+    # If this process was forked, __main__ may already be populated
+    current_main = sys.modules['__main__']
+
+    # Unfortunately, the main ipython launch script historically had no
+    # "if __name__ == '__main__'" guard, so we work around that
+    # by treating it like a __main__.py file
+    # See https://github.com/ipython/ipython/issues/4698
+    main_name = os.path.splitext(os.path.basename(main_path))[0]
+    if main_name == 'ipython':
+        return
+
+    # Otherwise, if __file__ already has the setting we expect,
+    # there's nothing more to do
+    if getattr(current_main, '__file__', None) == main_path:
+        return
+
+    # If the parent process has sent a path through rather than a module
+    # name we assume it is an executable script that may contain
+    # non-main code that needs to be executed
+    old_main_modules.append(current_main)
+    main_module = types.ModuleType("__mp_main__")
+    main_content = runpy.run_path(main_path,
+                                  run_name="__mp_main__")
+    main_module.__dict__.update(main_content)
+    sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
+
+
+def import_main_path(main_path):
+    '''
+    Set sys.modules['__main__'] to module at main_path
+    '''
+    _fixup_main_from_path(main_path)