better-dsh 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -178
- package/cordis.patch.yml +4 -35
- package/docs/adr/0015-independent-bundles-agent-remote.md +40 -0
- package/docs/remote-tools.md +56 -0
- package/docs/specs/omp-python-repl-port/runtime-guide.md +33 -0
- package/docs/specs/omp-python-repl-port/spec.md +61 -0
- package/docs/specs/remote-tool-enhancement/spec.md +203 -0
- package/dsh-docs/baseline.md +13 -0
- package/dsh-docs/subsystems/invariants.md +3 -0
- package/dsh-docs/subsystems/invariants.zh.md +3 -0
- package/eval-description.md +7 -3
- package/lib/{LaunchOptions-kGXcZOe0.js → LaunchOptions-5eg7E0ye.js} +1 -1
- package/lib/LaunchOptions-DxQQ0s-A.js +4 -0
- package/lib/{bidi-BalpKaJc.js → bidi-CwabFH07.js} +1 -1
- package/lib/client/index.js +30 -6
- package/lib/failover/index.js +1 -1
- package/lib/fs-aware/sandbox-plugin.js +3 -3
- package/lib/{index-browser-CysrjMyM.js → index-browser-B-9jm12n.js} +4 -4
- package/lib/index.d.ts +17 -41
- package/lib/index.js +473 -1678
- package/lib/kernel-env-BOF0fNHn.js +91 -0
- package/lib/kernel-env.d.ts +4 -58
- package/lib/kernel-env.js +2 -2
- package/lib/{lsp-server-registry-BXvUTBY7.js → lsp-server-registry-BcdkHROD.js} +1 -1
- package/lib/{lsp-server-registry-CH9WNFFt.js → lsp-server-registry-D4VLs1Sm.js} +1 -1
- package/lib/{main-CupOhMp5.js → main-BeCgUd20.js} +8 -6
- package/lib/{main-BQeFrBM1.js → main-Dv_UQqG2.js} +1 -1
- package/lib/mobile/plugin.d.ts +1 -1
- package/lib/omp-python/DILL-LICENSE +35 -0
- package/lib/omp-python/OMP-LICENSE +23 -0
- package/lib/omp-python/better_snapshot.py +391 -0
- package/lib/omp-python/better_support.py +425 -0
- package/lib/omp-python/dill-provenance.json +55 -0
- package/lib/omp-python/provenance.json +13 -0
- package/lib/omp-python/runner.py +2499 -0
- package/lib/omp-python/vendor/dill/__diff.py +234 -0
- package/lib/omp-python/vendor/dill/__info__.py +291 -0
- package/lib/omp-python/vendor/dill/__init__.py +119 -0
- package/lib/omp-python/vendor/dill/_dill.py +2265 -0
- package/lib/omp-python/vendor/dill/_objects.py +545 -0
- package/lib/omp-python/vendor/dill/_shims.py +193 -0
- package/lib/omp-python/vendor/dill/detect.py +287 -0
- package/lib/omp-python/vendor/dill/logger.py +285 -0
- package/lib/omp-python/vendor/dill/objtypes.py +24 -0
- package/lib/omp-python/vendor/dill/pointers.py +122 -0
- package/lib/omp-python/vendor/dill/session.py +612 -0
- package/lib/omp-python/vendor/dill/settings.py +25 -0
- package/lib/omp-python/vendor/dill/source.py +1025 -0
- package/lib/omp-python/vendor/dill/temp.py +252 -0
- package/lib/{puppeteer-core-CSk2PtSm.js → puppeteer-core-Dnd7eqel.js} +9 -9
- package/lib/remote/plugin.d.ts +459 -5
- package/lib/remote/plugin.js +2389 -210
- package/lib/url-schemes/index.d.ts +11 -11
- package/lib/url-schemes/index.js +12 -12
- package/lib/web-password.d.ts +8 -8
- package/lib/web-password.js +5 -1
- package/lib/{web-trust-Dej_RI2B.d.ts → web-trust-CwZrTBcl.d.ts} +11 -11
- package/lib/web-trust.d.ts +1 -2
- package/lib/welcome-notice.d.ts +5 -7
- package/lib/welcome-notice.js +9 -48
- package/lib/{wrap-T4I0nfPM.js → wrap-0VA0uFx8.js} +1 -1
- package/package.json +6 -23
- package/scripts/kernel-provision.mjs +4 -34
- package/lib/LaunchOptions-B874eBu2.js +0 -4
- package/lib/bundle-delivery.d.ts +0 -20
- package/lib/bundle-delivery.js +0 -209
- package/lib/hmr.d.ts +0 -32
- package/lib/hmr.js +0 -49
- package/lib/kernel-bridge.py +0 -172
- package/lib/kernel-env-D8UOWeji.js +0 -195
- package/lib/webserver.d.ts +0 -10
- package/lib/webserver.js +0 -19
- package/scripts/register-bundle.mjs +0 -80
|
@@ -0,0 +1,2265 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
#
|
|
3
|
+
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
|
|
4
|
+
# Copyright (c) 2008-2015 California Institute of Technology.
|
|
5
|
+
# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
|
|
6
|
+
# License: 3-clause BSD. The full license text is available at:
|
|
7
|
+
# - https://github.com/uqfoundation/dill/blob/master/LICENSE
|
|
8
|
+
"""
|
|
9
|
+
dill: a utility for serialization of python objects
|
|
10
|
+
|
|
11
|
+
The primary functions in `dill` are :func:`dump` and
|
|
12
|
+
:func:`dumps` for serialization ("pickling") to a
|
|
13
|
+
file or to a string, respectively, and :func:`load`
|
|
14
|
+
and :func:`loads` for deserialization ("unpickling"),
|
|
15
|
+
similarly, from a file or from a string. Other notable
|
|
16
|
+
functions are :func:`~dill.dump_module` and
|
|
17
|
+
:func:`~dill.load_module`, which are used to save and
|
|
18
|
+
restore module objects, including an intepreter session.
|
|
19
|
+
|
|
20
|
+
Based on code written by Oren Tirosh and Armin Ronacher.
|
|
21
|
+
Extended to a (near) full set of the builtin types (in types module),
|
|
22
|
+
and coded to the pickle interface, by <mmckerns@caltech.edu>.
|
|
23
|
+
Initial port to python3 by Jonathan Dobson, continued by mmckerns.
|
|
24
|
+
Tested against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns.
|
|
25
|
+
Tested against CH16+ Std. Lib. ... TBD.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
'dump','dumps','load','loads','copy',
|
|
32
|
+
'Pickler','Unpickler','register','pickle','pickles','check',
|
|
33
|
+
'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE',
|
|
34
|
+
'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError',
|
|
35
|
+
'UnpicklingWarning',
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
__module__ = 'dill'
|
|
39
|
+
|
|
40
|
+
import warnings
|
|
41
|
+
from .logger import adapter as logger
|
|
42
|
+
from .logger import trace as _trace
|
|
43
|
+
log = logger # backward compatibility (see issue #582)
|
|
44
|
+
|
|
45
|
+
import os
|
|
46
|
+
import sys
|
|
47
|
+
diff = None
|
|
48
|
+
_use_diff = False
|
|
49
|
+
OLD38 = (sys.hexversion < 0x3080000)
|
|
50
|
+
OLD39 = (sys.hexversion < 0x3090000)
|
|
51
|
+
OLD310 = (sys.hexversion < 0x30a0000)
|
|
52
|
+
OLD312a7 = (sys.hexversion < 0x30c00a7)
|
|
53
|
+
#XXX: get types from .objtypes ?
|
|
54
|
+
import builtins as __builtin__
|
|
55
|
+
from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler
|
|
56
|
+
from pickle import GLOBAL, POP
|
|
57
|
+
from _contextvars import Context as ContextType
|
|
58
|
+
from _thread import LockType
|
|
59
|
+
from _thread import RLock as RLockType
|
|
60
|
+
try:
|
|
61
|
+
from _thread import _ExceptHookArgs as ExceptHookArgsType
|
|
62
|
+
except ImportError:
|
|
63
|
+
ExceptHookArgsType = None
|
|
64
|
+
try:
|
|
65
|
+
from _thread import _ThreadHandle as ThreadHandleType
|
|
66
|
+
except ImportError:
|
|
67
|
+
ThreadHandleType = None
|
|
68
|
+
#from io import IOBase
|
|
69
|
+
from types import CodeType, FunctionType, MethodType, GeneratorType, \
|
|
70
|
+
TracebackType, FrameType, ModuleType, BuiltinMethodType
|
|
71
|
+
BufferType = memoryview #XXX: unregistered
|
|
72
|
+
ClassType = type # no 'old-style' classes
|
|
73
|
+
EllipsisType = type(Ellipsis)
|
|
74
|
+
#FileType = IOBase
|
|
75
|
+
NotImplementedType = type(NotImplemented)
|
|
76
|
+
SliceType = slice
|
|
77
|
+
TypeType = type # 'new-style' classes #XXX: unregistered
|
|
78
|
+
XRangeType = range
|
|
79
|
+
from types import MappingProxyType as DictProxyType, new_class
|
|
80
|
+
from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError
|
|
81
|
+
import __main__ as _main_module
|
|
82
|
+
import marshal
|
|
83
|
+
import gc
|
|
84
|
+
# import zlib
|
|
85
|
+
import abc
|
|
86
|
+
import dataclasses
|
|
87
|
+
from weakref import ReferenceType, ProxyType, CallableProxyType
|
|
88
|
+
from collections import OrderedDict
|
|
89
|
+
from enum import Enum, EnumMeta
|
|
90
|
+
from functools import partial
|
|
91
|
+
from operator import itemgetter, attrgetter
|
|
92
|
+
GENERATOR_FAIL = False
|
|
93
|
+
import importlib.machinery
|
|
94
|
+
EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES)
|
|
95
|
+
try:
|
|
96
|
+
import ctypes
|
|
97
|
+
HAS_CTYPES = True
|
|
98
|
+
# if using `pypy`, pythonapi is not found
|
|
99
|
+
IS_PYPY = not hasattr(ctypes, 'pythonapi')
|
|
100
|
+
except ImportError:
|
|
101
|
+
HAS_CTYPES = False
|
|
102
|
+
IS_PYPY = False
|
|
103
|
+
NumpyUfuncType = None
|
|
104
|
+
NumpyDType = None
|
|
105
|
+
NumpyArrayType = None
|
|
106
|
+
try:
|
|
107
|
+
if not importlib.machinery.PathFinder().find_spec('numpy'):
|
|
108
|
+
raise ImportError("No module named 'numpy'")
|
|
109
|
+
NumpyUfuncType = True
|
|
110
|
+
NumpyDType = True
|
|
111
|
+
NumpyArrayType = True
|
|
112
|
+
except ImportError:
|
|
113
|
+
pass
|
|
114
|
+
def __hook__():
|
|
115
|
+
global NumpyArrayType, NumpyDType, NumpyUfuncType
|
|
116
|
+
from numpy import ufunc as NumpyUfuncType
|
|
117
|
+
from numpy import ndarray as NumpyArrayType
|
|
118
|
+
from numpy import dtype as NumpyDType
|
|
119
|
+
return True
|
|
120
|
+
if NumpyArrayType: # then has numpy
|
|
121
|
+
def ndarraysubclassinstance(obj_type):
|
|
122
|
+
if all((c.__module__, c.__name__) != ('numpy', 'ndarray') for c in obj_type.__mro__):
|
|
123
|
+
return False
|
|
124
|
+
# anything below here is a numpy array (or subclass) instance
|
|
125
|
+
__hook__() # import numpy (so the following works!!!)
|
|
126
|
+
# verify that __reduce__ has not been overridden
|
|
127
|
+
if obj_type.__reduce_ex__ is not NumpyArrayType.__reduce_ex__ \
|
|
128
|
+
or obj_type.__reduce__ is not NumpyArrayType.__reduce__:
|
|
129
|
+
return False
|
|
130
|
+
return True
|
|
131
|
+
def numpyufunc(obj_type):
|
|
132
|
+
return any((c.__module__, c.__name__) == ('numpy', 'ufunc') for c in obj_type.__mro__)
|
|
133
|
+
def numpydtype(obj_type):
|
|
134
|
+
if all((c.__module__, c.__name__) != ('numpy', 'dtype') for c in obj_type.__mro__):
|
|
135
|
+
return False
|
|
136
|
+
# anything below here is a numpy dtype
|
|
137
|
+
__hook__() # import numpy (so the following works!!!)
|
|
138
|
+
return obj_type is type(NumpyDType) # handles subclasses
|
|
139
|
+
else:
|
|
140
|
+
def ndarraysubclassinstance(obj): return False
|
|
141
|
+
def numpyufunc(obj): return False
|
|
142
|
+
def numpydtype(obj): return False
|
|
143
|
+
|
|
144
|
+
from types import GetSetDescriptorType, ClassMethodDescriptorType, \
|
|
145
|
+
WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \
|
|
146
|
+
MethodWrapperType #XXX: unused
|
|
147
|
+
|
|
148
|
+
# make sure to add these 'hand-built' types to _typemap
|
|
149
|
+
CellType = type((lambda x: lambda y: x)(0).__closure__[0])
|
|
150
|
+
PartialType = type(partial(int, base=2))
|
|
151
|
+
SuperType = type(super(Exception, TypeError()))
|
|
152
|
+
ItemGetterType = type(itemgetter(0))
|
|
153
|
+
AttrGetterType = type(attrgetter('__repr__'))
|
|
154
|
+
|
|
155
|
+
try:
|
|
156
|
+
from functools import _lru_cache_wrapper as LRUCacheType
|
|
157
|
+
except ImportError:
|
|
158
|
+
LRUCacheType = None
|
|
159
|
+
|
|
160
|
+
if not isinstance(LRUCacheType, type):
|
|
161
|
+
LRUCacheType = None
|
|
162
|
+
|
|
163
|
+
def get_file_type(*args, **kwargs):
|
|
164
|
+
open = kwargs.pop("open", __builtin__.open)
|
|
165
|
+
f = open(os.devnull, *args, **kwargs)
|
|
166
|
+
t = type(f)
|
|
167
|
+
f.close()
|
|
168
|
+
return t
|
|
169
|
+
|
|
170
|
+
IS_PYODIDE = sys.platform == 'emscripten'
|
|
171
|
+
|
|
172
|
+
FileType = get_file_type('rb', buffering=0)
|
|
173
|
+
TextWrapperType = get_file_type('r', buffering=-1)
|
|
174
|
+
BufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1)
|
|
175
|
+
BufferedReaderType = get_file_type('rb', buffering=-1)
|
|
176
|
+
BufferedWriterType = get_file_type('wb', buffering=-1)
|
|
177
|
+
try:
|
|
178
|
+
from _pyio import open as _open
|
|
179
|
+
PyTextWrapperType = get_file_type('r', buffering=-1, open=_open)
|
|
180
|
+
PyBufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1, open=_open)
|
|
181
|
+
PyBufferedReaderType = get_file_type('rb', buffering=-1, open=_open)
|
|
182
|
+
PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open)
|
|
183
|
+
except ImportError:
|
|
184
|
+
PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None
|
|
185
|
+
from io import BytesIO as StringIO
|
|
186
|
+
InputType = OutputType = None
|
|
187
|
+
from socket import socket as SocketType
|
|
188
|
+
#FIXME: additionally calls ForkingPickler.register several times
|
|
189
|
+
from multiprocessing.reduction import _reduce_socket as reduce_socket
|
|
190
|
+
try: #pragma: no cover
|
|
191
|
+
IS_IPYTHON = __IPYTHON__ # is True
|
|
192
|
+
ExitType = None # IPython.core.autocall.ExitAutocall
|
|
193
|
+
IPYTHON_SINGLETONS = ('exit', 'quit', 'get_ipython')
|
|
194
|
+
except NameError:
|
|
195
|
+
IS_IPYTHON = False
|
|
196
|
+
try: ExitType = type(exit) # apparently 'exit' can be removed
|
|
197
|
+
except NameError: ExitType = None
|
|
198
|
+
IPYTHON_SINGLETONS = ()
|
|
199
|
+
|
|
200
|
+
import inspect
|
|
201
|
+
import typing
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
### Shims for different versions of Python and dill
|
|
205
|
+
class Sentinel(object):
|
|
206
|
+
"""
|
|
207
|
+
Create a unique sentinel object that is pickled as a constant.
|
|
208
|
+
"""
|
|
209
|
+
def __init__(self, name, module_name=None):
|
|
210
|
+
self.name = name
|
|
211
|
+
if module_name is None:
|
|
212
|
+
# Use the calling frame's module
|
|
213
|
+
self.__module__ = inspect.currentframe().f_back.f_globals['__name__']
|
|
214
|
+
else:
|
|
215
|
+
self.__module__ = module_name # pragma: no cover
|
|
216
|
+
def __repr__(self):
|
|
217
|
+
return self.__module__ + '.' + self.name # pragma: no cover
|
|
218
|
+
def __copy__(self):
|
|
219
|
+
return self # pragma: no cover
|
|
220
|
+
def __deepcopy__(self, memo):
|
|
221
|
+
return self # pragma: no cover
|
|
222
|
+
def __reduce__(self):
|
|
223
|
+
return self.name
|
|
224
|
+
def __reduce_ex__(self, protocol):
|
|
225
|
+
return self.name
|
|
226
|
+
|
|
227
|
+
from . import _shims
|
|
228
|
+
from ._shims import Reduce, Getattr
|
|
229
|
+
|
|
230
|
+
### File modes
|
|
231
|
+
#: Pickles the file handle, preserving mode. The position of the unpickled
|
|
232
|
+
#: object is as for a new file handle.
|
|
233
|
+
HANDLE_FMODE = 0
|
|
234
|
+
#: Pickles the file contents, creating a new file if on load the file does
|
|
235
|
+
#: not exist. The position = min(pickled position, EOF) and mode is chosen
|
|
236
|
+
#: as such that "best" preserves behavior of the original file.
|
|
237
|
+
CONTENTS_FMODE = 1
|
|
238
|
+
#: Pickles the entire file (handle and contents), preserving mode and position.
|
|
239
|
+
FILE_FMODE = 2
|
|
240
|
+
|
|
241
|
+
### Shorthands (modified from python2.5/lib/pickle.py)
|
|
242
|
+
def copy(obj, *args, **kwds):
|
|
243
|
+
"""
|
|
244
|
+
Use pickling to 'copy' an object (i.e. `loads(dumps(obj))`).
|
|
245
|
+
|
|
246
|
+
See :func:`dumps` and :func:`loads` for keyword arguments.
|
|
247
|
+
"""
|
|
248
|
+
ignore = kwds.pop('ignore', Unpickler.settings['ignore'])
|
|
249
|
+
return loads(dumps(obj, *args, **kwds), ignore=ignore)
|
|
250
|
+
|
|
251
|
+
def dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
|
|
252
|
+
"""
|
|
253
|
+
Pickle an object to a file.
|
|
254
|
+
|
|
255
|
+
See :func:`dumps` for keyword arguments.
|
|
256
|
+
"""
|
|
257
|
+
from .settings import settings
|
|
258
|
+
protocol = settings['protocol'] if protocol is None else int(protocol)
|
|
259
|
+
_kwds = kwds.copy()
|
|
260
|
+
_kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
|
|
261
|
+
Pickler(file, protocol, **_kwds).dump(obj)
|
|
262
|
+
return
|
|
263
|
+
|
|
264
|
+
def dumps(obj, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
|
|
265
|
+
"""
|
|
266
|
+
Pickle an object to a string.
|
|
267
|
+
|
|
268
|
+
*protocol* is the pickler protocol, as defined for Python *pickle*.
|
|
269
|
+
|
|
270
|
+
If *byref=True*, then dill behaves a lot more like pickle as certain
|
|
271
|
+
objects (like modules) are pickled by reference as opposed to attempting
|
|
272
|
+
to pickle the object itself.
|
|
273
|
+
|
|
274
|
+
If *recurse=True*, then objects referred to in the global dictionary
|
|
275
|
+
are recursively traced and pickled, instead of the default behavior
|
|
276
|
+
of attempting to store the entire global dictionary. This is needed for
|
|
277
|
+
functions defined via *exec()*.
|
|
278
|
+
|
|
279
|
+
*fmode* (:const:`HANDLE_FMODE`, :const:`CONTENTS_FMODE`,
|
|
280
|
+
or :const:`FILE_FMODE`) indicates how file handles will be pickled.
|
|
281
|
+
For example, when pickling a data file handle for transfer to a remote
|
|
282
|
+
compute service, *FILE_FMODE* will include the file contents in the
|
|
283
|
+
pickle and cursor position so that a remote method can operate
|
|
284
|
+
transparently on an object with an open file handle.
|
|
285
|
+
|
|
286
|
+
Default values for keyword arguments can be set in :mod:`dill.settings`.
|
|
287
|
+
"""
|
|
288
|
+
file = StringIO()
|
|
289
|
+
dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio)
|
|
290
|
+
return file.getvalue()
|
|
291
|
+
|
|
292
|
+
def load(file, ignore=None, **kwds):
|
|
293
|
+
"""
|
|
294
|
+
Unpickle an object from a file.
|
|
295
|
+
|
|
296
|
+
See :func:`loads` for keyword arguments.
|
|
297
|
+
"""
|
|
298
|
+
return Unpickler(file, ignore=ignore, **kwds).load()
|
|
299
|
+
|
|
300
|
+
def loads(str, ignore=None, **kwds):
|
|
301
|
+
"""
|
|
302
|
+
Unpickle an object from a string.
|
|
303
|
+
|
|
304
|
+
If *ignore=False* then objects whose class is defined in the module
|
|
305
|
+
*__main__* are updated to reference the existing class in *__main__*,
|
|
306
|
+
otherwise they are left to refer to the reconstructed type, which may
|
|
307
|
+
be different.
|
|
308
|
+
|
|
309
|
+
Default values for keyword arguments can be set in :mod:`dill.settings`.
|
|
310
|
+
"""
|
|
311
|
+
file = StringIO(str)
|
|
312
|
+
return load(file, ignore, **kwds)
|
|
313
|
+
|
|
314
|
+
# def dumpzs(obj, protocol=None):
|
|
315
|
+
# """pickle an object to a compressed string"""
|
|
316
|
+
# return zlib.compress(dumps(obj, protocol))
|
|
317
|
+
|
|
318
|
+
# def loadzs(str):
|
|
319
|
+
# """unpickle an object from a compressed string"""
|
|
320
|
+
# return loads(zlib.decompress(str))
|
|
321
|
+
|
|
322
|
+
### End: Shorthands ###
|
|
323
|
+
|
|
324
|
+
class MetaCatchingDict(dict):
|
|
325
|
+
def get(self, key, default=None):
|
|
326
|
+
try:
|
|
327
|
+
return self[key]
|
|
328
|
+
except KeyError:
|
|
329
|
+
return default
|
|
330
|
+
|
|
331
|
+
def __missing__(self, key):
|
|
332
|
+
if issubclass(key, type):
|
|
333
|
+
return save_type
|
|
334
|
+
else:
|
|
335
|
+
raise KeyError()
|
|
336
|
+
|
|
337
|
+
class PickleWarning(Warning, PickleError):
|
|
338
|
+
pass
|
|
339
|
+
|
|
340
|
+
class PicklingWarning(PickleWarning, PicklingError):
|
|
341
|
+
pass
|
|
342
|
+
|
|
343
|
+
class UnpicklingWarning(PickleWarning, UnpicklingError):
|
|
344
|
+
pass
|
|
345
|
+
|
|
346
|
+
### Extend the Picklers
|
|
347
|
+
class Pickler(StockPickler):
|
|
348
|
+
"""python's Pickler extended to interpreter sessions"""
|
|
349
|
+
dispatch: typing.Dict[type, typing.Callable[[Pickler, typing.Any], None]] \
|
|
350
|
+
= MetaCatchingDict(StockPickler.dispatch.copy())
|
|
351
|
+
"""The dispatch table, a dictionary of serializing functions used
|
|
352
|
+
by Pickler to save objects of specific types. Use :func:`pickle`
|
|
353
|
+
or :func:`register` to associate types to custom functions.
|
|
354
|
+
|
|
355
|
+
:meta hide-value:
|
|
356
|
+
"""
|
|
357
|
+
_session = False
|
|
358
|
+
from .settings import settings
|
|
359
|
+
|
|
360
|
+
def __init__(self, file, *args, **kwds):
|
|
361
|
+
settings = Pickler.settings
|
|
362
|
+
_byref = kwds.pop('byref', None)
|
|
363
|
+
#_strictio = kwds.pop('strictio', None)
|
|
364
|
+
_fmode = kwds.pop('fmode', None)
|
|
365
|
+
_recurse = kwds.pop('recurse', None)
|
|
366
|
+
StockPickler.__init__(self, file, *args, **kwds)
|
|
367
|
+
self._main = _main_module
|
|
368
|
+
self._diff_cache = {}
|
|
369
|
+
self._byref = settings['byref'] if _byref is None else _byref
|
|
370
|
+
self._strictio = False #_strictio
|
|
371
|
+
self._fmode = settings['fmode'] if _fmode is None else _fmode
|
|
372
|
+
self._recurse = settings['recurse'] if _recurse is None else _recurse
|
|
373
|
+
self._postproc = OrderedDict()
|
|
374
|
+
self._file = file
|
|
375
|
+
|
|
376
|
+
def save(self, obj, save_persistent_id=True):
|
|
377
|
+
# numpy hack
|
|
378
|
+
obj_type = type(obj)
|
|
379
|
+
if NumpyArrayType and not (obj_type is type or obj_type in Pickler.dispatch):
|
|
380
|
+
# register if the object is a numpy ufunc
|
|
381
|
+
# thanks to Paul Kienzle for pointing out ufuncs didn't pickle
|
|
382
|
+
if numpyufunc(obj_type):
|
|
383
|
+
@register(obj_type)
|
|
384
|
+
def save_numpy_ufunc(pickler, obj):
|
|
385
|
+
logger.trace(pickler, "Nu: %s", obj)
|
|
386
|
+
name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
|
|
387
|
+
StockPickler.save_global(pickler, obj, name=name)
|
|
388
|
+
logger.trace(pickler, "# Nu")
|
|
389
|
+
return
|
|
390
|
+
# NOTE: the above 'save' performs like:
|
|
391
|
+
# import copy_reg
|
|
392
|
+
# def udump(f): return f.__name__
|
|
393
|
+
# def uload(name): return getattr(numpy, name)
|
|
394
|
+
# copy_reg.pickle(NumpyUfuncType, udump, uload)
|
|
395
|
+
# register if the object is a numpy dtype
|
|
396
|
+
if numpydtype(obj_type):
|
|
397
|
+
@register(obj_type)
|
|
398
|
+
def save_numpy_dtype(pickler, obj):
|
|
399
|
+
logger.trace(pickler, "Dt: %s", obj)
|
|
400
|
+
pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj)
|
|
401
|
+
logger.trace(pickler, "# Dt")
|
|
402
|
+
return
|
|
403
|
+
# NOTE: the above 'save' performs like:
|
|
404
|
+
# import copy_reg
|
|
405
|
+
# def uload(name): return type(NumpyDType(name))
|
|
406
|
+
# def udump(f): return uload, (f.type,)
|
|
407
|
+
# copy_reg.pickle(NumpyDTypeType, udump, uload)
|
|
408
|
+
# register if the object is a subclassed numpy array instance
|
|
409
|
+
if ndarraysubclassinstance(obj_type):
|
|
410
|
+
@register(obj_type)
|
|
411
|
+
def save_numpy_array(pickler, obj):
|
|
412
|
+
logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype)
|
|
413
|
+
npdict = getattr(obj, '__dict__', None)
|
|
414
|
+
f, args, state = obj.__reduce__()
|
|
415
|
+
pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj)
|
|
416
|
+
logger.trace(pickler, "# Nu")
|
|
417
|
+
return
|
|
418
|
+
# end numpy hack
|
|
419
|
+
|
|
420
|
+
if GENERATOR_FAIL and obj_type is GeneratorType:
|
|
421
|
+
msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
|
|
422
|
+
raise PicklingError(msg)
|
|
423
|
+
StockPickler.save(self, obj, save_persistent_id)
|
|
424
|
+
|
|
425
|
+
save.__doc__ = StockPickler.save.__doc__
|
|
426
|
+
|
|
427
|
+
def dump(self, obj): #NOTE: if settings change, need to update attributes
|
|
428
|
+
logger.trace_setup(self)
|
|
429
|
+
StockPickler.dump(self, obj)
|
|
430
|
+
dump.__doc__ = StockPickler.dump.__doc__
|
|
431
|
+
|
|
432
|
+
class Unpickler(StockUnpickler):
|
|
433
|
+
"""python's Unpickler extended to interpreter sessions and more types"""
|
|
434
|
+
from .settings import settings
|
|
435
|
+
_session = False
|
|
436
|
+
|
|
437
|
+
def find_class(self, module, name):
|
|
438
|
+
if (module, name) == ('__builtin__', '__main__'):
|
|
439
|
+
return self._main.__dict__ #XXX: above set w/save_module_dict
|
|
440
|
+
elif (module, name) == ('__builtin__', 'NoneType'):
|
|
441
|
+
return type(None) #XXX: special case: NoneType missing
|
|
442
|
+
if module == 'dill.dill': module = 'dill._dill'
|
|
443
|
+
return StockUnpickler.find_class(self, module, name)
|
|
444
|
+
|
|
445
|
+
def __init__(self, *args, **kwds):
|
|
446
|
+
settings = Pickler.settings
|
|
447
|
+
_ignore = kwds.pop('ignore', None)
|
|
448
|
+
StockUnpickler.__init__(self, *args, **kwds)
|
|
449
|
+
self._main = _main_module
|
|
450
|
+
self._ignore = settings['ignore'] if _ignore is None else _ignore
|
|
451
|
+
|
|
452
|
+
def load(self): #NOTE: if settings change, need to update attributes
|
|
453
|
+
obj = StockUnpickler.load(self)
|
|
454
|
+
if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'):
|
|
455
|
+
if not self._ignore:
|
|
456
|
+
# point obj class to main
|
|
457
|
+
try: obj.__class__ = getattr(self._main, type(obj).__name__)
|
|
458
|
+
except (AttributeError,TypeError): pass # defined in a file
|
|
459
|
+
#_main_module.__dict__.update(obj.__dict__) #XXX: should update globals ?
|
|
460
|
+
return obj
|
|
461
|
+
load.__doc__ = StockUnpickler.load.__doc__
|
|
462
|
+
pass
|
|
463
|
+
|
|
464
|
+
'''
|
|
465
|
+
def dispatch_table():
|
|
466
|
+
"""get the dispatch table of registered types"""
|
|
467
|
+
return Pickler.dispatch
|
|
468
|
+
'''
|
|
469
|
+
|
|
470
|
+
pickle_dispatch_copy = StockPickler.dispatch.copy()
|
|
471
|
+
|
|
472
|
+
def pickle(t, func):
|
|
473
|
+
"""expose :attr:`~Pickler.dispatch` table for user-created extensions"""
|
|
474
|
+
Pickler.dispatch[t] = func
|
|
475
|
+
return
|
|
476
|
+
|
|
477
|
+
def register(t):
|
|
478
|
+
"""decorator to register types to Pickler's :attr:`~Pickler.dispatch` table"""
|
|
479
|
+
def proxy(func):
|
|
480
|
+
Pickler.dispatch[t] = func
|
|
481
|
+
return func
|
|
482
|
+
return proxy
|
|
483
|
+
|
|
484
|
+
def _revert_extension():
|
|
485
|
+
"""drop dill-registered types from pickle's dispatch table"""
|
|
486
|
+
for type, func in list(StockPickler.dispatch.items()):
|
|
487
|
+
if func.__module__ == __name__:
|
|
488
|
+
del StockPickler.dispatch[type]
|
|
489
|
+
if type in pickle_dispatch_copy:
|
|
490
|
+
StockPickler.dispatch[type] = pickle_dispatch_copy[type]
|
|
491
|
+
|
|
492
|
+
def use_diff(on=True):
|
|
493
|
+
"""
|
|
494
|
+
Reduces size of pickles by only including object which have changed.
|
|
495
|
+
|
|
496
|
+
Decreases pickle size but increases CPU time needed.
|
|
497
|
+
Also helps avoid some unpickleable objects.
|
|
498
|
+
MUST be called at start of script, otherwise changes will not be recorded.
|
|
499
|
+
"""
|
|
500
|
+
global _use_diff, diff
|
|
501
|
+
_use_diff = on
|
|
502
|
+
if _use_diff and diff is None:
|
|
503
|
+
try:
|
|
504
|
+
from . import diff as d
|
|
505
|
+
except ImportError:
|
|
506
|
+
import diff as d
|
|
507
|
+
diff = d
|
|
508
|
+
|
|
509
|
+
def _create_typemap():
|
|
510
|
+
import types
|
|
511
|
+
d = dict(list(__builtin__.__dict__.items()) + \
|
|
512
|
+
list(types.__dict__.items())).items()
|
|
513
|
+
for key, value in d:
|
|
514
|
+
if getattr(value, '__module__', None) == 'builtins' \
|
|
515
|
+
and type(value) is type:
|
|
516
|
+
yield key, value
|
|
517
|
+
return
|
|
518
|
+
_reverse_typemap = dict(_create_typemap())
|
|
519
|
+
_reverse_typemap.update({
|
|
520
|
+
'PartialType': PartialType,
|
|
521
|
+
'SuperType': SuperType,
|
|
522
|
+
'ItemGetterType': ItemGetterType,
|
|
523
|
+
'AttrGetterType': AttrGetterType,
|
|
524
|
+
})
|
|
525
|
+
if sys.hexversion < 0x30800a2:
|
|
526
|
+
_reverse_typemap.update({
|
|
527
|
+
'CellType': CellType,
|
|
528
|
+
})
|
|
529
|
+
|
|
530
|
+
# "Incidental" implementation specific types. Unpickling these types in another
|
|
531
|
+
# implementation of Python (PyPy -> CPython) is not guaranteed to work
|
|
532
|
+
|
|
533
|
+
# This dictionary should contain all types that appear in Python implementations
|
|
534
|
+
# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types
|
|
535
|
+
x=OrderedDict()
|
|
536
|
+
_incedental_reverse_typemap = {
|
|
537
|
+
'FileType': FileType,
|
|
538
|
+
'BufferedRandomType': BufferedRandomType,
|
|
539
|
+
'BufferedReaderType': BufferedReaderType,
|
|
540
|
+
'BufferedWriterType': BufferedWriterType,
|
|
541
|
+
'TextWrapperType': TextWrapperType,
|
|
542
|
+
'PyBufferedRandomType': PyBufferedRandomType,
|
|
543
|
+
'PyBufferedReaderType': PyBufferedReaderType,
|
|
544
|
+
'PyBufferedWriterType': PyBufferedWriterType,
|
|
545
|
+
'PyTextWrapperType': PyTextWrapperType,
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
_incedental_reverse_typemap.update({
|
|
549
|
+
"DictKeysType": type({}.keys()),
|
|
550
|
+
"DictValuesType": type({}.values()),
|
|
551
|
+
"DictItemsType": type({}.items()),
|
|
552
|
+
|
|
553
|
+
"OdictKeysType": type(x.keys()),
|
|
554
|
+
"OdictValuesType": type(x.values()),
|
|
555
|
+
"OdictItemsType": type(x.items()),
|
|
556
|
+
})
|
|
557
|
+
|
|
558
|
+
if ExitType:
|
|
559
|
+
_incedental_reverse_typemap['ExitType'] = ExitType
|
|
560
|
+
if InputType:
|
|
561
|
+
_incedental_reverse_typemap['InputType'] = InputType
|
|
562
|
+
_incedental_reverse_typemap['OutputType'] = OutputType
|
|
563
|
+
|
|
564
|
+
'''
|
|
565
|
+
try:
|
|
566
|
+
import symtable
|
|
567
|
+
_incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table)
|
|
568
|
+
except: #FIXME: fails to pickle
|
|
569
|
+
pass
|
|
570
|
+
|
|
571
|
+
if sys.hexversion >= 0x30a00a0:
|
|
572
|
+
_incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines())
|
|
573
|
+
'''
|
|
574
|
+
|
|
575
|
+
if sys.hexversion >= 0x30b00b0 and not IS_PYPY:
|
|
576
|
+
from types import GenericAlias
|
|
577
|
+
_incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,))))
|
|
578
|
+
'''
|
|
579
|
+
_incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions())
|
|
580
|
+
'''
|
|
581
|
+
|
|
582
|
+
try:
|
|
583
|
+
import winreg
|
|
584
|
+
_incedental_reverse_typemap["HKEYType"] = winreg.HKEYType
|
|
585
|
+
except ImportError:
|
|
586
|
+
pass
|
|
587
|
+
|
|
588
|
+
_reverse_typemap.update(_incedental_reverse_typemap)
|
|
589
|
+
_incedental_types = set(_incedental_reverse_typemap.values())
|
|
590
|
+
|
|
591
|
+
del x
|
|
592
|
+
|
|
593
|
+
_typemap = dict((v, k) for k, v in _reverse_typemap.items())
|
|
594
|
+
|
|
595
|
+
def _unmarshal(string):
|
|
596
|
+
return marshal.loads(string)
|
|
597
|
+
|
|
598
|
+
def _load_type(name):
|
|
599
|
+
return _reverse_typemap[name]
|
|
600
|
+
|
|
601
|
+
def _create_type(typeobj, *args):
|
|
602
|
+
return typeobj(*args)
|
|
603
|
+
|
|
604
|
+
def _create_function(fcode, fglobals, fname=None, fdefaults=None,
|
|
605
|
+
fclosure=None, fdict=None, fkwdefaults=None):
|
|
606
|
+
# same as FunctionType, but enable passing __dict__ to new function,
|
|
607
|
+
# __dict__ is the storehouse for attributes added after function creation
|
|
608
|
+
func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure)
|
|
609
|
+
if fdict is not None:
|
|
610
|
+
func.__dict__.update(fdict) #XXX: better copy? option to copy?
|
|
611
|
+
if fkwdefaults is not None:
|
|
612
|
+
func.__kwdefaults__ = fkwdefaults
|
|
613
|
+
# 'recurse' only stores referenced modules/objects in fglobals,
|
|
614
|
+
# thus we need to make sure that we have __builtins__ as well
|
|
615
|
+
if "__builtins__" not in func.__globals__:
|
|
616
|
+
func.__globals__["__builtins__"] = globals()["__builtins__"]
|
|
617
|
+
# assert id(fglobals) == id(func.__globals__)
|
|
618
|
+
return func
|
|
619
|
+
|
|
620
|
+
class match:
|
|
621
|
+
"""
|
|
622
|
+
Make avaialable a limited structural pattern matching-like syntax for Python < 3.10
|
|
623
|
+
|
|
624
|
+
Patterns can be only tuples (without types) currently.
|
|
625
|
+
Inspired by the package pattern-matching-PEP634.
|
|
626
|
+
|
|
627
|
+
Usage:
|
|
628
|
+
>>> with match(args) as m:
|
|
629
|
+
>>> if m.case(('x', 'y')):
|
|
630
|
+
>>> # use m.x and m.y
|
|
631
|
+
>>> elif m.case(('x', 'y', 'z')):
|
|
632
|
+
>>> # use m.x, m.y and m.z
|
|
633
|
+
|
|
634
|
+
Equivalent native code for Python >= 3.10:
|
|
635
|
+
>>> match args:
|
|
636
|
+
>>> case (x, y):
|
|
637
|
+
>>> # use x and y
|
|
638
|
+
>>> case (x, y, z):
|
|
639
|
+
>>> # use x, y and z
|
|
640
|
+
"""
|
|
641
|
+
def __init__(self, value):
|
|
642
|
+
self.value = value
|
|
643
|
+
self._fields = None
|
|
644
|
+
def __enter__(self):
|
|
645
|
+
return self
|
|
646
|
+
def __exit__(self, *exc_info):
|
|
647
|
+
return False
|
|
648
|
+
def case(self, args): # *args, **kwargs):
|
|
649
|
+
"""just handles tuple patterns"""
|
|
650
|
+
if len(self.value) != len(args): # + len(kwargs):
|
|
651
|
+
return False
|
|
652
|
+
#if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())):
|
|
653
|
+
# return False
|
|
654
|
+
self.args = args # (*args, *kwargs)
|
|
655
|
+
return True
|
|
656
|
+
@property
|
|
657
|
+
def fields(self):
|
|
658
|
+
# Only bind names to values if necessary.
|
|
659
|
+
if self._fields is None:
|
|
660
|
+
self._fields = dict(zip(self.args, self.value))
|
|
661
|
+
return self._fields
|
|
662
|
+
def __getattr__(self, item):
|
|
663
|
+
return self.fields[item]
|
|
664
|
+
|
|
665
|
+
ALL_CODE_PARAMS = [
|
|
666
|
+
# Version New attribute CodeType parameters
|
|
667
|
+
((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'),
|
|
668
|
+
((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'),
|
|
669
|
+
((3,11,'p'), 'co_qualname', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable freevars cellvars'),
|
|
670
|
+
((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'),
|
|
671
|
+
((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
|
|
672
|
+
((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
|
|
673
|
+
]
|
|
674
|
+
for version, new_attr, params in ALL_CODE_PARAMS:
|
|
675
|
+
if hasattr(CodeType, new_attr):
|
|
676
|
+
CODE_VERSION = version
|
|
677
|
+
CODE_PARAMS = params.split()
|
|
678
|
+
break
|
|
679
|
+
ENCODE_PARAMS = set(CODE_PARAMS).intersection(
|
|
680
|
+
['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable'])
|
|
681
|
+
|
|
682
|
+
def _create_code(*args):
|
|
683
|
+
if not isinstance(args[0], int): # co_lnotab stored from >= 3.10
|
|
684
|
+
LNOTAB, *args = args
|
|
685
|
+
else: # from < 3.10 (or pre-LNOTAB storage)
|
|
686
|
+
LNOTAB = b''
|
|
687
|
+
|
|
688
|
+
with match(args) as m:
|
|
689
|
+
# Python 3.11/3.12a (18 members)
|
|
690
|
+
if m.case((
|
|
691
|
+
'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
|
|
692
|
+
'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
|
|
693
|
+
'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
|
|
694
|
+
)):
|
|
695
|
+
if CODE_VERSION == (3,11):
|
|
696
|
+
return CodeType(
|
|
697
|
+
*args[:6],
|
|
698
|
+
args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
|
|
699
|
+
*args[7:14],
|
|
700
|
+
args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
|
|
701
|
+
args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable
|
|
702
|
+
args[16],
|
|
703
|
+
args[17],
|
|
704
|
+
)
|
|
705
|
+
fields = m.fields
|
|
706
|
+
# PyPy 3.11 7.3.19+ (17 members)
|
|
707
|
+
elif m.case((
|
|
708
|
+
'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
|
|
709
|
+
'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', # args[6:13]
|
|
710
|
+
'firstlineno', 'linetable', 'freevars', 'cellvars' # args[13:]
|
|
711
|
+
)):
|
|
712
|
+
if CODE_VERSION == (3,11,'p'):
|
|
713
|
+
return CodeType(
|
|
714
|
+
*args[:6],
|
|
715
|
+
args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
|
|
716
|
+
*args[7:14],
|
|
717
|
+
args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
|
|
718
|
+
args[15],
|
|
719
|
+
args[16],
|
|
720
|
+
)
|
|
721
|
+
fields = m.fields
|
|
722
|
+
# Python 3.10 or 3.8/3.9 (16 members)
|
|
723
|
+
elif m.case((
|
|
724
|
+
'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
|
|
725
|
+
'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13]
|
|
726
|
+
'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:]
|
|
727
|
+
)):
|
|
728
|
+
if CODE_VERSION == (3,10) or CODE_VERSION == (3,8):
|
|
729
|
+
return CodeType(
|
|
730
|
+
*args[:6],
|
|
731
|
+
args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
|
|
732
|
+
*args[7:13],
|
|
733
|
+
args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable
|
|
734
|
+
args[14],
|
|
735
|
+
args[15],
|
|
736
|
+
)
|
|
737
|
+
fields = m.fields
|
|
738
|
+
if CODE_VERSION >= (3,10):
|
|
739
|
+
fields['linetable'] = m.LNOTAB_OR_LINETABLE
|
|
740
|
+
else:
|
|
741
|
+
fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE
|
|
742
|
+
# Python 3.7 (15 args)
|
|
743
|
+
elif m.case((
|
|
744
|
+
'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5]
|
|
745
|
+
'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12]
|
|
746
|
+
'lnotab', 'freevars', 'cellvars' # args[12:]
|
|
747
|
+
)):
|
|
748
|
+
if CODE_VERSION == (3,7):
|
|
749
|
+
return CodeType(
|
|
750
|
+
*args[:5],
|
|
751
|
+
args[5].encode() if hasattr(args[5], 'encode') else args[5], # code
|
|
752
|
+
*args[6:12],
|
|
753
|
+
args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab
|
|
754
|
+
args[13],
|
|
755
|
+
args[14],
|
|
756
|
+
)
|
|
757
|
+
fields = m.fields
|
|
758
|
+
# Python 3.11a (20 members)
|
|
759
|
+
elif m.case((
|
|
760
|
+
'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
|
|
761
|
+
'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
|
|
762
|
+
'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
|
|
763
|
+
)):
|
|
764
|
+
if CODE_VERSION == (3,11,'a'):
|
|
765
|
+
return CodeType(
|
|
766
|
+
*args[:6],
|
|
767
|
+
args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
|
|
768
|
+
*args[7:14],
|
|
769
|
+
*(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable
|
|
770
|
+
args[18],
|
|
771
|
+
args[19],
|
|
772
|
+
)
|
|
773
|
+
fields = m.fields
|
|
774
|
+
else:
|
|
775
|
+
raise UnpicklingError("pattern match for code object failed")
|
|
776
|
+
|
|
777
|
+
# The args format doesn't match this version.
|
|
778
|
+
fields.setdefault('posonlyargcount', 0) # from python <= 3.7
|
|
779
|
+
fields.setdefault('lnotab', LNOTAB) # from python >= 3.10
|
|
780
|
+
fields.setdefault('linetable', b'') # from python <= 3.9
|
|
781
|
+
fields.setdefault('qualname', fields['name']) # from python <= 3.10
|
|
782
|
+
fields.setdefault('exceptiontable', b'') # from python <= 3.10
|
|
783
|
+
fields.setdefault('endlinetable', None) # from python != 3.11a
|
|
784
|
+
fields.setdefault('columntable', None) # from python != 3.11a
|
|
785
|
+
|
|
786
|
+
args = (fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k]
|
|
787
|
+
for k in CODE_PARAMS)
|
|
788
|
+
return CodeType(*args)
|
|
789
|
+
|
|
790
|
+
def _create_ftype(ftypeobj, func, args, kwds):
|
|
791
|
+
if kwds is None:
|
|
792
|
+
kwds = {}
|
|
793
|
+
if args is None:
|
|
794
|
+
args = ()
|
|
795
|
+
return ftypeobj(func, *args, **kwds)
|
|
796
|
+
|
|
797
|
+
def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245
|
|
798
|
+
if not argz:
|
|
799
|
+
return typing.Tuple[()].copy_with(())
|
|
800
|
+
if argz == ((),):
|
|
801
|
+
return typing.Tuple[()]
|
|
802
|
+
return typing.Tuple[argz]
|
|
803
|
+
|
|
804
|
+
if ThreadHandleType:
|
|
805
|
+
def _create_thread_handle(ident, done, *args): #XXX: ignores 'blocking'
|
|
806
|
+
from threading import _make_thread_handle
|
|
807
|
+
handle = _make_thread_handle(ident)
|
|
808
|
+
if done:
|
|
809
|
+
handle._set_done()
|
|
810
|
+
return handle
|
|
811
|
+
|
|
812
|
+
def _create_lock(locked, *args): #XXX: ignores 'blocking'
|
|
813
|
+
from threading import Lock
|
|
814
|
+
lock = Lock()
|
|
815
|
+
if locked:
|
|
816
|
+
if not lock.acquire(False):
|
|
817
|
+
raise UnpicklingError("Cannot acquire lock")
|
|
818
|
+
return lock
|
|
819
|
+
|
|
820
|
+
def _create_rlock(count, owner, *args): #XXX: ignores 'blocking'
|
|
821
|
+
lock = RLockType()
|
|
822
|
+
if owner is not None:
|
|
823
|
+
lock._acquire_restore((count, owner))
|
|
824
|
+
if owner and not lock._is_owned():
|
|
825
|
+
raise UnpicklingError("Cannot acquire lock")
|
|
826
|
+
return lock
|
|
827
|
+
|
|
828
|
+
# thanks to matsjoyce for adding all the different file modes
|
|
829
|
+
def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0
|
|
830
|
+
# only pickles the handle, not the file contents... good? or StringIO(data)?
|
|
831
|
+
# (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
|
|
832
|
+
# NOTE: handle special cases first (are there more special cases?)
|
|
833
|
+
names = {'<stdin>':sys.__stdin__, '<stdout>':sys.__stdout__,
|
|
834
|
+
'<stderr>':sys.__stderr__} #XXX: better fileno=(0,1,2) ?
|
|
835
|
+
if name in list(names.keys()):
|
|
836
|
+
f = names[name] #XXX: safer "f=sys.stdin"
|
|
837
|
+
elif name == '<tmpfile>':
|
|
838
|
+
f = os.tmpfile()
|
|
839
|
+
elif name == '<fdopen>':
|
|
840
|
+
import tempfile
|
|
841
|
+
f = tempfile.TemporaryFile(mode)
|
|
842
|
+
else:
|
|
843
|
+
try:
|
|
844
|
+
exists = os.path.exists(name)
|
|
845
|
+
except Exception:
|
|
846
|
+
exists = False
|
|
847
|
+
if not exists:
|
|
848
|
+
if strictio:
|
|
849
|
+
raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name)
|
|
850
|
+
elif "r" in mode and fmode != FILE_FMODE:
|
|
851
|
+
name = '<fdopen>' # or os.devnull?
|
|
852
|
+
current_size = 0 # or maintain position?
|
|
853
|
+
else:
|
|
854
|
+
current_size = os.path.getsize(name)
|
|
855
|
+
|
|
856
|
+
if position > current_size:
|
|
857
|
+
if strictio:
|
|
858
|
+
raise ValueError("invalid buffer size")
|
|
859
|
+
elif fmode == CONTENTS_FMODE:
|
|
860
|
+
position = current_size
|
|
861
|
+
# try to open the file by name
|
|
862
|
+
# NOTE: has different fileno
|
|
863
|
+
try:
|
|
864
|
+
#FIXME: missing: *buffering*, encoding, softspace
|
|
865
|
+
if fmode == FILE_FMODE:
|
|
866
|
+
f = open(name, mode if "w" in mode else "w")
|
|
867
|
+
f.write(fdata)
|
|
868
|
+
if "w" not in mode:
|
|
869
|
+
f.close()
|
|
870
|
+
f = open(name, mode)
|
|
871
|
+
elif name == '<fdopen>': # file did not exist
|
|
872
|
+
import tempfile
|
|
873
|
+
f = tempfile.TemporaryFile(mode)
|
|
874
|
+
# treat x mode as w mode
|
|
875
|
+
elif fmode == CONTENTS_FMODE \
|
|
876
|
+
and ("w" in mode or "x" in mode):
|
|
877
|
+
# stop truncation when opening
|
|
878
|
+
flags = os.O_CREAT
|
|
879
|
+
if "+" in mode:
|
|
880
|
+
flags |= os.O_RDWR
|
|
881
|
+
else:
|
|
882
|
+
flags |= os.O_WRONLY
|
|
883
|
+
f = os.fdopen(os.open(name, flags), mode)
|
|
884
|
+
# set name to the correct value
|
|
885
|
+
r = getattr(f, "buffer", f)
|
|
886
|
+
r = getattr(r, "raw", r)
|
|
887
|
+
r.name = name
|
|
888
|
+
assert f.name == name
|
|
889
|
+
else:
|
|
890
|
+
f = open(name, mode)
|
|
891
|
+
except (IOError, FileNotFoundError):
|
|
892
|
+
err = sys.exc_info()[1]
|
|
893
|
+
raise UnpicklingError(err)
|
|
894
|
+
if closed:
|
|
895
|
+
f.close()
|
|
896
|
+
elif position >= 0 and fmode != HANDLE_FMODE:
|
|
897
|
+
f.seek(position)
|
|
898
|
+
return f
|
|
899
|
+
|
|
900
|
+
def _create_stringi(value, position, closed):
|
|
901
|
+
f = StringIO(value)
|
|
902
|
+
if closed: f.close()
|
|
903
|
+
else: f.seek(position)
|
|
904
|
+
return f
|
|
905
|
+
|
|
906
|
+
def _create_stringo(value, position, closed):
|
|
907
|
+
f = StringIO()
|
|
908
|
+
if closed: f.close()
|
|
909
|
+
else:
|
|
910
|
+
f.write(value)
|
|
911
|
+
f.seek(position)
|
|
912
|
+
return f
|
|
913
|
+
|
|
914
|
+
class _itemgetter_helper(object):
|
|
915
|
+
def __init__(self):
|
|
916
|
+
self.items = []
|
|
917
|
+
def __getitem__(self, item):
|
|
918
|
+
self.items.append(item)
|
|
919
|
+
return
|
|
920
|
+
|
|
921
|
+
class _attrgetter_helper(object):
|
|
922
|
+
def __init__(self, attrs, index=None):
|
|
923
|
+
self.attrs = attrs
|
|
924
|
+
self.index = index
|
|
925
|
+
def __getattribute__(self, attr):
|
|
926
|
+
attrs = object.__getattribute__(self, "attrs")
|
|
927
|
+
index = object.__getattribute__(self, "index")
|
|
928
|
+
if index is None:
|
|
929
|
+
index = len(attrs)
|
|
930
|
+
attrs.append(attr)
|
|
931
|
+
else:
|
|
932
|
+
attrs[index] = ".".join([attrs[index], attr])
|
|
933
|
+
return type(self)(attrs, index)
|
|
934
|
+
|
|
935
|
+
class _dictproxy_helper(dict):
|
|
936
|
+
def __ror__(self, a):
|
|
937
|
+
return a
|
|
938
|
+
|
|
939
|
+
_dictproxy_helper_instance = _dictproxy_helper()
|
|
940
|
+
|
|
941
|
+
__d = {}
|
|
942
|
+
try:
|
|
943
|
+
# In CPython 3.9 and later, this trick can be used to exploit the
|
|
944
|
+
# implementation of the __or__ function of MappingProxyType to get the true
|
|
945
|
+
# mapping referenced by the proxy. It may work for other implementations,
|
|
946
|
+
# but is not guaranteed.
|
|
947
|
+
MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance)
|
|
948
|
+
except Exception:
|
|
949
|
+
MAPPING_PROXY_TRICK = False
|
|
950
|
+
del __d
|
|
951
|
+
|
|
952
|
+
# _CELL_REF and _CELL_EMPTY are used to stay compatible with versions of dill
|
|
953
|
+
# whose _create_cell functions do not have a default value.
|
|
954
|
+
# _CELL_REF can be safely removed entirely (replaced by empty tuples for calls
|
|
955
|
+
# to _create_cell) once breaking changes are allowed.
|
|
956
|
+
_CELL_REF = None
|
|
957
|
+
_CELL_EMPTY = Sentinel('_CELL_EMPTY')
|
|
958
|
+
|
|
959
|
+
def _create_cell(contents=None):
|
|
960
|
+
if contents is not _CELL_EMPTY:
|
|
961
|
+
value = contents
|
|
962
|
+
return (lambda: value).__closure__[0]
|
|
963
|
+
|
|
964
|
+
def _create_weakref(obj, *args):
|
|
965
|
+
from weakref import ref
|
|
966
|
+
if obj is None: # it's dead
|
|
967
|
+
from collections import UserDict
|
|
968
|
+
return ref(UserDict(), *args)
|
|
969
|
+
return ref(obj, *args)
|
|
970
|
+
|
|
971
|
+
def _create_weakproxy(obj, callable=False, *args):
|
|
972
|
+
from weakref import proxy
|
|
973
|
+
if obj is None: # it's dead
|
|
974
|
+
if callable: return proxy(lambda x:x, *args)
|
|
975
|
+
from collections import UserDict
|
|
976
|
+
return proxy(UserDict(), *args)
|
|
977
|
+
return proxy(obj, *args)
|
|
978
|
+
|
|
979
|
+
def _eval_repr(repr_str):
|
|
980
|
+
return eval(repr_str)
|
|
981
|
+
|
|
982
|
+
def _create_array(f, args, state, npdict=None):
|
|
983
|
+
#array = numpy.core.multiarray._reconstruct(*args)
|
|
984
|
+
array = f(*args)
|
|
985
|
+
array.__setstate__(state)
|
|
986
|
+
if npdict is not None: # we also have saved state in __dict__
|
|
987
|
+
array.__dict__.update(npdict)
|
|
988
|
+
return array
|
|
989
|
+
|
|
990
|
+
def _create_dtypemeta(scalar_type):
|
|
991
|
+
if NumpyDType is True: __hook__() # a bit hacky I think
|
|
992
|
+
if scalar_type is None:
|
|
993
|
+
return NumpyDType
|
|
994
|
+
return type(NumpyDType(scalar_type))
|
|
995
|
+
|
|
996
|
+
def _create_namedtuple(name, fieldnames, modulename, defaults=None):
|
|
997
|
+
class_ = _import_module(modulename + '.' + name, safe=True)
|
|
998
|
+
if class_ is not None:
|
|
999
|
+
return class_
|
|
1000
|
+
import collections
|
|
1001
|
+
t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename)
|
|
1002
|
+
return t
|
|
1003
|
+
|
|
1004
|
+
def _create_capsule(pointer, name, context, destructor):
|
|
1005
|
+
attr_found = False
|
|
1006
|
+
try:
|
|
1007
|
+
# based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231
|
|
1008
|
+
uname = name.decode('utf8')
|
|
1009
|
+
for i in range(1, uname.count('.')+1):
|
|
1010
|
+
names = uname.rsplit('.', i)
|
|
1011
|
+
try:
|
|
1012
|
+
module = __import__(names[0])
|
|
1013
|
+
except ImportError:
|
|
1014
|
+
pass
|
|
1015
|
+
obj = module
|
|
1016
|
+
for attr in names[1:]:
|
|
1017
|
+
obj = getattr(obj, attr)
|
|
1018
|
+
capsule = obj
|
|
1019
|
+
attr_found = True
|
|
1020
|
+
break
|
|
1021
|
+
except Exception:
|
|
1022
|
+
pass
|
|
1023
|
+
|
|
1024
|
+
if attr_found:
|
|
1025
|
+
if _PyCapsule_IsValid(capsule, name):
|
|
1026
|
+
return capsule
|
|
1027
|
+
raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name))
|
|
1028
|
+
else:
|
|
1029
|
+
#warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning)
|
|
1030
|
+
capsule = _PyCapsule_New(pointer, name, destructor)
|
|
1031
|
+
_PyCapsule_SetContext(capsule, context)
|
|
1032
|
+
return capsule
|
|
1033
|
+
|
|
1034
|
+
def _getattr(objclass, name, repr_str):
|
|
1035
|
+
# hack to grab the reference directly
|
|
1036
|
+
try: #XXX: works only for __builtin__ ?
|
|
1037
|
+
attr = repr_str.split("'")[3]
|
|
1038
|
+
return eval(attr+'.__dict__["'+name+'"]')
|
|
1039
|
+
except Exception:
|
|
1040
|
+
try:
|
|
1041
|
+
attr = objclass.__dict__
|
|
1042
|
+
if type(attr) is DictProxyType:
|
|
1043
|
+
if sys.hexversion > 0x30f00a0 and name in ('__weakref__','__dict__'):
|
|
1044
|
+
attr = _dictproxy_helper.__dict__[name]
|
|
1045
|
+
else:
|
|
1046
|
+
attr = attr[name]
|
|
1047
|
+
else:
|
|
1048
|
+
attr = getattr(objclass,name)
|
|
1049
|
+
except (AttributeError, KeyError):
|
|
1050
|
+
attr = getattr(objclass,name)
|
|
1051
|
+
return attr
|
|
1052
|
+
|
|
1053
|
+
def _get_attr(self, name):
|
|
1054
|
+
# stop recursive pickling
|
|
1055
|
+
return getattr(self, name, None) or getattr(__builtin__, name)
|
|
1056
|
+
|
|
1057
|
+
def _import_module(import_name, safe=False):
|
|
1058
|
+
try:
|
|
1059
|
+
if import_name.startswith('__runtime__.'):
|
|
1060
|
+
return sys.modules[import_name]
|
|
1061
|
+
elif '.' in import_name:
|
|
1062
|
+
items = import_name.split('.')
|
|
1063
|
+
module = '.'.join(items[:-1])
|
|
1064
|
+
obj = items[-1]
|
|
1065
|
+
submodule = getattr(__import__(module, None, None, [obj]), obj)
|
|
1066
|
+
if isinstance(submodule, (ModuleType, type)):
|
|
1067
|
+
return submodule
|
|
1068
|
+
return __import__(import_name, None, None, [obj])
|
|
1069
|
+
else:
|
|
1070
|
+
return __import__(import_name)
|
|
1071
|
+
except (ImportError, AttributeError, KeyError):
|
|
1072
|
+
if safe:
|
|
1073
|
+
return None
|
|
1074
|
+
raise
|
|
1075
|
+
|
|
1076
|
+
# https://github.com/python/cpython/blob/a8912a0f8d9eba6d502c37d522221f9933e976db/Lib/pickle.py#L322-L333
|
|
1077
|
+
def _getattribute(obj, name):
|
|
1078
|
+
for subpath in name.split('.'):
|
|
1079
|
+
if subpath == '<locals>':
|
|
1080
|
+
raise AttributeError("Can't get local attribute {!r} on {!r}"
|
|
1081
|
+
.format(name, obj))
|
|
1082
|
+
try:
|
|
1083
|
+
parent = obj
|
|
1084
|
+
obj = getattr(obj, subpath)
|
|
1085
|
+
except AttributeError:
|
|
1086
|
+
raise AttributeError("Can't get attribute {!r} on {!r}"
|
|
1087
|
+
.format(name, obj))
|
|
1088
|
+
return obj, parent
|
|
1089
|
+
|
|
1090
|
+
def _locate_function(obj, pickler=None):
|
|
1091
|
+
module_name = getattr(obj, '__module__', None)
|
|
1092
|
+
if module_name in ['__main__', None] or \
|
|
1093
|
+
pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__:
|
|
1094
|
+
return False
|
|
1095
|
+
if hasattr(obj, '__qualname__'):
|
|
1096
|
+
module = _import_module(module_name, safe=True)
|
|
1097
|
+
try:
|
|
1098
|
+
found, _ = _getattribute(module, obj.__qualname__)
|
|
1099
|
+
return found is obj
|
|
1100
|
+
except AttributeError:
|
|
1101
|
+
return False
|
|
1102
|
+
else:
|
|
1103
|
+
found = _import_module(module_name + '.' + obj.__name__, safe=True)
|
|
1104
|
+
return found is obj
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def _setitems(dest, source):
|
|
1108
|
+
for k, v in source.items():
|
|
1109
|
+
dest[k] = v
|
|
1110
|
+
|
|
1111
|
+
|
|
1112
|
+
def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO_DEFAULT, postproc_list=None):
|
|
1113
|
+
if obj is Getattr.NO_DEFAULT:
|
|
1114
|
+
obj = Reduce(reduction) # pragma: no cover
|
|
1115
|
+
|
|
1116
|
+
if is_pickler_dill is None:
|
|
1117
|
+
is_pickler_dill = is_dill(pickler, child=True)
|
|
1118
|
+
if is_pickler_dill:
|
|
1119
|
+
# assert id(obj) not in pickler._postproc, str(obj) + ' already pushed on stack!'
|
|
1120
|
+
# if not hasattr(pickler, 'x'): pickler.x = 0
|
|
1121
|
+
# print(pickler.x*' ', 'push', obj, id(obj), pickler._recurse)
|
|
1122
|
+
# pickler.x += 1
|
|
1123
|
+
if postproc_list is None:
|
|
1124
|
+
postproc_list = []
|
|
1125
|
+
|
|
1126
|
+
# Recursive object not supported. Default to a global instead.
|
|
1127
|
+
if id(obj) in pickler._postproc:
|
|
1128
|
+
name = '%s.%s ' % (obj.__module__, getattr(obj, '__qualname__', obj.__name__)) if hasattr(obj, '__module__') else ''
|
|
1129
|
+
warnings.warn('Cannot pickle %r: %shas recursive self-references that trigger a RecursionError.' % (obj, name), PicklingWarning)
|
|
1130
|
+
pickler.save_global(obj)
|
|
1131
|
+
return
|
|
1132
|
+
pickler._postproc[id(obj)] = postproc_list
|
|
1133
|
+
|
|
1134
|
+
# TODO: Use state_setter in Python 3.8 to allow for faster cPickle implementations
|
|
1135
|
+
pickler.save_reduce(*reduction, obj=obj)
|
|
1136
|
+
|
|
1137
|
+
if is_pickler_dill:
|
|
1138
|
+
# pickler.x -= 1
|
|
1139
|
+
# print(pickler.x*' ', 'pop', obj, id(obj))
|
|
1140
|
+
postproc = pickler._postproc.pop(id(obj))
|
|
1141
|
+
# assert postproc_list == postproc, 'Stack tampered!'
|
|
1142
|
+
for reduction in reversed(postproc):
|
|
1143
|
+
if reduction[0] is _setitems:
|
|
1144
|
+
# use the internal machinery of pickle.py to speedup when
|
|
1145
|
+
# updating a dictionary in postproc
|
|
1146
|
+
dest, source = reduction[1]
|
|
1147
|
+
if source:
|
|
1148
|
+
pickler.write(pickler.get(pickler.memo[id(dest)][0]))
|
|
1149
|
+
if sys.hexversion < 0x30e00a1:
|
|
1150
|
+
pickler._batch_setitems(iter(source.items()))
|
|
1151
|
+
else:
|
|
1152
|
+
pickler._batch_setitems(iter(source.items()), obj=obj)
|
|
1153
|
+
else:
|
|
1154
|
+
# Updating with an empty dictionary. Same as doing nothing.
|
|
1155
|
+
continue
|
|
1156
|
+
else:
|
|
1157
|
+
pickler.save_reduce(*reduction)
|
|
1158
|
+
# pop None created by calling preprocessing step off stack
|
|
1159
|
+
pickler.write(POP)
|
|
1160
|
+
|
|
1161
|
+
#@register(CodeType)
|
|
1162
|
+
#def save_code(pickler, obj):
|
|
1163
|
+
# logger.trace(pickler, "Co: %s", obj)
|
|
1164
|
+
# pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj)
|
|
1165
|
+
# logger.trace(pickler, "# Co")
|
|
1166
|
+
# return
|
|
1167
|
+
|
|
1168
|
+
# The following function is based on 'save_codeobject' from 'cloudpickle'
|
|
1169
|
+
# Copyright (c) 2012, Regents of the University of California.
|
|
1170
|
+
# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
|
|
1171
|
+
# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
|
|
1172
|
+
@register(CodeType)
|
|
1173
|
+
def save_code(pickler, obj):
|
|
1174
|
+
logger.trace(pickler, "Co: %s", obj)
|
|
1175
|
+
if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
|
|
1176
|
+
args = (
|
|
1177
|
+
obj.co_lnotab, # for < python 3.10 [not counted in args]
|
|
1178
|
+
obj.co_argcount, obj.co_posonlyargcount,
|
|
1179
|
+
obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
|
|
1180
|
+
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
|
|
1181
|
+
obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
|
|
1182
|
+
obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable,
|
|
1183
|
+
obj.co_columntable, obj.co_exceptiontable, obj.co_freevars,
|
|
1184
|
+
obj.co_cellvars
|
|
1185
|
+
)
|
|
1186
|
+
elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
|
|
1187
|
+
with warnings.catch_warnings():
|
|
1188
|
+
if not OLD312a7: # issue 597
|
|
1189
|
+
warnings.filterwarnings('ignore', category=DeprecationWarning)
|
|
1190
|
+
args = (
|
|
1191
|
+
obj.co_lnotab, # for < python 3.10 [not counted in args]
|
|
1192
|
+
obj.co_argcount, obj.co_posonlyargcount,
|
|
1193
|
+
obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
|
|
1194
|
+
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
|
|
1195
|
+
obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
|
|
1196
|
+
obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable,
|
|
1197
|
+
obj.co_freevars, obj.co_cellvars
|
|
1198
|
+
)
|
|
1199
|
+
elif hasattr(obj, "co_qualname"): # pypy 3.11 7.3.19+ (17 args)
|
|
1200
|
+
args = (
|
|
1201
|
+
obj.co_lnotab, obj.co_argcount, obj.co_posonlyargcount,
|
|
1202
|
+
obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
|
|
1203
|
+
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
|
|
1204
|
+
obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
|
|
1205
|
+
obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
|
|
1206
|
+
obj.co_cellvars
|
|
1207
|
+
)
|
|
1208
|
+
elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
|
|
1209
|
+
args = (
|
|
1210
|
+
obj.co_lnotab, # for < python 3.10 [not counted in args]
|
|
1211
|
+
obj.co_argcount, obj.co_posonlyargcount,
|
|
1212
|
+
obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
|
|
1213
|
+
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
|
|
1214
|
+
obj.co_varnames, obj.co_filename, obj.co_name,
|
|
1215
|
+
obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
|
|
1216
|
+
obj.co_cellvars
|
|
1217
|
+
)
|
|
1218
|
+
elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
|
|
1219
|
+
args = (
|
|
1220
|
+
obj.co_argcount, obj.co_posonlyargcount,
|
|
1221
|
+
obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
|
|
1222
|
+
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
|
|
1223
|
+
obj.co_varnames, obj.co_filename, obj.co_name,
|
|
1224
|
+
obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
|
|
1225
|
+
obj.co_cellvars
|
|
1226
|
+
)
|
|
1227
|
+
else: # python 3.7 (15 args)
|
|
1228
|
+
args = (
|
|
1229
|
+
obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
|
|
1230
|
+
obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
|
|
1231
|
+
obj.co_names, obj.co_varnames, obj.co_filename,
|
|
1232
|
+
obj.co_name, obj.co_firstlineno, obj.co_lnotab,
|
|
1233
|
+
obj.co_freevars, obj.co_cellvars
|
|
1234
|
+
)
|
|
1235
|
+
|
|
1236
|
+
pickler.save_reduce(_create_code, args, obj=obj)
|
|
1237
|
+
logger.trace(pickler, "# Co")
|
|
1238
|
+
return
|
|
1239
|
+
|
|
1240
|
+
def _repr_dict(obj):
|
|
1241
|
+
"""Make a short string representation of a dictionary."""
|
|
1242
|
+
return "<%s object at %#012x>" % (type(obj).__name__, id(obj))
|
|
1243
|
+
|
|
1244
|
+
@register(dict)
|
|
1245
|
+
def save_module_dict(pickler, obj):
|
|
1246
|
+
if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \
|
|
1247
|
+
not (pickler._session and pickler._first_pass):
|
|
1248
|
+
logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj
|
|
1249
|
+
pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8'))
|
|
1250
|
+
logger.trace(pickler, "# D1")
|
|
1251
|
+
elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__):
|
|
1252
|
+
logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj
|
|
1253
|
+
pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general?
|
|
1254
|
+
logger.trace(pickler, "# D3")
|
|
1255
|
+
elif '__name__' in obj and obj != _main_module.__dict__ \
|
|
1256
|
+
and type(obj['__name__']) is str \
|
|
1257
|
+
and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None):
|
|
1258
|
+
logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj
|
|
1259
|
+
pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8'))
|
|
1260
|
+
logger.trace(pickler, "# D4")
|
|
1261
|
+
else:
|
|
1262
|
+
logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj
|
|
1263
|
+
if is_dill(pickler, child=False) and pickler._session:
|
|
1264
|
+
# we only care about session the first pass thru
|
|
1265
|
+
pickler._first_pass = False
|
|
1266
|
+
StockPickler.save_dict(pickler, obj)
|
|
1267
|
+
logger.trace(pickler, "# D2")
|
|
1268
|
+
return
|
|
1269
|
+
|
|
1270
|
+
|
|
1271
|
+
if not OLD310 and MAPPING_PROXY_TRICK:
|
|
1272
|
+
def save_dict_view(dicttype):
|
|
1273
|
+
def save_dict_view_for_function(func):
|
|
1274
|
+
def _save_dict_view(pickler, obj):
|
|
1275
|
+
logger.trace(pickler, "Dkvi: <%s>", obj)
|
|
1276
|
+
mapping = obj.mapping | _dictproxy_helper_instance
|
|
1277
|
+
pickler.save_reduce(func, (mapping,), obj=obj)
|
|
1278
|
+
logger.trace(pickler, "# Dkvi")
|
|
1279
|
+
return _save_dict_view
|
|
1280
|
+
return [
|
|
1281
|
+
(funcname, save_dict_view_for_function(getattr(dicttype, funcname)))
|
|
1282
|
+
for funcname in ('keys', 'values', 'items')
|
|
1283
|
+
]
|
|
1284
|
+
else:
|
|
1285
|
+
# The following functions are based on 'cloudpickle'
|
|
1286
|
+
# https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L922-L940
|
|
1287
|
+
# Copyright (c) 2012, Regents of the University of California.
|
|
1288
|
+
# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
|
|
1289
|
+
# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
|
|
1290
|
+
def save_dict_view(dicttype):
|
|
1291
|
+
def save_dict_keys(pickler, obj):
|
|
1292
|
+
logger.trace(pickler, "Dk: <%s>", obj)
|
|
1293
|
+
dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),))
|
|
1294
|
+
pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj)
|
|
1295
|
+
logger.trace(pickler, "# Dk")
|
|
1296
|
+
|
|
1297
|
+
def save_dict_values(pickler, obj):
|
|
1298
|
+
logger.trace(pickler, "Dv: <%s>", obj)
|
|
1299
|
+
dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),))
|
|
1300
|
+
pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj)
|
|
1301
|
+
logger.trace(pickler, "# Dv")
|
|
1302
|
+
|
|
1303
|
+
def save_dict_items(pickler, obj):
|
|
1304
|
+
logger.trace(pickler, "Di: <%s>", obj)
|
|
1305
|
+
pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj)
|
|
1306
|
+
logger.trace(pickler, "# Di")
|
|
1307
|
+
|
|
1308
|
+
return (
|
|
1309
|
+
('keys', save_dict_keys),
|
|
1310
|
+
('values', save_dict_values),
|
|
1311
|
+
('items', save_dict_items)
|
|
1312
|
+
)
|
|
1313
|
+
|
|
1314
|
+
for __dicttype in (
|
|
1315
|
+
dict,
|
|
1316
|
+
OrderedDict
|
|
1317
|
+
):
|
|
1318
|
+
__obj = __dicttype()
|
|
1319
|
+
for __funcname, __savefunc in save_dict_view(__dicttype):
|
|
1320
|
+
__tview = type(getattr(__obj, __funcname)())
|
|
1321
|
+
if __tview not in Pickler.dispatch:
|
|
1322
|
+
Pickler.dispatch[__tview] = __savefunc
|
|
1323
|
+
del __dicttype, __obj, __funcname, __tview, __savefunc
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
@register(ClassType)
|
|
1327
|
+
def save_classobj(pickler, obj): #FIXME: enable pickler._byref
|
|
1328
|
+
if not _locate_function(obj, pickler):
|
|
1329
|
+
logger.trace(pickler, "C1: %s", obj)
|
|
1330
|
+
pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__,
|
|
1331
|
+
obj.__dict__), obj=obj)
|
|
1332
|
+
#XXX: or obj.__dict__.copy()), obj=obj) ?
|
|
1333
|
+
logger.trace(pickler, "# C1")
|
|
1334
|
+
else:
|
|
1335
|
+
logger.trace(pickler, "C2: %s", obj)
|
|
1336
|
+
name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
|
|
1337
|
+
StockPickler.save_global(pickler, obj, name=name)
|
|
1338
|
+
logger.trace(pickler, "# C2")
|
|
1339
|
+
return
|
|
1340
|
+
|
|
1341
|
+
@register(typing._GenericAlias)
|
|
1342
|
+
def save_generic_alias(pickler, obj):
|
|
1343
|
+
args = obj.__args__
|
|
1344
|
+
if type(obj.__reduce__()) is str:
|
|
1345
|
+
logger.trace(pickler, "Ga0: %s", obj)
|
|
1346
|
+
StockPickler.save_global(pickler, obj, name=obj.__reduce__())
|
|
1347
|
+
logger.trace(pickler, "# Ga0")
|
|
1348
|
+
elif obj.__origin__ is tuple and (not args or args == ((),)):
|
|
1349
|
+
logger.trace(pickler, "Ga1: %s", obj)
|
|
1350
|
+
pickler.save_reduce(_create_typing_tuple, (args,), obj=obj)
|
|
1351
|
+
logger.trace(pickler, "# Ga1")
|
|
1352
|
+
else:
|
|
1353
|
+
logger.trace(pickler, "Ga2: %s", obj)
|
|
1354
|
+
StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj)
|
|
1355
|
+
logger.trace(pickler, "# Ga2")
|
|
1356
|
+
return
|
|
1357
|
+
|
|
1358
|
+
if ThreadHandleType:
|
|
1359
|
+
@register(ThreadHandleType)
|
|
1360
|
+
def save_thread_handle(pickler, obj):
|
|
1361
|
+
logger.trace(pickler, "Th: %s", obj)
|
|
1362
|
+
pickler.save_reduce(_create_thread_handle, (obj.ident, obj.is_done()), obj=obj)
|
|
1363
|
+
logger.trace(pickler, "# Th")
|
|
1364
|
+
return
|
|
1365
|
+
|
|
1366
|
+
@register(LockType) #XXX: copied Thread will have new Event (due to new Lock)
|
|
1367
|
+
def save_lock(pickler, obj):
|
|
1368
|
+
logger.trace(pickler, "Lo: %s", obj)
|
|
1369
|
+
pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj)
|
|
1370
|
+
logger.trace(pickler, "# Lo")
|
|
1371
|
+
return
|
|
1372
|
+
|
|
1373
|
+
@register(RLockType)
|
|
1374
|
+
def save_rlock(pickler, obj):
|
|
1375
|
+
logger.trace(pickler, "RL: %s", obj)
|
|
1376
|
+
r = obj.__repr__() # don't use _release_save as it unlocks the lock
|
|
1377
|
+
count = int(r.split('count=')[1].split()[0].rstrip('>'))
|
|
1378
|
+
owner = int(r.split('owner=')[1].split()[0])
|
|
1379
|
+
pickler.save_reduce(_create_rlock, (count,owner,), obj=obj)
|
|
1380
|
+
logger.trace(pickler, "# RL")
|
|
1381
|
+
return
|
|
1382
|
+
|
|
1383
|
+
#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL
|
|
1384
|
+
def save_socket(pickler, obj):
|
|
1385
|
+
logger.trace(pickler, "So: %s", obj)
|
|
1386
|
+
pickler.save_reduce(*reduce_socket(obj))
|
|
1387
|
+
logger.trace(pickler, "# So")
|
|
1388
|
+
return
|
|
1389
|
+
|
|
1390
|
+
def _save_file(pickler, obj, open_):
|
|
1391
|
+
if obj.closed:
|
|
1392
|
+
position = 0
|
|
1393
|
+
else:
|
|
1394
|
+
obj.flush()
|
|
1395
|
+
if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__):
|
|
1396
|
+
position = -1
|
|
1397
|
+
else:
|
|
1398
|
+
position = obj.tell()
|
|
1399
|
+
if is_dill(pickler, child=True) and pickler._fmode == FILE_FMODE:
|
|
1400
|
+
f = open_(obj.name, "r")
|
|
1401
|
+
fdata = f.read()
|
|
1402
|
+
f.close()
|
|
1403
|
+
else:
|
|
1404
|
+
fdata = ""
|
|
1405
|
+
if is_dill(pickler, child=True):
|
|
1406
|
+
strictio = pickler._strictio
|
|
1407
|
+
fmode = pickler._fmode
|
|
1408
|
+
else:
|
|
1409
|
+
strictio = False
|
|
1410
|
+
fmode = 0 # HANDLE_FMODE
|
|
1411
|
+
pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position,
|
|
1412
|
+
obj.closed, open_, strictio,
|
|
1413
|
+
fmode, fdata), obj=obj)
|
|
1414
|
+
return
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
@register(FileType) #XXX: in 3.x has buffer=0, needs different _create?
|
|
1418
|
+
@register(BufferedReaderType)
|
|
1419
|
+
@register(BufferedWriterType)
|
|
1420
|
+
@register(TextWrapperType)
|
|
1421
|
+
def save_file(pickler, obj):
|
|
1422
|
+
logger.trace(pickler, "Fi: %s", obj)
|
|
1423
|
+
f = _save_file(pickler, obj, open)
|
|
1424
|
+
logger.trace(pickler, "# Fi")
|
|
1425
|
+
return f
|
|
1426
|
+
|
|
1427
|
+
if BufferedRandomType:
|
|
1428
|
+
@register(BufferedRandomType)
|
|
1429
|
+
def save_file(pickler, obj):
|
|
1430
|
+
logger.trace(pickler, "Fi: %s", obj)
|
|
1431
|
+
f = _save_file(pickler, obj, open)
|
|
1432
|
+
logger.trace(pickler, "# Fi")
|
|
1433
|
+
return f
|
|
1434
|
+
|
|
1435
|
+
if PyTextWrapperType:
|
|
1436
|
+
@register(PyBufferedReaderType)
|
|
1437
|
+
@register(PyBufferedWriterType)
|
|
1438
|
+
@register(PyTextWrapperType)
|
|
1439
|
+
def save_file(pickler, obj):
|
|
1440
|
+
logger.trace(pickler, "Fi: %s", obj)
|
|
1441
|
+
f = _save_file(pickler, obj, _open)
|
|
1442
|
+
logger.trace(pickler, "# Fi")
|
|
1443
|
+
return f
|
|
1444
|
+
|
|
1445
|
+
if PyBufferedRandomType:
|
|
1446
|
+
@register(PyBufferedRandomType)
|
|
1447
|
+
def save_file(pickler, obj):
|
|
1448
|
+
logger.trace(pickler, "Fi: %s", obj)
|
|
1449
|
+
f = _save_file(pickler, obj, _open)
|
|
1450
|
+
logger.trace(pickler, "# Fi")
|
|
1451
|
+
return f
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
# The following two functions are based on 'saveCStringIoInput'
|
|
1455
|
+
# and 'saveCStringIoOutput' from spickle
|
|
1456
|
+
# Copyright (c) 2011 by science+computing ag
|
|
1457
|
+
# License: http://www.apache.org/licenses/LICENSE-2.0
|
|
1458
|
+
if InputType:
|
|
1459
|
+
@register(InputType)
|
|
1460
|
+
def save_stringi(pickler, obj):
|
|
1461
|
+
logger.trace(pickler, "Io: %s", obj)
|
|
1462
|
+
if obj.closed:
|
|
1463
|
+
value = ''; position = 0
|
|
1464
|
+
else:
|
|
1465
|
+
value = obj.getvalue(); position = obj.tell()
|
|
1466
|
+
pickler.save_reduce(_create_stringi, (value, position, \
|
|
1467
|
+
obj.closed), obj=obj)
|
|
1468
|
+
logger.trace(pickler, "# Io")
|
|
1469
|
+
return
|
|
1470
|
+
|
|
1471
|
+
@register(OutputType)
|
|
1472
|
+
def save_stringo(pickler, obj):
|
|
1473
|
+
logger.trace(pickler, "Io: %s", obj)
|
|
1474
|
+
if obj.closed:
|
|
1475
|
+
value = ''; position = 0
|
|
1476
|
+
else:
|
|
1477
|
+
value = obj.getvalue(); position = obj.tell()
|
|
1478
|
+
pickler.save_reduce(_create_stringo, (value, position, \
|
|
1479
|
+
obj.closed), obj=obj)
|
|
1480
|
+
logger.trace(pickler, "# Io")
|
|
1481
|
+
return
|
|
1482
|
+
|
|
1483
|
+
if LRUCacheType is not None:
|
|
1484
|
+
from functools import lru_cache
|
|
1485
|
+
@register(LRUCacheType)
|
|
1486
|
+
def save_lru_cache(pickler, obj):
|
|
1487
|
+
logger.trace(pickler, "LRU: %s", obj)
|
|
1488
|
+
if OLD39:
|
|
1489
|
+
kwargs = obj.cache_info()
|
|
1490
|
+
args = (kwargs.maxsize,)
|
|
1491
|
+
else:
|
|
1492
|
+
kwargs = obj.cache_parameters()
|
|
1493
|
+
args = (kwargs['maxsize'], kwargs['typed'])
|
|
1494
|
+
if args != lru_cache.__defaults__:
|
|
1495
|
+
wrapper = Reduce(lru_cache, args, is_callable=True)
|
|
1496
|
+
else:
|
|
1497
|
+
wrapper = lru_cache
|
|
1498
|
+
pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj)
|
|
1499
|
+
logger.trace(pickler, "# LRU")
|
|
1500
|
+
return
|
|
1501
|
+
|
|
1502
|
+
@register(SuperType)
|
|
1503
|
+
def save_super(pickler, obj):
|
|
1504
|
+
logger.trace(pickler, "Su: %s", obj)
|
|
1505
|
+
pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj)
|
|
1506
|
+
logger.trace(pickler, "# Su")
|
|
1507
|
+
return
|
|
1508
|
+
|
|
1509
|
+
if IS_PYPY:
|
|
1510
|
+
@register(MethodType)
|
|
1511
|
+
def save_instancemethod0(pickler, obj):
|
|
1512
|
+
code = getattr(obj.__func__, '__code__', None)
|
|
1513
|
+
if code is not None and type(code) is not CodeType \
|
|
1514
|
+
and getattr(obj.__self__, obj.__name__) == obj:
|
|
1515
|
+
# Some PyPy builtin functions have no module name
|
|
1516
|
+
logger.trace(pickler, "Me2: %s", obj)
|
|
1517
|
+
# TODO: verify that this works for all PyPy builtin methods
|
|
1518
|
+
pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
|
|
1519
|
+
logger.trace(pickler, "# Me2")
|
|
1520
|
+
return
|
|
1521
|
+
|
|
1522
|
+
logger.trace(pickler, "Me1: %s", obj)
|
|
1523
|
+
pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
|
|
1524
|
+
logger.trace(pickler, "# Me1")
|
|
1525
|
+
return
|
|
1526
|
+
else:
|
|
1527
|
+
@register(MethodType)
|
|
1528
|
+
def save_instancemethod0(pickler, obj):
|
|
1529
|
+
logger.trace(pickler, "Me1: %s", obj)
|
|
1530
|
+
pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
|
|
1531
|
+
logger.trace(pickler, "# Me1")
|
|
1532
|
+
return
|
|
1533
|
+
|
|
1534
|
+
if not IS_PYPY:
|
|
1535
|
+
@register(MemberDescriptorType)
|
|
1536
|
+
@register(GetSetDescriptorType)
|
|
1537
|
+
@register(MethodDescriptorType)
|
|
1538
|
+
@register(WrapperDescriptorType)
|
|
1539
|
+
@register(ClassMethodDescriptorType)
|
|
1540
|
+
def save_wrapper_descriptor(pickler, obj):
|
|
1541
|
+
logger.trace(pickler, "Wr: %s", obj)
|
|
1542
|
+
pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
|
|
1543
|
+
obj.__repr__()), obj=obj)
|
|
1544
|
+
logger.trace(pickler, "# Wr")
|
|
1545
|
+
return
|
|
1546
|
+
else:
|
|
1547
|
+
@register(MemberDescriptorType)
|
|
1548
|
+
@register(GetSetDescriptorType)
|
|
1549
|
+
def save_wrapper_descriptor(pickler, obj):
|
|
1550
|
+
logger.trace(pickler, "Wr: %s", obj)
|
|
1551
|
+
pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
|
|
1552
|
+
obj.__repr__()), obj=obj)
|
|
1553
|
+
logger.trace(pickler, "# Wr")
|
|
1554
|
+
return
|
|
1555
|
+
|
|
1556
|
+
@register(CellType)
|
|
1557
|
+
def save_cell(pickler, obj):
|
|
1558
|
+
try:
|
|
1559
|
+
f = obj.cell_contents
|
|
1560
|
+
except ValueError: # cell is empty
|
|
1561
|
+
logger.trace(pickler, "Ce3: %s", obj)
|
|
1562
|
+
# _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7.
|
|
1563
|
+
# It unpickles to a sentinel object _dill._CELL_EMPTY, also created in
|
|
1564
|
+
# _shims.py. This object is not present in Python 3 because the cell's
|
|
1565
|
+
# contents can be deleted in newer versions of Python. The reduce object
|
|
1566
|
+
# will instead unpickle to None if unpickled in Python 3.
|
|
1567
|
+
|
|
1568
|
+
# When breaking changes are made to dill, (_shims._CELL_EMPTY,) can
|
|
1569
|
+
# be replaced by () OR the delattr function can be removed repending on
|
|
1570
|
+
# whichever is more convienient.
|
|
1571
|
+
pickler.save_reduce(_create_cell, (_shims._CELL_EMPTY,), obj=obj)
|
|
1572
|
+
# Call the function _delattr on the cell's cell_contents attribute
|
|
1573
|
+
# The result of this function call will be None
|
|
1574
|
+
pickler.save_reduce(_shims._delattr, (obj, 'cell_contents'))
|
|
1575
|
+
# pop None created by calling _delattr off stack
|
|
1576
|
+
pickler.write(POP)
|
|
1577
|
+
logger.trace(pickler, "# Ce3")
|
|
1578
|
+
return
|
|
1579
|
+
if is_dill(pickler, child=True):
|
|
1580
|
+
if id(f) in pickler._postproc:
|
|
1581
|
+
# Already seen. Add to its postprocessing.
|
|
1582
|
+
postproc = pickler._postproc[id(f)]
|
|
1583
|
+
else:
|
|
1584
|
+
# Haven't seen it. Add to the highest possible object and set its
|
|
1585
|
+
# value as late as possible to prevent cycle.
|
|
1586
|
+
postproc = next(iter(pickler._postproc.values()), None)
|
|
1587
|
+
if postproc is not None:
|
|
1588
|
+
logger.trace(pickler, "Ce2: %s", obj)
|
|
1589
|
+
# _CELL_REF is defined in _shims.py to support older versions of
|
|
1590
|
+
# dill. When breaking changes are made to dill, (_CELL_REF,) can
|
|
1591
|
+
# be replaced by ()
|
|
1592
|
+
pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj)
|
|
1593
|
+
postproc.append((_shims._setattr, (obj, 'cell_contents', f)))
|
|
1594
|
+
logger.trace(pickler, "# Ce2")
|
|
1595
|
+
return
|
|
1596
|
+
logger.trace(pickler, "Ce1: %s", obj)
|
|
1597
|
+
pickler.save_reduce(_create_cell, (f,), obj=obj)
|
|
1598
|
+
logger.trace(pickler, "# Ce1")
|
|
1599
|
+
return
|
|
1600
|
+
|
|
1601
|
+
if MAPPING_PROXY_TRICK:
|
|
1602
|
+
@register(DictProxyType)
|
|
1603
|
+
def save_dictproxy(pickler, obj):
|
|
1604
|
+
logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
|
|
1605
|
+
mapping = obj | _dictproxy_helper_instance
|
|
1606
|
+
pickler.save_reduce(DictProxyType, (mapping,), obj=obj)
|
|
1607
|
+
logger.trace(pickler, "# Mp")
|
|
1608
|
+
return
|
|
1609
|
+
else:
|
|
1610
|
+
@register(DictProxyType)
|
|
1611
|
+
def save_dictproxy(pickler, obj):
|
|
1612
|
+
logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
|
|
1613
|
+
pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj)
|
|
1614
|
+
logger.trace(pickler, "# Mp")
|
|
1615
|
+
return
|
|
1616
|
+
|
|
1617
|
+
@register(SliceType)
|
|
1618
|
+
def save_slice(pickler, obj):
|
|
1619
|
+
logger.trace(pickler, "Sl: %s", obj)
|
|
1620
|
+
pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj)
|
|
1621
|
+
logger.trace(pickler, "# Sl")
|
|
1622
|
+
return
|
|
1623
|
+
|
|
1624
|
+
@register(XRangeType)
|
|
1625
|
+
@register(EllipsisType)
|
|
1626
|
+
@register(NotImplementedType)
|
|
1627
|
+
def save_singleton(pickler, obj):
|
|
1628
|
+
logger.trace(pickler, "Si: %s", obj)
|
|
1629
|
+
pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj)
|
|
1630
|
+
logger.trace(pickler, "# Si")
|
|
1631
|
+
return
|
|
1632
|
+
|
|
1633
|
+
def _proxy_helper(obj): # a dead proxy returns a reference to None
|
|
1634
|
+
"""get memory address of proxy's reference object"""
|
|
1635
|
+
_repr = repr(obj)
|
|
1636
|
+
try: _str = str(obj)
|
|
1637
|
+
except ReferenceError: # it's a dead proxy
|
|
1638
|
+
return id(None)
|
|
1639
|
+
if _str == _repr: return id(obj) # it's a repr
|
|
1640
|
+
try: # either way, it's a proxy from here
|
|
1641
|
+
address = int(_str.rstrip('>').split(' at ')[-1], base=16)
|
|
1642
|
+
except ValueError: # special case: proxy of a 'type'
|
|
1643
|
+
if not IS_PYPY:
|
|
1644
|
+
address = int(_repr.rstrip('>').split(' at ')[-1], base=16)
|
|
1645
|
+
else:
|
|
1646
|
+
objects = iter(gc.get_objects())
|
|
1647
|
+
for _obj in objects:
|
|
1648
|
+
if repr(_obj) == _str: return id(_obj)
|
|
1649
|
+
# all bad below... nothing found so throw ReferenceError
|
|
1650
|
+
msg = "Cannot reference object for proxy at '%s'" % id(obj)
|
|
1651
|
+
raise ReferenceError(msg)
|
|
1652
|
+
return address
|
|
1653
|
+
|
|
1654
|
+
def _locate_object(address, module=None):
|
|
1655
|
+
"""get object located at the given memory address (inverse of id(obj))"""
|
|
1656
|
+
special = [None, True, False] #XXX: more...?
|
|
1657
|
+
for obj in special:
|
|
1658
|
+
if address == id(obj): return obj
|
|
1659
|
+
if module:
|
|
1660
|
+
objects = iter(module.__dict__.values())
|
|
1661
|
+
else: objects = iter(gc.get_objects())
|
|
1662
|
+
for obj in objects:
|
|
1663
|
+
if address == id(obj): return obj
|
|
1664
|
+
# all bad below... nothing found so throw ReferenceError or TypeError
|
|
1665
|
+
try: address = hex(address)
|
|
1666
|
+
except TypeError:
|
|
1667
|
+
raise TypeError("'%s' is not a valid memory address" % str(address))
|
|
1668
|
+
raise ReferenceError("Cannot reference object at '%s'" % address)
|
|
1669
|
+
|
|
1670
|
+
@register(ReferenceType)
|
|
1671
|
+
def save_weakref(pickler, obj):
|
|
1672
|
+
refobj = obj()
|
|
1673
|
+
logger.trace(pickler, "R1: %s", obj)
|
|
1674
|
+
#refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None"
|
|
1675
|
+
pickler.save_reduce(_create_weakref, (refobj,), obj=obj)
|
|
1676
|
+
logger.trace(pickler, "# R1")
|
|
1677
|
+
return
|
|
1678
|
+
|
|
1679
|
+
@register(ProxyType)
|
|
1680
|
+
@register(CallableProxyType)
|
|
1681
|
+
def save_weakproxy(pickler, obj):
|
|
1682
|
+
# Must do string substitution here and use %r to avoid ReferenceError.
|
|
1683
|
+
logger.trace(pickler, "R2: %r" % obj)
|
|
1684
|
+
refobj = _locate_object(_proxy_helper(obj))
|
|
1685
|
+
pickler.save_reduce(_create_weakproxy, (refobj, callable(obj)), obj=obj)
|
|
1686
|
+
logger.trace(pickler, "# R2")
|
|
1687
|
+
return
|
|
1688
|
+
|
|
1689
|
+
def _is_builtin_module(module):
|
|
1690
|
+
if not hasattr(module, "__file__"): return True
|
|
1691
|
+
if module.__file__ is None: return False
|
|
1692
|
+
# If a module file name starts with prefix, it should be a builtin
|
|
1693
|
+
# module, so should always be pickled as a reference.
|
|
1694
|
+
names = ["base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"]
|
|
1695
|
+
rp = os.path.realpath
|
|
1696
|
+
# See https://github.com/uqfoundation/dill/issues/566
|
|
1697
|
+
return (
|
|
1698
|
+
any(
|
|
1699
|
+
module.__file__.startswith(getattr(sys, name))
|
|
1700
|
+
or rp(module.__file__).startswith(rp(getattr(sys, name)))
|
|
1701
|
+
for name in names
|
|
1702
|
+
if hasattr(sys, name)
|
|
1703
|
+
)
|
|
1704
|
+
or module.__file__.endswith(EXTENSION_SUFFIXES)
|
|
1705
|
+
or 'site-packages' in module.__file__
|
|
1706
|
+
)
|
|
1707
|
+
|
|
1708
|
+
def _is_imported_module(module):
|
|
1709
|
+
return getattr(module, '__loader__', None) is not None or module in sys.modules.values()
|
|
1710
|
+
|
|
1711
|
+
@register(ModuleType)
|
|
1712
|
+
def save_module(pickler, obj):
|
|
1713
|
+
if False: #_use_diff:
|
|
1714
|
+
if obj.__name__.split('.', 1)[0] != "dill":
|
|
1715
|
+
try:
|
|
1716
|
+
changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0]
|
|
1717
|
+
except RuntimeError: # not memorised module, probably part of dill
|
|
1718
|
+
pass
|
|
1719
|
+
else:
|
|
1720
|
+
logger.trace(pickler, "M2: %s with diff", obj)
|
|
1721
|
+
logger.info("Diff: %s", changed.keys())
|
|
1722
|
+
pickler.save_reduce(_import_module, (obj.__name__,), obj=obj,
|
|
1723
|
+
state=changed)
|
|
1724
|
+
logger.trace(pickler, "# M2")
|
|
1725
|
+
return
|
|
1726
|
+
|
|
1727
|
+
logger.trace(pickler, "M1: %s", obj)
|
|
1728
|
+
pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
|
|
1729
|
+
logger.trace(pickler, "# M1")
|
|
1730
|
+
else:
|
|
1731
|
+
builtin_mod = _is_builtin_module(obj)
|
|
1732
|
+
is_session_main = is_dill(pickler, child=True) and obj is pickler._main
|
|
1733
|
+
if (obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod
|
|
1734
|
+
or is_session_main):
|
|
1735
|
+
logger.trace(pickler, "M1: %s", obj)
|
|
1736
|
+
# Hack for handling module-type objects in load_module().
|
|
1737
|
+
mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__
|
|
1738
|
+
# Second references are saved as __builtin__.__main__ in save_module_dict().
|
|
1739
|
+
main_dict = obj.__dict__.copy()
|
|
1740
|
+
for item in ('__builtins__', '__loader__'):
|
|
1741
|
+
main_dict.pop(item, None)
|
|
1742
|
+
for item in IPYTHON_SINGLETONS: #pragma: no cover
|
|
1743
|
+
if getattr(main_dict.get(item), '__module__', '').startswith('IPython'):
|
|
1744
|
+
del main_dict[item]
|
|
1745
|
+
pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=main_dict)
|
|
1746
|
+
logger.trace(pickler, "# M1")
|
|
1747
|
+
elif obj.__name__ == "dill._dill":
|
|
1748
|
+
logger.trace(pickler, "M2: %s", obj)
|
|
1749
|
+
pickler.save_global(obj, name="_dill")
|
|
1750
|
+
logger.trace(pickler, "# M2")
|
|
1751
|
+
else:
|
|
1752
|
+
logger.trace(pickler, "M2: %s", obj)
|
|
1753
|
+
pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
|
|
1754
|
+
logger.trace(pickler, "# M2")
|
|
1755
|
+
return
|
|
1756
|
+
|
|
1757
|
+
# The following function is based on '_extract_class_dict' from 'cloudpickle'
|
|
1758
|
+
# Copyright (c) 2012, Regents of the University of California.
|
|
1759
|
+
# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
|
|
1760
|
+
# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
|
|
1761
|
+
def _get_typedict_type(cls, clsdict, attrs, postproc_list):
|
|
1762
|
+
"""Retrieve a copy of the dict of a class without the inherited methods"""
|
|
1763
|
+
if len(cls.__bases__) == 1:
|
|
1764
|
+
inherited_dict = cls.__bases__[0].__dict__
|
|
1765
|
+
else:
|
|
1766
|
+
inherited_dict = {}
|
|
1767
|
+
for base in reversed(cls.__bases__):
|
|
1768
|
+
inherited_dict.update(base.__dict__)
|
|
1769
|
+
to_remove = []
|
|
1770
|
+
for name, value in dict.items(clsdict):
|
|
1771
|
+
try:
|
|
1772
|
+
base_value = inherited_dict[name]
|
|
1773
|
+
if value is base_value and hasattr(value, '__qualname__'):
|
|
1774
|
+
to_remove.append(name)
|
|
1775
|
+
except KeyError:
|
|
1776
|
+
pass
|
|
1777
|
+
for name in to_remove:
|
|
1778
|
+
dict.pop(clsdict, name)
|
|
1779
|
+
|
|
1780
|
+
if issubclass(type(cls), type):
|
|
1781
|
+
clsdict.pop('__dict__', None)
|
|
1782
|
+
clsdict.pop('__weakref__', None)
|
|
1783
|
+
# clsdict.pop('__prepare__', None)
|
|
1784
|
+
return clsdict, attrs
|
|
1785
|
+
|
|
1786
|
+
def _get_typedict_abc(obj, _dict, attrs, postproc_list):
|
|
1787
|
+
if hasattr(abc, '_get_dump'):
|
|
1788
|
+
(registry, _, _, _) = abc._get_dump(obj)
|
|
1789
|
+
register = obj.register
|
|
1790
|
+
postproc_list.extend((register, (reg(),)) for reg in registry)
|
|
1791
|
+
elif hasattr(obj, '_abc_registry'):
|
|
1792
|
+
registry = obj._abc_registry
|
|
1793
|
+
register = obj.register
|
|
1794
|
+
postproc_list.extend((register, (reg,)) for reg in registry)
|
|
1795
|
+
else:
|
|
1796
|
+
raise PicklingError("Cannot find registry of ABC %s", obj)
|
|
1797
|
+
|
|
1798
|
+
if '_abc_registry' in _dict:
|
|
1799
|
+
_dict.pop('_abc_registry', None)
|
|
1800
|
+
_dict.pop('_abc_cache', None)
|
|
1801
|
+
_dict.pop('_abc_negative_cache', None)
|
|
1802
|
+
# _dict.pop('_abc_negative_cache_version', None)
|
|
1803
|
+
else:
|
|
1804
|
+
_dict.pop('_abc_impl', None)
|
|
1805
|
+
return _dict, attrs
|
|
1806
|
+
|
|
1807
|
+
@register(TypeType)
|
|
1808
|
+
def save_type(pickler, obj, postproc_list=None):
|
|
1809
|
+
if obj in _typemap:
|
|
1810
|
+
logger.trace(pickler, "T1: %s", obj)
|
|
1811
|
+
# if obj in _incedental_types:
|
|
1812
|
+
# warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning)
|
|
1813
|
+
pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
|
|
1814
|
+
logger.trace(pickler, "# T1")
|
|
1815
|
+
elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]):
|
|
1816
|
+
# special case: namedtuples
|
|
1817
|
+
logger.trace(pickler, "T6: %s", obj)
|
|
1818
|
+
|
|
1819
|
+
obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
|
|
1820
|
+
if obj.__name__ != obj_name:
|
|
1821
|
+
if postproc_list is None:
|
|
1822
|
+
postproc_list = []
|
|
1823
|
+
postproc_list.append((setattr, (obj, '__qualname__', obj_name)))
|
|
1824
|
+
|
|
1825
|
+
if not obj._field_defaults:
|
|
1826
|
+
_save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list)
|
|
1827
|
+
else:
|
|
1828
|
+
defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults]
|
|
1829
|
+
_save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list)
|
|
1830
|
+
logger.trace(pickler, "# T6")
|
|
1831
|
+
return
|
|
1832
|
+
|
|
1833
|
+
# special caes: NoneType, NotImplementedType, EllipsisType, EnumMeta, etc
|
|
1834
|
+
elif obj is type(None):
|
|
1835
|
+
logger.trace(pickler, "T7: %s", obj)
|
|
1836
|
+
#XXX: pickler.save_reduce(type, (None,), obj=obj)
|
|
1837
|
+
pickler.write(GLOBAL + b'__builtin__\nNoneType\n')
|
|
1838
|
+
logger.trace(pickler, "# T7")
|
|
1839
|
+
elif obj is NotImplementedType:
|
|
1840
|
+
logger.trace(pickler, "T7: %s", obj)
|
|
1841
|
+
pickler.save_reduce(type, (NotImplemented,), obj=obj)
|
|
1842
|
+
logger.trace(pickler, "# T7")
|
|
1843
|
+
elif obj is EllipsisType:
|
|
1844
|
+
logger.trace(pickler, "T7: %s", obj)
|
|
1845
|
+
pickler.save_reduce(type, (Ellipsis,), obj=obj)
|
|
1846
|
+
logger.trace(pickler, "# T7")
|
|
1847
|
+
elif obj is EnumMeta:
|
|
1848
|
+
logger.trace(pickler, "T7: %s", obj)
|
|
1849
|
+
pickler.write(GLOBAL + b'enum\nEnumMeta\n')
|
|
1850
|
+
logger.trace(pickler, "# T7")
|
|
1851
|
+
elif obj is ExceptHookArgsType: #NOTE: must be after NoneType for pypy
|
|
1852
|
+
logger.trace(pickler, "T7: %s", obj)
|
|
1853
|
+
pickler.write(GLOBAL + b'threading\nExceptHookArgs\n')
|
|
1854
|
+
logger.trace(pickler, "# T7")
|
|
1855
|
+
|
|
1856
|
+
else:
|
|
1857
|
+
_byref = getattr(pickler, '_byref', None)
|
|
1858
|
+
obj_recursive = id(obj) in getattr(pickler, '_postproc', ())
|
|
1859
|
+
incorrectly_named = not _locate_function(obj, pickler)
|
|
1860
|
+
if not _byref and not obj_recursive and incorrectly_named: # not a function, but the name was held over
|
|
1861
|
+
if postproc_list is None:
|
|
1862
|
+
postproc_list = []
|
|
1863
|
+
|
|
1864
|
+
# thanks to Tom Stepleton pointing out pickler._session unneeded
|
|
1865
|
+
logger.trace(pickler, "T2: %s", obj)
|
|
1866
|
+
_dict, attrs = _get_typedict_type(obj, obj.__dict__.copy(), None, postproc_list) # copy dict proxy to a dict
|
|
1867
|
+
|
|
1868
|
+
#print (_dict)
|
|
1869
|
+
#print ("%s\n%s" % (type(obj), obj.__name__))
|
|
1870
|
+
#print ("%s\n%s" % (obj.__bases__, obj.__dict__))
|
|
1871
|
+
slots = _dict.get('__slots__', ())
|
|
1872
|
+
if type(slots) == str:
|
|
1873
|
+
# __slots__ accepts a single string
|
|
1874
|
+
slots = (slots,)
|
|
1875
|
+
|
|
1876
|
+
for name in slots:
|
|
1877
|
+
_dict.pop(name, None)
|
|
1878
|
+
|
|
1879
|
+
if isinstance(obj, abc.ABCMeta):
|
|
1880
|
+
logger.trace(pickler, "ABC: %s", obj)
|
|
1881
|
+
_dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list)
|
|
1882
|
+
logger.trace(pickler, "# ABC")
|
|
1883
|
+
|
|
1884
|
+
qualname = getattr(obj, '__qualname__', None)
|
|
1885
|
+
if attrs is not None:
|
|
1886
|
+
for k, v in attrs.items():
|
|
1887
|
+
postproc_list.append((setattr, (obj, k, v)))
|
|
1888
|
+
# TODO: Consider using the state argument to save_reduce?
|
|
1889
|
+
if qualname is not None:
|
|
1890
|
+
postproc_list.append((setattr, (obj, '__qualname__', qualname)))
|
|
1891
|
+
|
|
1892
|
+
if not hasattr(obj, '__orig_bases__'):
|
|
1893
|
+
_save_with_postproc(pickler, (_create_type, (
|
|
1894
|
+
type(obj), obj.__name__, obj.__bases__, _dict
|
|
1895
|
+
)), obj=obj, postproc_list=postproc_list)
|
|
1896
|
+
else:
|
|
1897
|
+
# This case will always work, but might be overkill.
|
|
1898
|
+
_metadict = {
|
|
1899
|
+
'metaclass': type(obj)
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
if _dict:
|
|
1903
|
+
_dict_update = PartialType(_setitems, source=_dict)
|
|
1904
|
+
else:
|
|
1905
|
+
_dict_update = None
|
|
1906
|
+
|
|
1907
|
+
_save_with_postproc(pickler, (new_class, (
|
|
1908
|
+
obj.__name__, obj.__orig_bases__, _metadict, _dict_update
|
|
1909
|
+
)), obj=obj, postproc_list=postproc_list)
|
|
1910
|
+
logger.trace(pickler, "# T2")
|
|
1911
|
+
else:
|
|
1912
|
+
obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
|
|
1913
|
+
logger.trace(pickler, "T4: %s", obj)
|
|
1914
|
+
if incorrectly_named:
|
|
1915
|
+
warnings.warn(
|
|
1916
|
+
"Cannot locate reference to %r." % (obj,),
|
|
1917
|
+
PicklingWarning,
|
|
1918
|
+
stacklevel=3,
|
|
1919
|
+
)
|
|
1920
|
+
if obj_recursive:
|
|
1921
|
+
warnings.warn(
|
|
1922
|
+
"Cannot pickle %r: %s.%s has recursive self-references that "
|
|
1923
|
+
"trigger a RecursionError." % (obj, obj.__module__, obj_name),
|
|
1924
|
+
PicklingWarning,
|
|
1925
|
+
stacklevel=3,
|
|
1926
|
+
)
|
|
1927
|
+
#print (obj.__dict__)
|
|
1928
|
+
#print ("%s\n%s" % (type(obj), obj.__name__))
|
|
1929
|
+
#print ("%s\n%s" % (obj.__bases__, obj.__dict__))
|
|
1930
|
+
StockPickler.save_global(pickler, obj, name=obj_name)
|
|
1931
|
+
logger.trace(pickler, "# T4")
|
|
1932
|
+
return
|
|
1933
|
+
|
|
1934
|
+
@register(property)
|
|
1935
|
+
@register(abc.abstractproperty)
|
|
1936
|
+
def save_property(pickler, obj):
|
|
1937
|
+
logger.trace(pickler, "Pr: %s", obj)
|
|
1938
|
+
pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__),
|
|
1939
|
+
obj=obj)
|
|
1940
|
+
logger.trace(pickler, "# Pr")
|
|
1941
|
+
|
|
1942
|
+
@register(staticmethod)
|
|
1943
|
+
@register(classmethod)
|
|
1944
|
+
@register(abc.abstractstaticmethod)
|
|
1945
|
+
@register(abc.abstractclassmethod)
|
|
1946
|
+
def save_classmethod(pickler, obj):
|
|
1947
|
+
logger.trace(pickler, "Cm: %s", obj)
|
|
1948
|
+
orig_func = obj.__func__
|
|
1949
|
+
|
|
1950
|
+
# if type(obj.__dict__) is dict:
|
|
1951
|
+
# if obj.__dict__:
|
|
1952
|
+
# state = obj.__dict__
|
|
1953
|
+
# else:
|
|
1954
|
+
# state = None
|
|
1955
|
+
# else:
|
|
1956
|
+
# state = (None, {'__dict__', obj.__dict__})
|
|
1957
|
+
|
|
1958
|
+
pickler.save_reduce(type(obj), (orig_func,), obj=obj)
|
|
1959
|
+
logger.trace(pickler, "# Cm")
|
|
1960
|
+
|
|
1961
|
+
@register(FunctionType)
|
|
1962
|
+
def save_function(pickler, obj):
|
|
1963
|
+
if not _locate_function(obj, pickler):
|
|
1964
|
+
if type(obj.__code__) is not CodeType:
|
|
1965
|
+
# Some PyPy builtin functions have no module name, and thus are not
|
|
1966
|
+
# able to be located
|
|
1967
|
+
module_name = getattr(obj, '__module__', None)
|
|
1968
|
+
if module_name is None:
|
|
1969
|
+
module_name = __builtin__.__name__
|
|
1970
|
+
module = _import_module(module_name, safe=True)
|
|
1971
|
+
_pypy_builtin = False
|
|
1972
|
+
try:
|
|
1973
|
+
found, _ = _getattribute(module, obj.__qualname__)
|
|
1974
|
+
if getattr(found, '__func__', None) is obj:
|
|
1975
|
+
_pypy_builtin = True
|
|
1976
|
+
except AttributeError:
|
|
1977
|
+
pass
|
|
1978
|
+
|
|
1979
|
+
if _pypy_builtin:
|
|
1980
|
+
logger.trace(pickler, "F3: %s", obj)
|
|
1981
|
+
pickler.save_reduce(getattr, (found, '__func__'), obj=obj)
|
|
1982
|
+
logger.trace(pickler, "# F3")
|
|
1983
|
+
return
|
|
1984
|
+
|
|
1985
|
+
logger.trace(pickler, "F1: %s", obj)
|
|
1986
|
+
_recurse = getattr(pickler, '_recurse', None)
|
|
1987
|
+
_postproc = getattr(pickler, '_postproc', None)
|
|
1988
|
+
_main_modified = getattr(pickler, '_main_modified', None)
|
|
1989
|
+
_original_main = getattr(pickler, '_original_main', __builtin__)#'None'
|
|
1990
|
+
postproc_list = []
|
|
1991
|
+
if _recurse:
|
|
1992
|
+
# recurse to get all globals referred to by obj
|
|
1993
|
+
from .detect import globalvars
|
|
1994
|
+
globs_copy = globalvars(obj, recurse=True, builtin=True)
|
|
1995
|
+
|
|
1996
|
+
# Add the name of the module to the globs dictionary to prevent
|
|
1997
|
+
# the duplication of the dictionary. Pickle the unpopulated
|
|
1998
|
+
# globals dictionary and set the remaining items after the function
|
|
1999
|
+
# is created to correctly handle recursion.
|
|
2000
|
+
globs = {'__name__': obj.__module__}
|
|
2001
|
+
else:
|
|
2002
|
+
globs_copy = obj.__globals__
|
|
2003
|
+
|
|
2004
|
+
# If the globals is the __dict__ from the module being saved as a
|
|
2005
|
+
# session, substitute it by the dictionary being actually saved.
|
|
2006
|
+
if _main_modified and globs_copy is _original_main.__dict__:
|
|
2007
|
+
globs_copy = getattr(pickler, '_main', _original_main).__dict__
|
|
2008
|
+
globs = globs_copy
|
|
2009
|
+
# If the globals is a module __dict__, do not save it in the pickle.
|
|
2010
|
+
elif globs_copy is not None and obj.__module__ is not None and \
|
|
2011
|
+
getattr(_import_module(obj.__module__, True), '__dict__', None) is globs_copy:
|
|
2012
|
+
globs = globs_copy
|
|
2013
|
+
else:
|
|
2014
|
+
globs = {'__name__': obj.__module__}
|
|
2015
|
+
|
|
2016
|
+
if globs_copy is not None and globs is not globs_copy:
|
|
2017
|
+
# In the case that the globals are copied, we need to ensure that
|
|
2018
|
+
# the globals dictionary is updated when all objects in the
|
|
2019
|
+
# dictionary are already created.
|
|
2020
|
+
glob_ids = {id(g) for g in globs_copy.values()}
|
|
2021
|
+
for stack_element in _postproc:
|
|
2022
|
+
if stack_element in glob_ids:
|
|
2023
|
+
_postproc[stack_element].append((_setitems, (globs, globs_copy)))
|
|
2024
|
+
break
|
|
2025
|
+
else:
|
|
2026
|
+
postproc_list.append((_setitems, (globs, globs_copy)))
|
|
2027
|
+
|
|
2028
|
+
closure = obj.__closure__
|
|
2029
|
+
state_dict = {}
|
|
2030
|
+
for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'):
|
|
2031
|
+
fattr = getattr(obj, fattrname, None)
|
|
2032
|
+
if fattr is not None:
|
|
2033
|
+
state_dict[fattrname] = fattr
|
|
2034
|
+
if obj.__qualname__ != obj.__name__:
|
|
2035
|
+
state_dict['__qualname__'] = obj.__qualname__
|
|
2036
|
+
if '__name__' not in globs or obj.__module__ != globs['__name__']:
|
|
2037
|
+
state_dict['__module__'] = obj.__module__
|
|
2038
|
+
|
|
2039
|
+
state = obj.__dict__
|
|
2040
|
+
if type(state) is not dict:
|
|
2041
|
+
state_dict['__dict__'] = state
|
|
2042
|
+
state = None
|
|
2043
|
+
if state_dict:
|
|
2044
|
+
state = state, state_dict
|
|
2045
|
+
|
|
2046
|
+
_save_with_postproc(pickler, (_create_function, (
|
|
2047
|
+
obj.__code__, globs, obj.__name__, obj.__defaults__,
|
|
2048
|
+
closure
|
|
2049
|
+
), state), obj=obj, postproc_list=postproc_list)
|
|
2050
|
+
|
|
2051
|
+
# Lift closure cell update to earliest function (#458)
|
|
2052
|
+
if _postproc:
|
|
2053
|
+
topmost_postproc = next(iter(_postproc.values()), None)
|
|
2054
|
+
if closure and topmost_postproc:
|
|
2055
|
+
for cell in closure:
|
|
2056
|
+
possible_postproc = (setattr, (cell, 'cell_contents', obj))
|
|
2057
|
+
try:
|
|
2058
|
+
topmost_postproc.remove(possible_postproc)
|
|
2059
|
+
except ValueError:
|
|
2060
|
+
continue
|
|
2061
|
+
|
|
2062
|
+
# Change the value of the cell
|
|
2063
|
+
pickler.save_reduce(*possible_postproc)
|
|
2064
|
+
# pop None created by calling preprocessing step off stack
|
|
2065
|
+
pickler.write(POP)
|
|
2066
|
+
|
|
2067
|
+
logger.trace(pickler, "# F1")
|
|
2068
|
+
else:
|
|
2069
|
+
logger.trace(pickler, "F2: %s", obj)
|
|
2070
|
+
name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
|
|
2071
|
+
StockPickler.save_global(pickler, obj, name=name)
|
|
2072
|
+
logger.trace(pickler, "# F2")
|
|
2073
|
+
return
|
|
2074
|
+
|
|
2075
|
+
if HAS_CTYPES and hasattr(ctypes, 'pythonapi'):
|
|
2076
|
+
_PyCapsule_New = ctypes.pythonapi.PyCapsule_New
|
|
2077
|
+
_PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p)
|
|
2078
|
+
_PyCapsule_New.restype = ctypes.py_object
|
|
2079
|
+
_PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer
|
|
2080
|
+
_PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p)
|
|
2081
|
+
_PyCapsule_GetPointer.restype = ctypes.c_void_p
|
|
2082
|
+
_PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor
|
|
2083
|
+
_PyCapsule_GetDestructor.argtypes = (ctypes.py_object,)
|
|
2084
|
+
_PyCapsule_GetDestructor.restype = ctypes.c_void_p
|
|
2085
|
+
_PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext
|
|
2086
|
+
_PyCapsule_GetContext.argtypes = (ctypes.py_object,)
|
|
2087
|
+
_PyCapsule_GetContext.restype = ctypes.c_void_p
|
|
2088
|
+
_PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName
|
|
2089
|
+
_PyCapsule_GetName.argtypes = (ctypes.py_object,)
|
|
2090
|
+
_PyCapsule_GetName.restype = ctypes.c_char_p
|
|
2091
|
+
_PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid
|
|
2092
|
+
_PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p)
|
|
2093
|
+
_PyCapsule_IsValid.restype = ctypes.c_bool
|
|
2094
|
+
_PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext
|
|
2095
|
+
_PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p)
|
|
2096
|
+
_PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor
|
|
2097
|
+
_PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p)
|
|
2098
|
+
_PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName
|
|
2099
|
+
_PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p)
|
|
2100
|
+
_PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer
|
|
2101
|
+
_PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p)
|
|
2102
|
+
#from _socket import CAPI as _testcapsule
|
|
2103
|
+
_testcapsule_name = b'dill._dill._testcapsule'
|
|
2104
|
+
_testcapsule = _PyCapsule_New(
|
|
2105
|
+
ctypes.cast(_PyCapsule_New, ctypes.c_void_p),
|
|
2106
|
+
ctypes.c_char_p(_testcapsule_name),
|
|
2107
|
+
None
|
|
2108
|
+
)
|
|
2109
|
+
PyCapsuleType = type(_testcapsule)
|
|
2110
|
+
@register(PyCapsuleType)
|
|
2111
|
+
def save_capsule(pickler, obj):
|
|
2112
|
+
logger.trace(pickler, "Cap: %s", obj)
|
|
2113
|
+
name = _PyCapsule_GetName(obj)
|
|
2114
|
+
#warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning)
|
|
2115
|
+
pointer = _PyCapsule_GetPointer(obj, name)
|
|
2116
|
+
context = _PyCapsule_GetContext(obj)
|
|
2117
|
+
destructor = _PyCapsule_GetDestructor(obj)
|
|
2118
|
+
pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj)
|
|
2119
|
+
logger.trace(pickler, "# Cap")
|
|
2120
|
+
_incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType
|
|
2121
|
+
_reverse_typemap['PyCapsuleType'] = PyCapsuleType
|
|
2122
|
+
_incedental_types.add(PyCapsuleType)
|
|
2123
|
+
else:
|
|
2124
|
+
_testcapsule = None
|
|
2125
|
+
|
|
2126
|
+
@register(ContextType)
|
|
2127
|
+
def save_context(pickler, obj):
|
|
2128
|
+
logger.trace(pickler, "Cx: %s", obj)
|
|
2129
|
+
pickler.save_reduce(ContextType, tuple(obj.items()), obj=obj)
|
|
2130
|
+
logger.trace(pickler, "# Cx")
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
#############################
|
|
2134
|
+
# A quick fix for issue #500
|
|
2135
|
+
# This should be removed when a better solution is found.
|
|
2136
|
+
|
|
2137
|
+
if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"):
|
|
2138
|
+
@register(dataclasses._HAS_DEFAULT_FACTORY_CLASS)
|
|
2139
|
+
def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj):
|
|
2140
|
+
logger.trace(pickler, "DcHDF: %s", obj)
|
|
2141
|
+
pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n")
|
|
2142
|
+
logger.trace(pickler, "# DcHDF")
|
|
2143
|
+
|
|
2144
|
+
if hasattr(dataclasses, "MISSING"):
|
|
2145
|
+
@register(type(dataclasses.MISSING))
|
|
2146
|
+
def save_dataclasses_MISSING_TYPE(pickler, obj):
|
|
2147
|
+
logger.trace(pickler, "DcM: %s", obj)
|
|
2148
|
+
pickler.write(GLOBAL + b"dataclasses\nMISSING\n")
|
|
2149
|
+
logger.trace(pickler, "# DcM")
|
|
2150
|
+
|
|
2151
|
+
if hasattr(dataclasses, "KW_ONLY"):
|
|
2152
|
+
@register(type(dataclasses.KW_ONLY))
|
|
2153
|
+
def save_dataclasses_KW_ONLY_TYPE(pickler, obj):
|
|
2154
|
+
logger.trace(pickler, "DcKWO: %s", obj)
|
|
2155
|
+
pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n")
|
|
2156
|
+
logger.trace(pickler, "# DcKWO")
|
|
2157
|
+
|
|
2158
|
+
if hasattr(dataclasses, "_FIELD_BASE"):
|
|
2159
|
+
@register(dataclasses._FIELD_BASE)
|
|
2160
|
+
def save_dataclasses_FIELD_BASE(pickler, obj):
|
|
2161
|
+
logger.trace(pickler, "DcFB: %s", obj)
|
|
2162
|
+
pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n")
|
|
2163
|
+
logger.trace(pickler, "# DcFB")
|
|
2164
|
+
|
|
2165
|
+
#############################
|
|
2166
|
+
|
|
2167
|
+
# quick sanity checking
|
|
2168
|
+
def pickles(obj,exact=False,safe=False,**kwds):
|
|
2169
|
+
"""
|
|
2170
|
+
Quick check if object pickles with dill.
|
|
2171
|
+
|
|
2172
|
+
If *exact=True* then an equality test is done to check if the reconstructed
|
|
2173
|
+
object matches the original object.
|
|
2174
|
+
|
|
2175
|
+
If *safe=True* then any exception will raised in copy signal that the
|
|
2176
|
+
object is not picklable, otherwise only pickling errors will be trapped.
|
|
2177
|
+
|
|
2178
|
+
Additional keyword arguments are as :func:`dumps` and :func:`loads`.
|
|
2179
|
+
"""
|
|
2180
|
+
if safe: exceptions = (Exception,) # RuntimeError, ValueError
|
|
2181
|
+
else:
|
|
2182
|
+
exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError)
|
|
2183
|
+
try:
|
|
2184
|
+
pik = copy(obj, **kwds)
|
|
2185
|
+
#FIXME: should check types match first, then check content if "exact"
|
|
2186
|
+
try:
|
|
2187
|
+
#FIXME: should be "(pik == obj).all()" for numpy comparison, though that'll fail if shapes differ
|
|
2188
|
+
result = bool(pik.all() == obj.all())
|
|
2189
|
+
except (AttributeError, TypeError):
|
|
2190
|
+
warnings.filterwarnings('ignore') #FIXME: be specific
|
|
2191
|
+
result = pik == obj
|
|
2192
|
+
if warnings.filters: del warnings.filters[0]
|
|
2193
|
+
if hasattr(result, 'toarray'): # for unusual types like sparse matrix
|
|
2194
|
+
result = result.toarray().all()
|
|
2195
|
+
if result: return True
|
|
2196
|
+
if not exact:
|
|
2197
|
+
result = type(pik) == type(obj)
|
|
2198
|
+
if result: return result
|
|
2199
|
+
# class instances might have been dumped with byref=False
|
|
2200
|
+
return repr(type(pik)) == repr(type(obj)) #XXX: InstanceType?
|
|
2201
|
+
return False
|
|
2202
|
+
except exceptions:
|
|
2203
|
+
return False
|
|
2204
|
+
|
|
2205
|
+
def check(obj, *args, **kwds):
|
|
2206
|
+
"""
|
|
2207
|
+
Check pickling of an object across another process.
|
|
2208
|
+
|
|
2209
|
+
*python* is the path to the python interpreter (defaults to sys.executable)
|
|
2210
|
+
|
|
2211
|
+
Set *verbose=True* to print the unpickled object in the other process.
|
|
2212
|
+
|
|
2213
|
+
Additional keyword arguments are as :func:`dumps` and :func:`loads`.
|
|
2214
|
+
"""
|
|
2215
|
+
# == undocumented ==
|
|
2216
|
+
# python -- the string path or executable name of the selected python
|
|
2217
|
+
# verbose -- if True, be verbose about printing warning messages
|
|
2218
|
+
# all other args and kwds are passed to dill.dumps #FIXME: ignore on load
|
|
2219
|
+
verbose = kwds.pop('verbose', False)
|
|
2220
|
+
python = kwds.pop('python', None)
|
|
2221
|
+
if python is None:
|
|
2222
|
+
import sys
|
|
2223
|
+
python = sys.executable
|
|
2224
|
+
# type check
|
|
2225
|
+
isinstance(python, str)
|
|
2226
|
+
import subprocess
|
|
2227
|
+
fail = True
|
|
2228
|
+
try:
|
|
2229
|
+
_obj = dumps(obj, *args, **kwds)
|
|
2230
|
+
fail = False
|
|
2231
|
+
finally:
|
|
2232
|
+
if fail and verbose:
|
|
2233
|
+
print("DUMP FAILED")
|
|
2234
|
+
#FIXME: fails if python interpreter path contains spaces
|
|
2235
|
+
# Use the following instead (which also processes the 'ignore' keyword):
|
|
2236
|
+
# ignore = kwds.pop('ignore', None)
|
|
2237
|
+
# unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore))
|
|
2238
|
+
# cmd = [python, "-c", "import dill; print(%s)"%unpickle]
|
|
2239
|
+
# msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED"
|
|
2240
|
+
msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj))
|
|
2241
|
+
msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED"
|
|
2242
|
+
if verbose:
|
|
2243
|
+
print(msg)
|
|
2244
|
+
return
|
|
2245
|
+
|
|
2246
|
+
# use to protect against missing attributes
|
|
2247
|
+
def is_dill(pickler, child=None):
|
|
2248
|
+
"check the dill-ness of your pickler"
|
|
2249
|
+
if child is False or not hasattr(pickler.__class__, 'mro'):
|
|
2250
|
+
return 'dill' in pickler.__module__
|
|
2251
|
+
return Pickler in pickler.__class__.mro()
|
|
2252
|
+
|
|
2253
|
+
def _extend():
|
|
2254
|
+
"""extend pickle with all of dill's registered types"""
|
|
2255
|
+
# need to have pickle not choke on _main_module? use is_dill(pickler)
|
|
2256
|
+
for t,func in Pickler.dispatch.items():
|
|
2257
|
+
try:
|
|
2258
|
+
StockPickler.dispatch[t] = func
|
|
2259
|
+
except Exception: #TypeError, PicklingError, UnpicklingError
|
|
2260
|
+
logger.trace(pickler, "skip: %s", t)
|
|
2261
|
+
return
|
|
2262
|
+
|
|
2263
|
+
del diff, _use_diff, use_diff
|
|
2264
|
+
|
|
2265
|
+
# EOF
|