pycall 1.2.1 → 1.3.0.dev
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +4 -0
- data/README.md +39 -0
- data/examples/classifier_comparison.rb +1 -1
- data/examples/hist.rb +1 -1
- data/examples/notebooks/classifier_comparison.ipynb +1 -1
- data/ext/pycall/pycall.c +60 -6
- data/ext/pycall/pycall_internal.h +26 -0
- data/ext/pycall/thread.c +36 -0
- data/lib/pycall/libpython/finder.rb +5 -5
- data/lib/pycall/version.rb +7 -1
- data/pycall.gemspec +7 -1
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 026f371fe891d8a8ef0fc80caefc2c22b1cefc22e937d3699d2aec0a42239d07
|
4
|
+
data.tar.gz: 51d1a6b56e187ceee4954eddf0d48e0c32d51d350e306ed05f6c7624bea914f4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e7c254cbd02e82a401ab75be52c84738d075b6cbc82c31013513d383af529746afa74df272a84432962d1fe1f0d4eca79a8688d149790ab0ccecf1e7cff679e
|
7
|
+
data.tar.gz: a7f31a993b56cd8b15df2d5d5c2847f71382bef26e26f3723194e71addbf44d35b2939d641a906f8ea12de796befc8f2bf975670db5edf2dd01855c1cac08d2b
|
data/CHANGES.md
CHANGED
data/README.md
CHANGED
@@ -61,6 +61,45 @@ the `Math.sin` in Ruby:
|
|
61
61
|
Type conversions from Ruby to Python are automatically performed for numeric,
|
62
62
|
boolean, string, arrays, and hashes.
|
63
63
|
|
64
|
+
### Releasing the RubyVM GVL during Python function calls
|
65
|
+
|
66
|
+
You may want to release the RubyVM GVL when you call a Python function that takes very long runtime.
|
67
|
+
PyCall provides `PyCall.without_gvl` method for such purpose. When PyCall performs python function call,
|
68
|
+
PyCall checks the current context, and then it releases the RubyVM GVL when the current context is in a `PyCall.without_gvl`'s block.
|
69
|
+
|
70
|
+
```ruby
|
71
|
+
PyCall.without_gvl do
|
72
|
+
# In this block, all Python function calls are performed without
|
73
|
+
# the GVL acquisition.
|
74
|
+
pyobj.long_running_function()
|
75
|
+
end
|
76
|
+
|
77
|
+
# Outside of PyCall.without_gvl block,
|
78
|
+
# all Python function calls are performed with the GVL acquisition.
|
79
|
+
pyobj.long_running_function()
|
80
|
+
```
|
81
|
+
|
82
|
+
### Debugging python finder
|
83
|
+
|
84
|
+
When you encounter `PyCall::PythonNotFound` error, you can investigate PyCall's python finder by setting `PYCALL_DEBUG_FIND_LIBPYTHON` environment variable to `1`. You can see the log like below:
|
85
|
+
|
86
|
+
```
|
87
|
+
$ PYCALL_DEBUG_FIND_LIBPYTHON=1 ruby -rpycall -ePyCall.builtins
|
88
|
+
DEBUG(find_libpython) find_libpython(nil)
|
89
|
+
DEBUG(find_libpython) investigate_python_config("python3")
|
90
|
+
DEBUG(find_libpython) libs: ["Python.framework/Versions/3.7/Python", "Python", "libpython3.7m", "libpython3.7", "libpython"]
|
91
|
+
DEBUG(find_libpython) libpaths: ["/opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib", "/opt/brew/opt/python/lib", "/opt/brew/opt/python/Frameworks", "/opt/brew/Cellar/python/3.7.2_1/Frameworks/Python.framework/Versions/3.7", "/opt/brew/Cellar/python/3.7.2_1/Frameworks/Python.framework/Versions/3.7/lib"]
|
92
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/Python.framework/Versions/3.7/Python
|
93
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/Python.framework/Versions/3.7/Python.dylib
|
94
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/darwin/Python.framework/Versions/3.7/Python
|
95
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/darwin/Python.framework/Versions/3.7/Python.dylib
|
96
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/Python.framework/Versions/3.7/Python
|
97
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/Python.framework/Versions/3.7/Python.dylib
|
98
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/darwin/Python.framework/Versions/3.7/Python
|
99
|
+
DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/darwin/Python.framework/Versions/3.7/Python.dylib
|
100
|
+
DEBUG(find_libpython) dlopen("/opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/Python") = #<Fiddle::Handle:0x00007fc012048650>
|
101
|
+
```
|
102
|
+
|
64
103
|
## PyCall object system
|
65
104
|
|
66
105
|
PyCall wraps pointers of Python objects in `PyCall::PyPtr` objects.
|
@@ -2,7 +2,7 @@ require 'pycall/import'
|
|
2
2
|
include PyCall::Import
|
3
3
|
|
4
4
|
pyimport 'numpy', as: :np
|
5
|
-
pyfrom 'sklearn.
|
5
|
+
pyfrom 'sklearn.model_selection', import: :train_test_split
|
6
6
|
pyfrom 'sklearn.preprocessing', import: :StandardScaler
|
7
7
|
pyfrom 'sklearn.datasets', import: %i(make_moons make_circles make_classification)
|
8
8
|
pyfrom 'sklearn.neighbors', import: :KNeighborsClassifier
|
data/examples/hist.rb
CHANGED
@@ -38,7 +38,7 @@
|
|
38
38
|
"include PyCall::Import\n",
|
39
39
|
"\n",
|
40
40
|
"pyimport 'numpy', as: :np\n",
|
41
|
-
"pyfrom 'sklearn.
|
41
|
+
"pyfrom 'sklearn.model_selection', import: :train_test_split\n",
|
42
42
|
"pyfrom 'sklearn.preprocessing', import: :StandardScaler\n",
|
43
43
|
"pyfrom 'sklearn.datasets', import: %i(make_moons make_circles make_classification)\n",
|
44
44
|
"pyfrom 'sklearn.neighbors', import: :KNeighborsClassifier\n",
|
data/ext/pycall/pycall.c
CHANGED
@@ -70,6 +70,52 @@ pycall_after_fork(VALUE mod)
|
|
70
70
|
return Qnil;
|
71
71
|
}
|
72
72
|
|
73
|
+
static volatile pycall_tls_key without_gvl_key;
|
74
|
+
|
75
|
+
int
|
76
|
+
pycall_without_gvl_p(void)
|
77
|
+
{
|
78
|
+
/*
|
79
|
+
* In pthread, the default value is NULL (== 0).
|
80
|
+
*
|
81
|
+
* In Win32 thread, the default value is 0 (initialized by TlsAlloc).
|
82
|
+
*/
|
83
|
+
return (int)pycall_tls_get(without_gvl_key);
|
84
|
+
}
|
85
|
+
|
86
|
+
static inline int
|
87
|
+
pycall_set_without_gvl(void)
|
88
|
+
{
|
89
|
+
return pycall_tls_set(without_gvl_key, (void *)1);
|
90
|
+
}
|
91
|
+
|
92
|
+
static inline int
|
93
|
+
pycall_set_with_gvl(void)
|
94
|
+
{
|
95
|
+
return pycall_tls_set(without_gvl_key, (void *)0);
|
96
|
+
}
|
97
|
+
|
98
|
+
VALUE
|
99
|
+
pycall_without_gvl(VALUE (* func)(VALUE), VALUE arg)
|
100
|
+
{
|
101
|
+
int state;
|
102
|
+
VALUE result;
|
103
|
+
|
104
|
+
pycall_set_without_gvl();
|
105
|
+
|
106
|
+
result = rb_protect(func, arg, &state);
|
107
|
+
|
108
|
+
pycall_set_with_gvl();
|
109
|
+
|
110
|
+
return result;
|
111
|
+
}
|
112
|
+
|
113
|
+
static VALUE
|
114
|
+
pycall_m_without_gvl(VALUE mod)
|
115
|
+
{
|
116
|
+
return pycall_without_gvl(rb_yield, Qnil);
|
117
|
+
}
|
118
|
+
|
73
119
|
/* ==== PyCall::PyPtr ==== */
|
74
120
|
|
75
121
|
const rb_data_type_t pycall_pyptr_data_type = {
|
@@ -890,7 +936,7 @@ struct call_pyobject_call_params {
|
|
890
936
|
PyObject *kwargs;
|
891
937
|
};
|
892
938
|
|
893
|
-
PyObject *
|
939
|
+
static inline PyObject *
|
894
940
|
call_pyobject_call(struct call_pyobject_call_params *params)
|
895
941
|
{
|
896
942
|
PyObject *res;
|
@@ -899,7 +945,7 @@ call_pyobject_call(struct call_pyobject_call_params *params)
|
|
899
945
|
}
|
900
946
|
|
901
947
|
PyObject *
|
902
|
-
|
948
|
+
pyobject_call(PyObject *pycallable, PyObject *args, PyObject *kwargs)
|
903
949
|
{
|
904
950
|
PyObject *res;
|
905
951
|
struct call_pyobject_call_params params;
|
@@ -907,9 +953,14 @@ pyobject_call_without_gvl(PyObject *pycallable, PyObject *args, PyObject *kwargs
|
|
907
953
|
params.args = args;
|
908
954
|
params.kwargs = kwargs;
|
909
955
|
|
910
|
-
|
911
|
-
|
912
|
-
|
956
|
+
if (pycall_without_gvl_p()) {
|
957
|
+
res = (PyObject *)rb_thread_call_without_gvl(
|
958
|
+
(void * (*)(void *))call_pyobject_call, (void *)¶ms,
|
959
|
+
(rb_unblock_function_t *)pycall_interrupt_python_thread, NULL);
|
960
|
+
}
|
961
|
+
else {
|
962
|
+
res = call_pyobject_call(¶ms);
|
963
|
+
}
|
913
964
|
|
914
965
|
return res;
|
915
966
|
}
|
@@ -961,7 +1012,7 @@ pycall_call_python_callable(PyObject *pycallable, int argc, VALUE *argv)
|
|
961
1012
|
}
|
962
1013
|
}
|
963
1014
|
|
964
|
-
res =
|
1015
|
+
res = pyobject_call(pycallable, args, kwargs); /* New reference */
|
965
1016
|
if (!res) {
|
966
1017
|
pycall_pyerror_fetch_and_raise("PyObject_Call in pycall_call_python_callable");
|
967
1018
|
}
|
@@ -2185,6 +2236,9 @@ Init_pycall(void)
|
|
2185
2236
|
|
2186
2237
|
rb_define_module_function(mPyCall, "after_fork", pycall_after_fork, 0);
|
2187
2238
|
|
2239
|
+
pycall_tls_create(&without_gvl_key);
|
2240
|
+
rb_define_module_function(mPyCall, "without_gvl", pycall_m_without_gvl, 0);
|
2241
|
+
|
2188
2242
|
/* PyCall::PyPtr */
|
2189
2243
|
|
2190
2244
|
cPyPtr = rb_define_class_under(mPyCall, "PyPtr", rb_cBasicObject);
|
@@ -11,10 +11,19 @@ extern "C" {
|
|
11
11
|
#include <ruby.h>
|
12
12
|
#include <ruby/encoding.h>
|
13
13
|
#include <ruby/thread.h>
|
14
|
+
|
14
15
|
#include <assert.h>
|
15
16
|
#include <inttypes.h>
|
16
17
|
#include <limits.h>
|
17
18
|
|
19
|
+
#if defined(_WIN32)
|
20
|
+
# define PYCALL_THREAD_WIN32
|
21
|
+
# include <ruby/win32.h>
|
22
|
+
#elif defined(HAVE_PTHREAD_H)
|
23
|
+
# define PYCALL_THREAD_PTHREAD
|
24
|
+
# include <pthread.h>
|
25
|
+
#endif
|
26
|
+
|
18
27
|
#if SIZEOF_LONG == SIZEOF_VOIDP
|
19
28
|
# define PTR2NUM(x) (LONG2NUM((long)(x)))
|
20
29
|
# define NUM2PTR(x) ((void*)(NUM2ULONG(x)))
|
@@ -492,6 +501,23 @@ extern PyTypeObject PyRuby_Type;
|
|
492
501
|
|
493
502
|
PyObject * PyRuby_New(VALUE ruby_object);
|
494
503
|
|
504
|
+
/* ==== thread support ==== */
|
505
|
+
|
506
|
+
#if defined(PYCALL_THREAD_WIN32)
|
507
|
+
typedef DWORD pycall_tls_key;
|
508
|
+
#elif defined(PYCALL_THREAD_PTHREAD)
|
509
|
+
typedef pthread_key_t pycall_tls_key;
|
510
|
+
#else
|
511
|
+
# error "unsupported thread type"
|
512
|
+
#endif
|
513
|
+
|
514
|
+
int pycall_tls_create(pycall_tls_key* tls_key);
|
515
|
+
void *pycall_tls_get(pycall_tls_key tls_key);
|
516
|
+
int pycall_tls_set(pycall_tls_key tls_key, void *ptr);
|
517
|
+
|
518
|
+
int pycall_without_gvl_p(void);
|
519
|
+
VALUE pycall_without_gvl(VALUE (* func)(VALUE), VALUE arg);
|
520
|
+
|
495
521
|
/* ==== pycall ==== */
|
496
522
|
|
497
523
|
typedef struct {
|
data/ext/pycall/thread.c
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
#include "pycall_internal.h"
|
2
|
+
|
3
|
+
#if defined(PYCALL_THREAD_WIN32)
|
4
|
+
int pycall_tls_create(pycall_tls_key *tls_key)
|
5
|
+
{
|
6
|
+
*tls_key = TlsAlloc();
|
7
|
+
return *tls_key == TLS_OUT_OF_INDEXES;
|
8
|
+
}
|
9
|
+
|
10
|
+
void *pycall_tls_get(pycall_tls_key tls_key)
|
11
|
+
{
|
12
|
+
return TlsGetValue(tls_key);
|
13
|
+
}
|
14
|
+
|
15
|
+
int pycall_tls_set(pycall_tls_key tls_key, void *ptr)
|
16
|
+
{
|
17
|
+
return 0 == TlsSetValue(tls_key, ptr);
|
18
|
+
}
|
19
|
+
#endif
|
20
|
+
|
21
|
+
#if defined(PYCALL_THREAD_PTHREAD)
|
22
|
+
int pycall_tls_create(pycall_tls_key *tls_key)
|
23
|
+
{
|
24
|
+
return pthread_key_create(tls_key, NULL);
|
25
|
+
}
|
26
|
+
|
27
|
+
void *pycall_tls_get(pycall_tls_key tls_key)
|
28
|
+
{
|
29
|
+
return pthread_getspecific(tls_key);
|
30
|
+
}
|
31
|
+
|
32
|
+
int pycall_tls_set(pycall_tls_key tls_key, void *ptr)
|
33
|
+
{
|
34
|
+
return pthread_setspecific(tls_key, ptr);
|
35
|
+
}
|
36
|
+
#endif
|
@@ -27,12 +27,12 @@ module PyCall
|
|
27
27
|
def find_python_config(python = nil)
|
28
28
|
python ||= DEFAULT_PYTHON
|
29
29
|
Array(python).each do |python_cmd|
|
30
|
-
|
31
|
-
|
30
|
+
begin
|
31
|
+
python_config = investigate_python_config(python_cmd)
|
32
|
+
return [python_cmd, python_config] unless python_config.empty?
|
33
|
+
rescue
|
34
|
+
end
|
32
35
|
end
|
33
|
-
rescue
|
34
|
-
raise ::PyCall::PythonNotFound
|
35
|
-
else
|
36
36
|
raise ::PyCall::PythonNotFound
|
37
37
|
end
|
38
38
|
|
data/lib/pycall/version.rb
CHANGED
data/pycall.gemspec
CHANGED
@@ -5,7 +5,13 @@ require 'pycall/version'
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "pycall"
|
8
|
-
|
8
|
+
version_components = [
|
9
|
+
PyCall::Version::MAJOR.to_s,
|
10
|
+
PyCall::Version::MINOR.to_s,
|
11
|
+
PyCall::Version::MICRO.to_s,
|
12
|
+
PyCall::Version::TAG,
|
13
|
+
]
|
14
|
+
spec.version = version_components.compact.join(".")
|
9
15
|
spec.authors = ["Kenta Murata"]
|
10
16
|
spec.email = ["mrkn@mrkn.jp"]
|
11
17
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pycall
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0.dev
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kenta Murata
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-05-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -171,6 +171,7 @@ files:
|
|
171
171
|
- ext/pycall/pycall_internal.h
|
172
172
|
- ext/pycall/range.c
|
173
173
|
- ext/pycall/ruby_wrapper.c
|
174
|
+
- ext/pycall/thread.c
|
174
175
|
- images/pycallrb_logo.png
|
175
176
|
- images/pycallrb_logo_200.png
|
176
177
|
- lib/pycall.rb
|
@@ -215,11 +216,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
215
216
|
version: '0'
|
216
217
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
217
218
|
requirements:
|
218
|
-
- - "
|
219
|
+
- - ">"
|
219
220
|
- !ruby/object:Gem::Version
|
220
|
-
version:
|
221
|
+
version: 1.3.1
|
221
222
|
requirements: []
|
222
|
-
rubygems_version: 3.0.
|
223
|
+
rubygems_version: 3.0.3
|
223
224
|
signing_key:
|
224
225
|
specification_version: 4
|
225
226
|
summary: pycall
|