pycall 1.2.1 → 1.3.0.dev

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6430d6cc17470fa4644b81bbe57f8362ac4ea97bf21f1006f6b63a3fcb2e12fa
-  data.tar.gz: 4251c7fd6fcd5e80b2ed99f45646189170a486ee76d7484a57d26dbcbaf4c71f
+  metadata.gz: 026f371fe891d8a8ef0fc80caefc2c22b1cefc22e937d3699d2aec0a42239d07
+  data.tar.gz: 51d1a6b56e187ceee4954eddf0d48e0c32d51d350e306ed05f6c7624bea914f4
 SHA512:
-  metadata.gz: f3fa8077a0a4cc76a3f620cb5f937822238b2190240b38d93f5791c3d9fd410efba82b9964ef6defa3d9732fa10385e5e9db5c2e28ea9b969b1597d83b40e20c
-  data.tar.gz: a36770a01ffe893a2fdc301658af30e683ac0a65e813134fe90a52ebcbf7baf8e9cf65e09332910a632108d35a10defc44db0faaf82bef5270c4bb427fa42f2c
+  metadata.gz: 4e7c254cbd02e82a401ab75be52c84738d075b6cbc82c31013513d383af529746afa74df272a84432962d1fe1f0d4eca79a8688d149790ab0ccecf1e7cff679e
+  data.tar.gz: a7f31a993b56cd8b15df2d5d5c2847f71382bef26e26f3723194e71addbf44d35b2939d641a906f8ea12de796befc8f2bf975670db5edf2dd01855c1cac08d2b

data/CHANGES.md

@@ -1,5 +1,9 @@
 # The change history of PyCall
 
+## master
+
+* Add `PyCall.without_gvl` for explicitly releasing the RubyVM GVL
+
 ## 1.2.1
 
 * Prevent circular require in pycall/iruby.rb

data/README.md

@@ -61,6 +61,45 @@ the `Math.sin` in Ruby:
 Type conversions from Ruby to Python are automatically performed for numeric,
 boolean, string, arrays, and hashes.
 
+### Releasing the RubyVM GVL during Python function calls
+
+You may want to release the RubyVM GVL when you call a Python function that takes very long runtime.
+PyCall provides `PyCall.without_gvl` method for such purpose.  When PyCall performs python function call,
+PyCall checks the current context, and then it releases the RubyVM GVL when the current context is in a `PyCall.without_gvl`'s block.
+
+```ruby
+PyCall.without_gvl do
+  # In this block, all Python function calls are performed without
+  # the GVL acquisition.
+  pyobj.long_running_function()
+end
+
+# Outside of PyCall.without_gvl block,
+# all Python function calls are performed with the GVL acquisition.
+pyobj.long_running_function()
+```
+
+### Debugging python finder
+
+When you encounter `PyCall::PythonNotFound` error, you can investigate PyCall's python finder by setting `PYCALL_DEBUG_FIND_LIBPYTHON` environment variable to `1`.  You can see the log like below:
+
+```
+$ PYCALL_DEBUG_FIND_LIBPYTHON=1 ruby -rpycall -ePyCall.builtins
+DEBUG(find_libpython) find_libpython(nil)
+DEBUG(find_libpython) investigate_python_config("python3")
+DEBUG(find_libpython) libs: ["Python.framework/Versions/3.7/Python", "Python", "libpython3.7m", "libpython3.7", "libpython"]
+DEBUG(find_libpython) libpaths: ["/opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib", "/opt/brew/opt/python/lib", "/opt/brew/opt/python/Frameworks", "/opt/brew/Cellar/python/3.7.2_1/Frameworks/Python.framework/Versions/3.7", "/opt/brew/Cellar/python/3.7.2_1/Frameworks/Python.framework/Versions/3.7/lib"]
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/Python.framework/Versions/3.7/Python
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/Python.framework/Versions/3.7/Python.dylib
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/darwin/Python.framework/Versions/3.7/Python
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/lib/darwin/Python.framework/Versions/3.7/Python.dylib
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/Python.framework/Versions/3.7/Python
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/Python.framework/Versions/3.7/Python.dylib
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/darwin/Python.framework/Versions/3.7/Python
+DEBUG(find_libpython) Unable to find /opt/brew/opt/python/lib/darwin/Python.framework/Versions/3.7/Python.dylib
+DEBUG(find_libpython) dlopen("/opt/brew/opt/python/Frameworks/Python.framework/Versions/3.7/Python") = #<Fiddle::Handle:0x00007fc012048650>
+```
+
 ## PyCall object system
 
 PyCall wraps pointers of Python objects in `PyCall::PyPtr` objects.

data/examples/classifier_comparison.rb

@@ -2,7 +2,7 @@ require 'pycall/import'
 include PyCall::Import
 
 pyimport 'numpy', as: :np
-pyfrom 'sklearn.cross_validation', import: :train_test_split
+pyfrom 'sklearn.model_selection', import: :train_test_split
 pyfrom 'sklearn.preprocessing', import: :StandardScaler
 pyfrom 'sklearn.datasets', import: %i(make_moons make_circles make_classification)
 pyfrom 'sklearn.neighbors', import: :KNeighborsClassifier

data/examples/hist.rb

@@ -20,7 +20,7 @@ num_bins = 50
 
 fig, ax = *plt.subplots
 
-n, bins, patches = *ax.hist(x, num_bins, normed: 1)
+n, bins, patches = *ax.hist(x, num_bins, density: 1)
 
 y = mlab.normpdf(bins, mu, sigma)
 ax.plot(bins, y, '--')

data/examples/notebooks/classifier_comparison.ipynb

@@ -38,7 +38,7 @@
     "include PyCall::Import\n",
     "\n",
     "pyimport 'numpy', as: :np\n",
-    "pyfrom 'sklearn.cross_validation', import: :train_test_split\n",
+    "pyfrom 'sklearn.model_selection', import: :train_test_split\n",
     "pyfrom 'sklearn.preprocessing', import: :StandardScaler\n",
     "pyfrom 'sklearn.datasets', import: %i(make_moons make_circles make_classification)\n",
     "pyfrom 'sklearn.neighbors', import: :KNeighborsClassifier\n",

data/ext/pycall/pycall.c

@@ -70,6 +70,52 @@ pycall_after_fork(VALUE mod)
   return Qnil;
 }
 
+static volatile pycall_tls_key without_gvl_key;
+
+int
+pycall_without_gvl_p(void)
+{
+  /*
+   * In pthread, the default value is NULL (== 0).
+   *
+   * In Win32 thread, the default value is 0 (initialized by TlsAlloc).
+   */
+  return (int)pycall_tls_get(without_gvl_key);
+}
+
+static inline int
+pycall_set_without_gvl(void)
+{
+  return pycall_tls_set(without_gvl_key, (void *)1);
+}
+
+static inline int
+pycall_set_with_gvl(void)
+{
+  return pycall_tls_set(without_gvl_key, (void *)0);
+}
+
+VALUE
+pycall_without_gvl(VALUE (* func)(VALUE), VALUE arg)
+{
+  int state;
+  VALUE result;
+
+  pycall_set_without_gvl();
+
+  result = rb_protect(func, arg, &state);
+
+  pycall_set_with_gvl();
+
+  return result;
+}
+
+static VALUE
+pycall_m_without_gvl(VALUE mod)
+{
+  return pycall_without_gvl(rb_yield, Qnil);
+}
+
 /* ==== PyCall::PyPtr ==== */
 
 const rb_data_type_t pycall_pyptr_data_type = {
@@ -890,7 +936,7 @@ struct call_pyobject_call_params {
   PyObject *kwargs;
 };
 
-PyObject *
+static inline PyObject *
 call_pyobject_call(struct call_pyobject_call_params *params)
 {
   PyObject *res;
@@ -899,7 +945,7 @@ call_pyobject_call(struct call_pyobject_call_params *params)
 }
 
 PyObject *
-pyobject_call_without_gvl(PyObject *pycallable, PyObject *args, PyObject *kwargs)
+pyobject_call(PyObject *pycallable, PyObject *args, PyObject *kwargs)
 {
   PyObject *res;
   struct call_pyobject_call_params params;
@@ -907,9 +953,14 @@ pyobject_call_without_gvl(PyObject *pycallable, PyObject *args, PyObject *kwargs
   params.args = args;
   params.kwargs = kwargs;
 
-  res = (PyObject *)rb_thread_call_without_gvl(
-      (void * (*)(void *))call_pyobject_call, (void *)&params,
-      (rb_unblock_function_t *)pycall_interrupt_python_thread, NULL);
+  if (pycall_without_gvl_p()) {
+    res = (PyObject *)rb_thread_call_without_gvl(
+        (void * (*)(void *))call_pyobject_call, (void *)&params,
+        (rb_unblock_function_t *)pycall_interrupt_python_thread, NULL);
+  }
+  else {
+    res = call_pyobject_call(&params);
+  }
 
   return res;
 }
@@ -961,7 +1012,7 @@ pycall_call_python_callable(PyObject *pycallable, int argc, VALUE *argv)
     }
   }
 
-  res = pyobject_call_without_gvl(pycallable, args, kwargs); /* New reference */
+  res = pyobject_call(pycallable, args, kwargs); /* New reference */
   if (!res) {
     pycall_pyerror_fetch_and_raise("PyObject_Call in pycall_call_python_callable");
   }
@@ -2185,6 +2236,9 @@ Init_pycall(void)
 
   rb_define_module_function(mPyCall, "after_fork", pycall_after_fork, 0);
 
+  pycall_tls_create(&without_gvl_key);
+  rb_define_module_function(mPyCall, "without_gvl", pycall_m_without_gvl, 0);
+
   /* PyCall::PyPtr */
 
   cPyPtr = rb_define_class_under(mPyCall, "PyPtr", rb_cBasicObject);

data/ext/pycall/pycall_internal.h

@@ -11,10 +11,19 @@ extern "C" {
 #include <ruby.h>
 #include <ruby/encoding.h>
 #include <ruby/thread.h>
+
 #include <assert.h>
 #include <inttypes.h>
 #include <limits.h>
 
+#if defined(_WIN32)
+# define PYCALL_THREAD_WIN32
+# include <ruby/win32.h>
+#elif defined(HAVE_PTHREAD_H)
+# define PYCALL_THREAD_PTHREAD
+# include <pthread.h>
+#endif
+
 #if SIZEOF_LONG == SIZEOF_VOIDP
 # define PTR2NUM(x)   (LONG2NUM((long)(x)))
 # define NUM2PTR(x)   ((void*)(NUM2ULONG(x)))
@@ -492,6 +501,23 @@ extern PyTypeObject PyRuby_Type;
 
 PyObject * PyRuby_New(VALUE ruby_object);
 
+/* ==== thread support ==== */
+
+#if   defined(PYCALL_THREAD_WIN32)
+typedef DWORD pycall_tls_key;
+#elif defined(PYCALL_THREAD_PTHREAD)
+typedef pthread_key_t pycall_tls_key;
+#else
+# error "unsupported thread type"
+#endif
+
+int pycall_tls_create(pycall_tls_key* tls_key);
+void *pycall_tls_get(pycall_tls_key tls_key);
+int pycall_tls_set(pycall_tls_key tls_key, void *ptr);
+
+int pycall_without_gvl_p(void);
+VALUE pycall_without_gvl(VALUE (* func)(VALUE), VALUE arg);
+
 /* ==== pycall ==== */
 
 typedef struct {

data/ext/pycall/thread.c

@@ -0,0 +1,36 @@
+#include "pycall_internal.h"
+
+#if defined(PYCALL_THREAD_WIN32)
+int pycall_tls_create(pycall_tls_key *tls_key)
+{
+  *tls_key = TlsAlloc();
+  return *tls_key == TLS_OUT_OF_INDEXES;
+}
+
+void *pycall_tls_get(pycall_tls_key tls_key)
+{
+  return TlsGetValue(tls_key);
+}
+
+int pycall_tls_set(pycall_tls_key tls_key, void *ptr)
+{
+  return 0 == TlsSetValue(tls_key, ptr);
+}
+#endif
+
+#if defined(PYCALL_THREAD_PTHREAD)
+int pycall_tls_create(pycall_tls_key *tls_key)
+{
+  return pthread_key_create(tls_key, NULL);
+}
+
+void *pycall_tls_get(pycall_tls_key tls_key)
+{
+  return pthread_getspecific(tls_key);
+}
+
+int pycall_tls_set(pycall_tls_key tls_key, void *ptr)
+{
+  return pthread_setspecific(tls_key, ptr);
+}
+#endif

data/lib/pycall/libpython/finder.rb

@@ -27,12 +27,12 @@ module PyCall
         def find_python_config(python = nil)
           python ||= DEFAULT_PYTHON
           Array(python).each do |python_cmd|
-            python_config = investigate_python_config(python_cmd)
-            return [python_cmd, python_config] unless python_config.empty?
+            begin
+              python_config = investigate_python_config(python_cmd)
+              return [python_cmd, python_config] unless python_config.empty?
+            rescue
+            end
           end
-        rescue
-          raise ::PyCall::PythonNotFound
-        else
           raise ::PyCall::PythonNotFound
         end
 

data/lib/pycall/version.rb

@@ -1,3 +1,9 @@
 module PyCall
-  VERSION = "1.2.1"
+  VERSION = "1.3.0-dev"
+
+  module Version
+    numbers, TAG = VERSION.split("-")
+    MAJOR, MINOR, MICRO = numbers.split(".").map(&:to_i)
+    STRING = VERSION
+  end
 end

data/pycall.gemspec

@@ -5,7 +5,13 @@ require 'pycall/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "pycall"
-  spec.version       = PyCall::VERSION
+  version_components = [
+    PyCall::Version::MAJOR.to_s,
+    PyCall::Version::MINOR.to_s,
+    PyCall::Version::MICRO.to_s,
+    PyCall::Version::TAG,
+  ]
+  spec.version       = version_components.compact.join(".")
   spec.authors       = ["Kenta Murata"]
   spec.email         = ["mrkn@mrkn.jp"]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pycall
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0.dev
 platform: ruby
 authors:
 - Kenta Murata
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-19 00:00:00.000000000 Z
+date: 2019-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -171,6 +171,7 @@ files:
 - ext/pycall/pycall_internal.h
 - ext/pycall/range.c
 - ext/pycall/ruby_wrapper.c
+- ext/pycall/thread.c
 - images/pycallrb_logo.png
 - images/pycallrb_logo_200.png
 - lib/pycall.rb
@@ -215,11 +216,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: pycall