cumo 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: febb3beb76c3e4994bee998b7d9911765a8f0279a7c0db550e45601a1c31c9da
-  data.tar.gz: 60c87eb1810387de34667847b354eaa7e3d8d09ca31e09bd06bf4bbba1190578
+  metadata.gz: 143344d9aa774213541e1e44bbb1f2d65348a2e9a410df67435c83a890db6c14
+  data.tar.gz: c59b67b28e70ed1421ddfceb6a7928c373634278e3f57a23ad631375e5e3111d
 SHA512:
-  metadata.gz: c7fe6ade2ebf7af02cae19cd403161f369cb65aaf13918597230f35f1ec7148f05b485c6d3db9063a02a60a9a197156de08b74a35abf51c65e4dd1e7e069243b
-  data.tar.gz: d2e291b33102290ed4f22fd83c7ab3b69f9aae3f58e789577b41c9492d1fea761adbcd3efeaa46f7d05534e50d0a8ba033aaab65934a4627f8afa56c8efea305
+  metadata.gz: 20dc89053b605cbc3f272fffb28a185885ea00c9d4be8646397e9d51703411118810961e00546b3f83f8984ec18518e796328305437eb1f22c9d4880fd1b4b16
+  data.tar.gz: 06acfc5740b18994aa4ea1276157098729c57768ef8b7eb1ff65f998959dee7d0f160697127392a02bdcfa9090f4cefa91b53729ec0cdcf0b17c7212a9143da4

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.4 (2019-05-04)
+
+Enhancements:
+
+* Support cuDNN fixed\_batch\_norm (cudnnBatchNormalizationForwardInference)
+
 # 0.3.3 (2019-05-02)
 
 Fixes:
@@ -26,7 +32,7 @@ Enhancements:
   * conv (cudnnConvolution)
   * conv\_transpose (cudnnConvolutionBackwardData)
   * conv\_grad\_w (cudnnConvolutionBackwardFilter)
-  * batch\_norm (cudnnBatchNormalization)
+  * batch\_norm (cudnnBatchNormalizationForwardTraining)
   * batch\_norm\_backward (cudnnBatchNormalizationBackward)
   * avg\_pool and max\_pool (cudnnPoolingForward)
   * avg\_pool\_backward and max\_pool\_backward (cudnnPoolingBackward)

data/ext/cumo/include/cumo.h

@@ -10,8 +10,8 @@ extern "C" {
 #endif
 #endif
 
-#define CUMO_VERSION "0.3.3"
-#define CUMO_VERSION_CODE 33
+#define CUMO_VERSION "0.3.4"
+#define CUMO_VERSION_CODE 34
 
 bool cumo_compatible_mode_enabled_p();
 bool cumo_show_warning_enabled_p();

data/ext/cumo/narray/gen/spec.rb

@@ -60,6 +60,7 @@ if is_float && !is_complex && !is_object
   def_id "conv_grad_w"
   def_id "batch_norm"
   def_id "batch_norm_backward"
+  def_id "fixed_batch_norm"
   def_id "pooling_forward"
   def_id "pooling_backward"
 end
@@ -348,6 +349,7 @@ if is_float && !is_complex && !is_object
   def_method "conv_grad_w" # conv_backward_filter
   def_method "batch_norm"
   def_method "batch_norm_backward"
+  def_method "fixed_batch_norm"
   def_method "pooling_forward" # max_pool, avg_pool
   def_method "pooling_backward"
 end

data/ext/cumo/narray/gen/tmpl/fixed_batch_norm.c

@@ -0,0 +1,149 @@
+#ifdef CUDNN_FOUND
+
+<%
+  cudnn_dtype =
+    case type_name
+    when 'sfloat'
+      'CUDNN_DATA_FLOAT'
+    when 'dfloat'
+      'CUDNN_DATA_DOUBLE'
+    else
+      # CUDNN_DATA_HALF
+      raise 'not supported'
+    end
+%>
+
+// y = x.fixed_batch_norm(gamma, beta, mean, var, eps:, axis:)
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+{
+    cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
+    cudnnStatus_t status = 0;
+    cudnnHandle_t handle = 0;
+    dtype coef_one = 1;
+    dtype coef_zero = 0;
+
+    VALUE x=self, gamma, beta, mean, var, eps, axis, y;
+    VALUE kw_hash = Qnil;
+    ID kw_table[] = {
+        rb_intern("eps"),
+        rb_intern("axis"),
+        rb_intern("y")
+    };
+    VALUE opts[] = {Qundef, Qundef, Qundef};
+
+    cumo_narray_t *nx; // , *ngamma, *nbeta;
+    size_t *x_shape; // *gamma_shape, *beta_shape, reduced_shape[CUMO_NA_MAX_DIMENSION];
+    size_t x_ndim;
+
+    VALUE x_cont, gamma_cont, beta_cont, mean_cont, var_cont;
+    cudnnTensorDescriptor_t x_desc = 0;
+    cudnnTensorDescriptor_t bn_desc = 0;
+    char *x_cont_ptr, *gamma_cont_ptr, *beta_cont_ptr, *mean_cont_ptr, *var_cont_ptr, *y_ptr;
+
+    cudnnBatchNormMode_t mode;
+
+    // default values
+    double double_eps = 2e-5;
+    int int_axis[CUMO_NA_MAX_DIMENSION] = {0};
+    size_t axis_ndim = 1;
+
+    rb_scan_args(argc, argv, "4:", &gamma, &beta, &mean, &var, &kw_hash);
+    rb_get_kwargs(kw_hash, kw_table, 0, 3, opts);
+    eps = cumo_cuda_cudnn_option_value(opts[0], Qnil);
+    axis = cumo_cuda_cudnn_option_value(opts[1], Qnil);
+    y = cumo_cuda_cudnn_option_value(opts[2], Qnil);
+
+    if (eps != Qnil) {
+        double_eps = NUM2DBL(eps);
+    }
+    if (axis != Qnil) {
+        axis_ndim = cumo_cuda_cudnn_get_int_axis(int_axis, axis);
+    }
+
+    CumoGetNArray(x, nx);
+    // CumoGetNArray(gamma, ngamma);
+    // CumoGetNArray(beta, nbeta);
+    x_ndim = nx->ndim;
+    x_shape = nx->shape;
+    // gamma_ndim = ngamma->ndim;
+    // gamma_shape = ngamma->shape;
+    // beta_ndim = nbeta->ndim;
+    // beta_shape = nbeta->shape;
+
+    // TODO: Size check of gammma, beta, running_mean, running_var, mean, inv_std
+    // are equivalent with either of reduced_shape(keepdims: false) or reduced_shape(keepdims: true)
+    // reduced_ndim = cumo_cuda_cudnn_ReduceShape(reduced_shape, x_ndim, x_shape, axis_ndim, int_axis, 1);
+    // CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_ndim, gamma_ndim);
+    // CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_ndim, beta_ndim);
+    // for (size_t idim = 0; idim < reduced_ndim; ++idim) {
+    //     CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_shape[idim], gamma_shape[idim]);
+    //     CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_shape[idim], beta_shape[idim]);
+    // }
+
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(x, cT);
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(gamma, cT);
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(beta, cT);
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(mean, cT);
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(var, cT);
+
+    x_cont = cumo_na_as_contiguous_array(x);
+    gamma_cont = cumo_na_as_contiguous_array(gamma);
+    beta_cont = cumo_na_as_contiguous_array(beta);
+    mean_cont = cumo_na_as_contiguous_array(mean);
+    var_cont = cumo_na_as_contiguous_array(var);
+
+    x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
+    gamma_cont_ptr = cumo_na_get_offset_pointer_for_read(gamma_cont);
+    beta_cont_ptr = cumo_na_get_offset_pointer_for_read(beta_cont);
+    mean_cont_ptr = cumo_na_get_offset_pointer_for_read(mean_cont);
+    var_cont_ptr = cumo_na_get_offset_pointer_for_read(var_cont);
+
+    // TODO: type and shape check
+    if (y == Qnil) y = cumo_na_new(cT, x_ndim, x_shape);
+    y_ptr = cumo_na_get_offset_pointer_for_write(y);
+
+    status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
+    if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
+
+    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
+    status = cumo_cuda_cudnn_CreateBNTensorDescriptor(&bn_desc, x_desc, mode);
+    if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
+    // TODO: bn_desc may return another type, and may need to cast gamma, beta, mean, var
+
+    handle = cumo_cuda_cudnn_handle();
+
+    status = cudnnBatchNormalizationForwardInference(
+            handle,
+            mode,
+            (void*)&coef_one,
+            (void*)&coef_zero,
+            x_desc,
+            x_cont_ptr,
+            x_desc,
+            y_ptr,
+            bn_desc,
+            gamma_cont_ptr,
+            beta_cont_ptr,
+            mean_cont_ptr,
+            var_cont_ptr,
+            double_eps);
+    if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
+
+BATCH_NORM_ERROR:
+    if (x_desc) cudnnDestroyTensorDescriptor(x_desc);
+    if (bn_desc) cudnnDestroyTensorDescriptor(bn_desc);
+    cumo_cuda_cudnn_check_status(status);
+
+    return y;
+}
+
+#else // CUDNN_FOUND
+VALUE cumo_cuda_eCUDNNError;
+
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+{
+    rb_raise(cumo_cuda_eCUDNNError, "cuDNN is not available");
+}
+#endif // CUDNN_FOUND

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cumo
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-02 00:00:00.000000000 Z
+date: 2019-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -240,6 +240,7 @@ files:
 - ext/cumo/narray/gen/tmpl/eye_kernel.cu
 - ext/cumo/narray/gen/tmpl/fill.c
 - ext/cumo/narray/gen/tmpl/fill_kernel.cu
+- ext/cumo/narray/gen/tmpl/fixed_batch_norm.c
 - ext/cumo/narray/gen/tmpl/float_accum_kernel.cu
 - ext/cumo/narray/gen/tmpl/format.c
 - ext/cumo/narray/gen/tmpl/format_to_a.c