cumo 0.2.5 → 0.3.0.pre1

data/ext/cumo/cuda/runtime.c

@@ -142,6 +142,7 @@ Init_cumo_cuda_runtime()
 {
     VALUE mCumo = rb_define_module("Cumo");
     VALUE mCUDA = rb_define_module_under(mCumo, "CUDA");
+    rb_define_const(mCumo, "Cuda", mCUDA); // alias
     mRuntime = rb_define_module_under(mCUDA, "Runtime");
     eRuntimeError = rb_define_class_under(mCUDA, "RuntimeError", rb_eStandardError);
 

data/ext/cumo/cumo.c

@@ -33,6 +33,8 @@ void Init_cumo_cuda_driver();
 void Init_cumo_cuda_memory_pool();
 void Init_cumo_cuda_runtime();
 void Init_cumo_cuda_nvrtc();
+void Init_cumo_cuda_cublas();
+void Init_cumo_cuda_cudnn();
 
 void
 cumo_debug_breakpoint(void)
@@ -167,4 +169,7 @@ Init_cumo()
     Init_cumo_cuda_memory_pool();
     Init_cumo_cuda_runtime();
     Init_cumo_cuda_nvrtc();
+
+    Init_cumo_cuda_cublas();
+    Init_cumo_cuda_cudnn();
 }

data/ext/cumo/extconf.rb

@@ -47,9 +47,9 @@ rm_f 'include/cumo/extconf.h'
 MakeMakefileCuda.install!(cxx: true)
 
 if ENV['DEBUG']
-  $CFLAGS="-g -O0 -Wall"
+  $CFLAGS << " -g -O0 -Wall"
 end
-$CXXFLAGS += " -std=c++14 "
+$CXXFLAGS << " -std=c++14"
 #$CFLAGS=" $(cflags) -O3 -m64 -msse2 -funroll-loops"
 #$CFLAGS=" $(cflags) -O3"
 $INCFLAGS = "-Iinclude -Inarray -Icuda #{$INCFLAGS}"
@@ -109,6 +109,8 @@ cuda/memory_pool
 cuda/memory_pool_impl
 cuda/runtime
 cuda/nvrtc
+cuda/cudnn
+cuda/cudnn_impl
 )
 
 if RUBY_VERSION[0..3] == "2.1."
@@ -179,5 +181,9 @@ have_library('nvrtc')
 have_library('cublas')
 # have_library('cusolver')
 # have_library('curand')
+if have_library('cudnn') # TODO(sonots): cuDNN version check
+  $CFLAGS << " -DCUDNN_FOUND"
+  $CXXFLAGS << " -DCUDNN_FOUND"
+end
 
 create_makefile('cumo')

data/ext/cumo/include/cumo.h

@@ -10,8 +10,8 @@ extern "C" {
 #endif
 #endif
 
-#define CUMO_VERSION "0.2.5"
-#define CUMO_VERSION_CODE 25
+#define CUMO_VERSION "0.3.0.pre1"
+#define CUMO_VERSION_CODE 301
 
 bool cumo_compatible_mode_enabled_p();
 bool cumo_show_warning_enabled_p();

data/ext/cumo/include/cumo/cuda/cudnn.h

@@ -0,0 +1,205 @@
+#ifndef CUMO_CUDA_CUDNN_H
+#define CUMO_CUDA_CUDNN_H
+
+#include <ruby.h>
+#ifdef CUDNN_FOUND
+#include <cudnn.h>
+#endif // CUDNN_FOUND
+
+#if defined(__cplusplus)
+extern "C" {
+#if 0
+} /* satisfy cc-mode */
+#endif
+#endif
+
+#ifdef CUDNN_FOUND
+
+VALUE cumo_na_eShapeError;
+
+#define CUMO_CUDA_CUDNN_DEFAULT_MAX_WORKSPACE_SIZE 8 * 1024 * 1024
+
+// TODO: Move to proper generic place
+#define CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(x,t)                 \
+    if (rb_obj_class(x)!=(t)) {                                \
+        rb_raise(rb_eTypeError,"invalid NArray type (class)"); \
+    }
+
+// TODO: Move to proper generic place
+#define CUMO_CUDA_CUDNN_CHECK_SIZE_EQ(sz1,sz2)        \
+    if ((sz1) != (sz2)) {                            \
+        rb_raise(cumo_na_eShapeError,                \
+                 "size mismatch: %d != %d",     \
+                 (int)(sz1), (int)(sz2));            \
+    }
+
+// TODO: Move to proper generic place
+#define CUMO_CUDA_CUDNN_CHECK_DIM_EQ(nd1,nd2)        \
+    if ((nd1) != (nd2)) {                            \
+        rb_raise(cumo_na_eShapeError,                \
+                 "dimention mismatch: %d != %d",     \
+                 (int)(nd1), (int)(nd2));            \
+    }
+
+void
+cumo_cuda_cudnn_check_status(cudnnStatus_t status);
+
+cudnnHandle_t
+cumo_cuda_cudnn_handle();
+
+// TODO: Move to more generic proper place
+static inline VALUE
+cumo_cuda_cudnn_option_value(VALUE value, VALUE default_value)
+{
+    switch(TYPE(value)) {
+    case T_NIL:
+    case T_UNDEF:
+        return default_value;
+    }
+    return value;
+}
+
+// VALUE is Ruby Array
+static inline void
+cumo_cuda_cudnn_get_int_ary(int* int_ary, VALUE ary, size_t ndim, int default_value)
+{
+    if (ary == Qnil) {
+        // default to 1
+        for (size_t idim = 0; idim < ndim; ++idim) {
+            int_ary[idim] = default_value;
+        }
+    } else if (TYPE(ary) == T_FIXNUM) {
+        for (size_t idim = 0; idim < ndim; ++idim) {
+            int_ary[idim] = NUM2INT(ary);
+        }
+    } else {
+        Check_Type(ary, T_ARRAY);
+        CUMO_CUDA_CUDNN_CHECK_DIM_EQ((size_t)(RARRAY_LEN(ary)), ndim);
+        for (size_t idim = 0; idim < ndim; ++idim) {
+            int_ary[idim] = NUM2INT(rb_ary_entry(ary, idim));
+        }
+    }
+}
+
+size_t
+cumo_cuda_cudnn_GetConvOutDim(
+        size_t in_dim,
+        size_t kernel_size,
+        size_t stride,
+        size_t pad);
+
+size_t
+cumo_cuda_cudnn_GetConvTransposeOutDim(
+        size_t in_dim,
+        size_t kernel_size,
+        size_t stride,
+        size_t pad);
+
+cudnnStatus_t
+cumo_cuda_cudnn_CreateTensorDescriptor(
+        cudnnTensorDescriptor_t *desc,
+        VALUE a,
+        cudnnDataType_t cudnn_dtype);
+
+cudnnStatus_t
+cumo_cuda_cudnn_CreateFilterDescriptor(
+        cudnnFilterDescriptor_t *desc,
+        VALUE a,
+        cudnnDataType_t cudnn_dtype);
+
+cudnnStatus_t
+cumo_cuda_cudnn_CreateConvolutionDescriptor(
+        cudnnConvolutionDescriptor_t *desc,
+        size_t ndim,
+        int* int_stride,
+        int* int_pad,
+        cudnnDataType_t cudnn_dtype);
+
+cudnnStatus_t
+cumo_cuda_cudnn_CreatePoolingDescriptor(
+        cudnnPoolingDescriptor_t *desc,
+        cudnnPoolingMode_t mode,
+        size_t ndim,
+        int* int_kernel_size,
+        int* int_stride,
+        int* int_pad);
+
+cudnnStatus_t
+cumo_cuda_cudnn_FindConvolutionForwardAlgorithm(
+        cudnnConvolutionFwdAlgoPerf_t *perf_result,
+        cudnnHandle_t handle,
+        cudnnTensorDescriptor_t x_desc,
+        VALUE x,
+        cudnnFilterDescriptor_t w_desc,
+        VALUE w,
+        cudnnConvolutionDescriptor_t conv_desc,
+        cudnnTensorDescriptor_t y_sec,
+        VALUE y,
+        size_t max_workspace_size,
+        int* int_stride,
+        int* int_pad,
+        size_t ndim,
+        cudnnDataType_t cudnn_dtype);
+
+cudnnStatus_t
+cumo_cuda_cudnn_FindConvolutionBackwardDataAlgorithm(
+        cudnnConvolutionBwdDataAlgoPerf_t *perf_result,
+        cudnnHandle_t handle,
+        cudnnFilterDescriptor_t w_desc,
+        VALUE w,
+        cudnnTensorDescriptor_t x_desc,
+        VALUE x,
+        cudnnConvolutionDescriptor_t conv_desc,
+        cudnnTensorDescriptor_t y_desc,
+        VALUE y,
+        size_t max_workspace_size,
+        int* int_stride,
+        int* int_pad,
+        size_t ndim,
+        cudnnDataType_t cudnn_dtype);
+
+cudnnStatus_t
+cumo_cuda_cudnn_FindConvolutionBackwardFilterAlgorithm(
+        cudnnConvolutionBwdFilterAlgoPerf_t *perf_result,
+        cudnnHandle_t handle,
+        cudnnTensorDescriptor_t x_desc,
+        VALUE x,
+        cudnnTensorDescriptor_t dy_desc,
+        VALUE dy,
+        cudnnConvolutionDescriptor_t conv_desc,
+        cudnnFilterDescriptor_t dw_desc,
+        VALUE dw,
+        size_t max_workspace_size,
+        int* int_stride,
+        int* int_pad,
+        size_t ndim,
+        cudnnDataType_t cudnn_dtype);
+
+cudnnBatchNormMode_t
+cumo_cuda_cudnn_GetBatchNormMode(size_t ndim, int* int_axis);
+
+cudnnStatus_t
+cumo_cuda_cudnn_CreateBNTensorDescriptor(
+        cudnnTensorDescriptor_t *desc,
+        cudnnTensorDescriptor_t x_desc,
+        cudnnBatchNormMode_t mode);
+
+size_t
+cumo_cuda_cudnn_ReduceShape(
+        size_t *reduced_shape,
+        size_t shape_ndim,
+        size_t *shape,
+        size_t axes_ndim,
+        int *axes,
+        char keepdims);
+
+#endif // CUDNN_FOUND
+
+#if defined(__cplusplus)
+#if 0
+{ /* satisfy cc-mode */
+#endif
+}  /* extern "C" { */
+#endif
+
+#endif /* ifndef CUMO_CUDA_CUDNN_H */

data/ext/cumo/include/cumo/hash_combine.hpp

@@ -0,0 +1,17 @@
+#ifndef CUMO_HASH_COMBINE_H
+#define CUMO_HASH_COMBINE_H
+
+#include <cstddef>
+
+namespace cumo {
+namespace internal {
+
+// Borrowed from boost::hash_combine
+//
+// TODO(sonots): hash combine in 64bit
+inline void HashCombine(std::size_t& seed, std::size_t hash_value) { seed ^= hash_value + 0x9e3779b9 + (seed << 6) + (seed >> 2); }
+
+}  // namespace internal
+}  // namespace cumo
+
+#endif /* ifndef CUMO_HASH_COMBINE_H */

data/ext/cumo/include/cumo/intern.h

@@ -26,11 +26,16 @@ char *cumo_na_get_pointer_for_write(VALUE);
 char *cumo_na_get_pointer_for_read(VALUE);
 char *cumo_na_get_pointer_for_read_write(VALUE);
 size_t cumo_na_get_offset(VALUE self);
+char* cumo_na_get_offset_pointer(VALUE);
+char* cumo_na_get_offset_pointer_for_write(VALUE);
+char* cumo_na_get_offset_pointer_for_read(VALUE);
+char* cumo_na_get_offset_pointer_for_read_write(VALUE);
 
 void cumo_na_copy_flags(VALUE src, VALUE dst);
 
 VALUE cumo_na_check_ladder(VALUE self, int start_dim);
 VALUE cumo_na_check_contiguous(VALUE self);
+VALUE cumo_na_as_contiguous_array(VALUE a);
 
 VALUE cumo_na_flatten_dim(VALUE self, int sd);
 

data/ext/cumo/include/cumo/types/dfloat.h

@@ -7,6 +7,7 @@ typedef double rtype;
 #include "float_macro.h"
 #include "cublas_v2.h"
 #include "cumo/cuda/cublas.h"
+#include "cumo/cuda/cudnn.h"
 
 #ifdef SFMT_H
 /* generates a random number on [0,1)-real-interval */

data/ext/cumo/include/cumo/types/sfloat.h

@@ -7,6 +7,7 @@ typedef float rtype;
 #include "float_macro.h"
 #include "cublas_v2.h"
 #include "cumo/cuda/cublas.h"
+#include "cumo/cuda/cudnn.h"
 
 #ifdef SFMT_H
 /* generates a random number on [0,1)-real-interval */

data/ext/cumo/narray/gen/spec.rb

@@ -53,6 +53,16 @@ end
 if (is_float || is_complex) && !is_object
   def_id "gemm"
 end
+# cudnn
+if is_float && !is_complex && !is_object
+  def_id "conv"
+  def_id "conv_transpose"
+  def_id "conv_grad_w"
+  def_id "batch_norm"
+  def_id "batch_norm_backward"
+  def_id "pooling_forward"
+  def_id "pooling_backward"
+end
 
 if is_int && !is_object
   def_id "minlength" # for bincount
@@ -331,6 +341,17 @@ if (is_float || is_complex) && !is_object
   def_method "gemm"
 end
 
+# cudnn
+if is_float && !is_complex && !is_object
+  def_method "conv"
+  def_method "conv_transpose" # conv_backward_data
+  def_method "conv_grad_w" # conv_backward_filter
+  def_method "batch_norm"
+  def_method "batch_norm_backward"
+  def_method "pooling_forward" # max_pool, avg_pool
+  def_method "pooling_backward"
+end
+
 # rmsdev
 # prod
 

data/ext/cumo/narray/gen/tmpl/batch_norm.c

@@ -0,0 +1,197 @@
+#ifdef CUDNN_FOUND
+
+<%
+  cudnn_dtype =
+    case type_name
+    when 'sfloat'
+      'CUDNN_DATA_FLOAT'
+    when 'dfloat'
+      'CUDNN_DATA_DOUBLE'
+    else
+      # CUDNN_DATA_HALF
+      raise 'not supported'
+    end
+%>
+
+// y = x.batch_norm(gamma, beta, running_mean:, running_var:, eps:, decay:, axis:, mean:, inv_std:)
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+{
+    cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
+    cudnnStatus_t status = 0;
+    cudnnHandle_t handle = 0;
+    dtype coef_alpha = 1;
+    dtype coef_beta = 0;
+
+    VALUE x=self, gamma, beta, running_mean, running_var, eps, decay, axis, mean, inv_std, y;
+    VALUE kw_hash = Qnil;
+    ID kw_table[] = {
+        rb_intern("running_mean"),
+        rb_intern("running_var"),
+        rb_intern("mean"),
+        rb_intern("inv_std"),
+        rb_intern("eps"),
+        rb_intern("decay"),
+        rb_intern("axis"),
+        rb_intern("y")
+    };
+    VALUE opts[] = {Qundef, Qundef, Qundef, Qundef, Qundef, Qundef, Qundef, Qundef};
+
+    cumo_narray_t *nx, *ngamma, *nbeta;
+    size_t *x_shape, *gamma_shape, *beta_shape, reduced_shape[CUMO_NA_MAX_DIMENSION];
+    size_t x_ndim, gamma_ndim, beta_ndim, reduced_ndim;
+
+    VALUE x_cont, gamma_cont, beta_cont;
+    cudnnTensorDescriptor_t x_desc = 0;
+    cudnnTensorDescriptor_t bn_desc = 0;
+    char *x_cont_ptr, *gamma_cont_ptr, *beta_cont_ptr, *y_ptr;
+
+    cudnnBatchNormMode_t mode;
+
+    // default values
+    char *running_mean_ptr=NULL;
+    char *running_var_ptr=NULL;
+    char *mean_ptr=NULL;
+    char *inv_std_ptr=NULL;
+    double double_eps = 2e-5;
+    double double_decay = 0.9;
+    int int_axis[CUMO_NA_MAX_DIMENSION] = {0};
+    size_t axis_ndim = 1;
+
+    rb_scan_args(argc, argv, "2:", &gamma, &beta, &kw_hash);
+    rb_get_kwargs(kw_hash, kw_table, 0, 8, opts);
+    running_mean = cumo_cuda_cudnn_option_value(opts[0], Qnil);
+    running_var = cumo_cuda_cudnn_option_value(opts[1], Qnil);
+    mean = cumo_cuda_cudnn_option_value(opts[2], Qnil);
+    inv_std = cumo_cuda_cudnn_option_value(opts[3], Qnil);
+    eps = cumo_cuda_cudnn_option_value(opts[4], Qnil);
+    decay = cumo_cuda_cudnn_option_value(opts[5], Qnil);
+    axis = cumo_cuda_cudnn_option_value(opts[6], Qnil);
+    y = cumo_cuda_cudnn_option_value(opts[7], Qnil);
+
+    if (running_mean != Qnil) {
+        running_mean_ptr = cumo_na_get_offset_pointer_for_write(running_mean);
+    }
+    if (running_var != Qnil) {
+        running_var_ptr = cumo_na_get_offset_pointer_for_write(running_var);
+    }
+    if (mean != Qnil) {
+        mean_ptr = cumo_na_get_offset_pointer_for_write(mean);
+    }
+    if (inv_std != Qnil) {
+        inv_std_ptr = cumo_na_get_offset_pointer_for_write(inv_std);
+    }
+    if (eps != Qnil) {
+        double_eps = NUM2DBL(eps);
+    }
+    if (decay != Qnil) {
+        double_decay = NUM2DBL(decay);
+    }
+    if (axis != Qnil) {
+        Check_Type(axis, T_ARRAY);
+        axis_ndim = (size_t)(RARRAY_LEN(axis));
+        for (size_t idim = 0; idim < axis_ndim; ++idim) {
+            int_axis[idim] = NUM2INT(rb_ary_entry(axis, (long)idim));
+        }
+        // TODO: check axis is sorted
+    }
+
+    CumoGetNArray(x, nx);
+    CumoGetNArray(gamma, ngamma);
+    CumoGetNArray(beta, nbeta);
+    x_ndim = nx->ndim;
+    x_shape = nx->shape;
+    gamma_ndim = ngamma->ndim;
+    gamma_shape = ngamma->shape;
+    beta_ndim = nbeta->ndim;
+    beta_shape = nbeta->shape;
+
+    // TODO: Size check of gammma, beta, running_mean, running_var, mean, inv_std
+    // are equivalent with either of reduced_shape(keepdims: false) or reduced_shape(keepdims: true)
+    reduced_ndim = cumo_cuda_cudnn_ReduceShape(reduced_shape, x_ndim, x_shape, axis_ndim, int_axis, 1);
+    // CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_ndim, gamma_ndim);
+    // CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_ndim, beta_ndim);
+    // for (size_t idim = 0; idim < reduced_ndim; ++idim) {
+    //     CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_shape[idim], gamma_shape[idim]);
+    //     CUMO_CUDA_CUDNN_CHECK_DIM_EQ(reduced_shape[idim], beta_shape[idim]);
+    // }
+
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(x, cT);
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(gamma, cT);
+    CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(beta, cT);
+    if (running_mean != Qnil) CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(running_mean, cT);
+    if (running_var != Qnil) CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(running_var, cT);
+    if (mean != Qnil) CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(mean, cT);
+    if (inv_std != Qnil) CUMO_CUDA_CUDNN_CHECK_NARRAY_TYPE(inv_std, cT);
+
+    x_cont = cumo_na_as_contiguous_array(x);
+    gamma_cont = cumo_na_as_contiguous_array(gamma);
+    beta_cont = cumo_na_as_contiguous_array(beta);
+    if (running_mean != Qnil && cumo_na_check_contiguous(running_mean) != Qtrue) {
+        rb_raise(rb_eRuntimeError, "running_mean must be contiguous");
+    }
+    if (running_var != Qnil && cumo_na_check_contiguous(running_var) != Qtrue) {
+        rb_raise(rb_eRuntimeError, "running_var must be contiguous");
+    }
+    if (mean != Qnil && cumo_na_check_contiguous(mean) != Qtrue) {
+        rb_raise(rb_eRuntimeError, "mean must be contiguous");
+    }
+    if (inv_std != Qnil && cumo_na_check_contiguous(inv_std) != Qtrue) {
+        rb_raise(rb_eRuntimeError, "inv_std must be contiguous");
+    }
+
+    x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
+    gamma_cont_ptr = cumo_na_get_offset_pointer_for_read(gamma_cont);
+    beta_cont_ptr = cumo_na_get_offset_pointer_for_read(beta_cont);
+
+    // TODO: type and shape check
+    if (y == Qnil) y = cumo_na_new(cT, x_ndim, x_shape);
+    y_ptr = cumo_na_get_offset_pointer_for_write(y);
+
+    status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
+    if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
+
+    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
+    status = cumo_cuda_cudnn_CreateBNTensorDescriptor(&bn_desc, x_desc, mode);
+    if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
+    // TODO: bn_desc may return another type, and may need to cast gamma, beta, mean, var
+
+    handle = cumo_cuda_cudnn_handle();
+
+    status = cudnnBatchNormalizationForwardTraining(
+            handle,
+            mode,
+            (void*)&coef_alpha,
+            (void*)&coef_beta,
+            x_desc,
+            x_cont_ptr,
+            x_desc,
+            y_ptr,
+            bn_desc,
+            gamma_cont_ptr,
+            beta_cont_ptr,
+            1.0 - double_decay,
+            running_mean_ptr,
+            running_var_ptr,
+            double_eps,
+            mean_ptr,
+            inv_std_ptr);
+    if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
+
+BATCH_NORM_ERROR:
+    if (x_desc) cudnnDestroyTensorDescriptor(x_desc);
+    if (bn_desc) cudnnDestroyTensorDescriptor(bn_desc);
+    cumo_cuda_cudnn_check_status(status);
+
+    return y;
+}
+
+#else // CUDNN_FOUND
+VALUE cumo_cuda_eCUDNNError;
+
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+{
+    rb_raise(cumo_cuda_eCUDNNError, "cuDNN is not available");
+}
+#endif // CUDNN_FOUND