cumo 0.6.0 → 0.7.0

data/ext/cumo/narray/gen/def/bit.rb

@@ -18,6 +18,7 @@ set is_object:     false
 set is_real:       false
 set is_comparable: false
 set is_double_precision: false
+set is_half:             false
 set need_align: false
 
 upcast_rb "Integer"
@@ -29,6 +30,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int32"
 upcast "Int16",  "Int16"

data/ext/cumo/narray/gen/def/dcomplex.rb

@@ -20,6 +20,7 @@ set is_complex:          true
 set is_object:           false
 set is_comparable:       false
 set is_double_precision: true
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -31,6 +32,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "DComplex"
 upcast "DFloat",   "DComplex"
 upcast "SFloat",   "DComplex"
+upcast "HFloat",   "DComplex"
 upcast "Int64",    "DComplex"
 upcast "Int32",    "DComplex"
 upcast "Int16",    "DComplex"

data/ext/cumo/narray/gen/def/dfloat.rb

@@ -18,8 +18,15 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: true
+set is_half:             false
 set need_align:          true
 
+set cudnn_dtype:         "CUDNN_DATA_DOUBLE"
+set cudnn_compute_dtype: "CUDNN_DATA_DOUBLE"
+set cudnn_scalar_t:      "dtype"
+set cudnn_param_class:   "cT"
+set cudnn_math_type:     "CUDNN_DEFAULT_MATH"
+
 upcast_rb "Integer"
 upcast_rb "Float"
 upcast_rb "Complex", "DComplex"
@@ -29,6 +36,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "DComplex"
 upcast "DFloat",   "DFloat"
 upcast "SFloat",   "DFloat"
+upcast "HFloat",   "DFloat"
 upcast "Int64",    "DFloat"
 upcast "Int32",    "DFloat"
 upcast "Int16",    "DFloat"

data/ext/cumo/narray/gen/def/hfloat.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+set name:                "hfloat"
+set type_name:           "hfloat"
+set full_class_name:     "Cumo::HFloat"
+set class_name:          "HFloat"
+set class_alias:         "Float16"
+set class_var:           "cT"
+set ctype:               "cumo_half"
+
+set has_math:            true
+set is_bit:              false
+set is_int:              false
+set is_unsigned:         false
+set is_float:            true
+set is_complex:          false
+set is_object:           false
+set is_real:             true
+set is_comparable:       true
+set is_double_precision: false
+set is_half:             true
+set need_align:          true
+
+set cudnn_dtype:         "CUDNN_DATA_HALF"
+set cudnn_compute_dtype: "CUDNN_DATA_FLOAT"
+set cudnn_scalar_t:      "float"
+set cudnn_param_class:   "cumo_cSFloat"
+set cudnn_math_type:     "CUDNN_TENSOR_OP_MATH"
+
+upcast_rb "Integer"
+upcast_rb "Float"
+upcast_rb "Complex", "SComplex"
+
+upcast "RObject",  "RObject"
+upcast "DComplex", "DComplex"
+upcast "SComplex", "SComplex"
+upcast "DFloat",   "DFloat"
+upcast "SFloat",   "SFloat"
+upcast "HFloat",   "HFloat"
+upcast "Int64",    "HFloat"
+upcast "Int32",    "HFloat"
+upcast "Int16",    "HFloat"
+upcast "Int8",     "HFloat"
+upcast "UInt64",   "HFloat"
+upcast "UInt32",   "HFloat"
+upcast "UInt16",   "HFloat"
+upcast "UInt8",    "HFloat"

data/ext/cumo/narray/gen/def/int16.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -28,6 +29,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int32"
 upcast "Int16"

data/ext/cumo/narray/gen/def/int32.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -28,6 +29,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32"
 upcast "Int16"

data/ext/cumo/narray/gen/def/int64.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -28,6 +29,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64"
 upcast "Int32"
 upcast "Int16"

data/ext/cumo/narray/gen/def/int8.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          false
 
 upcast_rb "Integer"
@@ -28,6 +29,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int32"
 upcast "Int16",  "Int16"

data/ext/cumo/narray/gen/def/robject.rb

@@ -19,6 +19,7 @@ set is_complex:          false
 set is_object:           true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          false
 
 upcast_rb "Integer"
@@ -29,6 +30,7 @@ upcast "DComplex", "RObject"
 upcast "SComplex", "RObject"
 upcast "DFloat",   "RObject"
 upcast "SFloat",   "RObject"
+upcast "HFloat",   "RObject"
 upcast "Int64",    "RObject"
 upcast "Int32",    "RObject"
 upcast "Int16",    "RObject"

data/ext/cumo/narray/gen/def/scomplex.rb

@@ -20,6 +20,7 @@ set is_complex:          true
 set is_object:           false
 set is_comparable:       false
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -31,6 +32,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat",   "DComplex"
 upcast "SFloat",   "SComplex"
+upcast "HFloat",   "SComplex"
 upcast "Int64",    "SComplex"
 upcast "Int32",    "SComplex"
 upcast "Int16",    "SComplex"

data/ext/cumo/narray/gen/def/sfloat.rb

@@ -18,8 +18,15 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
+set cudnn_dtype:         "CUDNN_DATA_FLOAT"
+set cudnn_compute_dtype: "CUDNN_DATA_FLOAT"
+set cudnn_scalar_t:      "dtype"
+set cudnn_param_class:   "cT"
+set cudnn_math_type:     "CUDNN_DEFAULT_MATH"
+
 upcast_rb "Integer"
 upcast_rb "Float"
 upcast_rb "Complex", "SComplex"
@@ -29,6 +36,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat",   "DFloat"
 upcast "SFloat",   "SFloat"
+upcast "HFloat",   "SFloat"
 upcast "Int64",    "SFloat"
 upcast "Int32",    "SFloat"
 upcast "Int16",    "SFloat"

data/ext/cumo/narray/gen/def/uint16.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -28,10 +29,11 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int32"
 upcast "Int16",  "Int16"
-upcast "Int8",   "Int16"
+upcast "Int8",   "Int32"
 upcast "UInt64", "UInt64"
 upcast "UInt32", "UInt32"
 upcast "UInt16"

data/ext/cumo/narray/gen/def/uint32.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -28,10 +29,11 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int32"
 upcast "Int16",  "Int32"
-upcast "Int8",   "Int32"
+upcast "Int8",   "Int64"
 upcast "UInt64", "UInt64"
 upcast "UInt32"
 upcast "UInt16"

data/ext/cumo/narray/gen/def/uint64.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          true
 
 upcast_rb "Integer"
@@ -28,6 +29,7 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int64"
 upcast "Int16",  "Int64"

data/ext/cumo/narray/gen/def/uint8.rb

@@ -17,6 +17,7 @@ set is_object:           false
 set is_real:             true
 set is_comparable:       true
 set is_double_precision: false
+set is_half:             false
 set need_align:          false
 
 upcast_rb "Integer"
@@ -28,10 +29,11 @@ upcast "DComplex", "DComplex"
 upcast "SComplex", "SComplex"
 upcast "DFloat", "DFloat"
 upcast "SFloat", "SFloat"
+upcast "HFloat",   "HFloat"
 upcast "Int64",  "Int64"
 upcast "Int32",  "Int32"
 upcast "Int16",  "Int16"
-upcast "Int8",   "Int8"
+upcast "Int8",   "Int16"
 upcast "UInt64", "UInt64"
 upcast "UInt32", "UInt32"
 upcast "UInt16", "UInt16"

data/ext/cumo/narray/gen/erbpp2.rb

@@ -110,6 +110,9 @@ class ErbPP
   def init_def
   end
 
+  def upcast_def
+  end
+
   def find_tmpl(name)
     @parent.children.find { |x| x.name == name }
   end
@@ -212,6 +215,9 @@ class DefClass < DefModule
   def init_erb
     @opts[:init_erb] || "init_class"
   end
+  def upcast_def
+    load_erb("init_upcast").result(binding)
+  end
   def super_class
     @opts[:super_class] || "rb_cObject"
   end

data/ext/cumo/narray/gen/spec.rb

@@ -127,6 +127,7 @@ def_method "store" do
     store_from "DComplex", "cumo_dcomplex", "m_from_dcomplex"
     store_from "SComplex", "cumo_scomplex", "m_from_scomplex"
   end
+  store_from "HFloat", "cumo_half", "m_from_half"
   store_from "DFloat", "double",   "m_from_real"
   store_from "SFloat", "float",    "m_from_real"
   store_from "Int64", "int64_t",  "m_from_int64"

data/ext/cumo/narray/gen/tmpl/accum.c

@@ -23,7 +23,7 @@ static void
         p2 = lp->args[1].ptr + lp->args[1].iter[0].pos;
 
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%><%=nan%>", "<%=type_name%>");
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
         *(<%=dtype%>*)p2 = f_<%=name%><%=nan%>(n,p1,s1);
     }
     //<% elsif !indexer_ops.include?(name) %>

data/ext/cumo/narray/gen/tmpl/accum_arg.c

@@ -20,7 +20,7 @@ static void
         o_ptr = CUMO_NDL_PTR(lp,1);
 
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%><%=nan%>", "<%=type_name%>");
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
         idx = f_<%=name[3..5]%>_index<%=nan%>(n,d_ptr,d_step);
         *(idx_t*)o_ptr = (idx_t)idx;
     }

data/ext/cumo/narray/gen/tmpl/accum_arg_kernel.cu

@@ -33,7 +33,7 @@ struct cumo_<%=type_name%>_argmin_int<%=i%>_impl {
             return;
         }
 <% end %>
-        if (accum.value > next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_gt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };
@@ -56,7 +56,7 @@ struct cumo_<%=type_name%>_argmin_nan_int<%=i%>_impl {
             if (next_nan && (!accum_nan || next.index < accum.index)) { accum = next; }
             return;
         }
-        if (accum.value > next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_gt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };
@@ -83,7 +83,7 @@ struct cumo_<%=type_name%>_argmax_int<%=i%>_impl {
             return;
         }
 <% end %>
-        if (accum.value < next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_lt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };
@@ -97,7 +97,7 @@ struct cumo_<%=type_name%>_argmax_nan_int<%=i%>_impl {
         dtype value;
         idx_t index;
     };
-    __device__ ValueAndIndex Identity(idx_t /*index*/) { return {-(dtype)INFINITY, INT<%=i%>_MAX}; }
+    __device__ ValueAndIndex Identity(idx_t /*index*/) { return {(dtype)(-INFINITY), INT<%=i%>_MAX}; }
     __device__ ValueAndIndex MapIn(dtype in, idx_t index) { return {in, index}; }
     __device__ void Reduce(ValueAndIndex next, ValueAndIndex& accum) {
         bool accum_nan = !not_nan(accum.value);
@@ -106,7 +106,7 @@ struct cumo_<%=type_name%>_argmax_nan_int<%=i%>_impl {
             if (next_nan && (!accum_nan || next.index < accum.index)) { accum = next; }
             return;
         }
-        if (accum.value < next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_lt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };

data/ext/cumo/narray/gen/tmpl/accum_binary_kernel.cu

@@ -14,8 +14,20 @@
 
 // The product is never an array of its own: MapIn takes one element of each
 // operand, which is what a zip reduction hands it. The accumulator stays dtype
-// rather than widening, as the host loop it replaces did.
+// rather than widening, as the host loop it replaces did, except for half,
+// which cannot carry a sum of more than a couple of thousand terms.
 struct <%="cumo_#{type_name}_#{name}#{nan}_impl"%> {
+<% if is_half %>
+    __device__ float Identity(int64_t /*index*/) { return 0.0f; }
+    __device__ float MapIn(dtype x, dtype y, int64_t /*index*/) {
+<% if nan == '_nan' %>
+        if (!not_nan(x) || !not_nan(y)) { return 0.0f; }
+<% end %>
+        return cumo_half2float(x) * cumo_half2float(y);
+    }
+    __device__ void Reduce(float next, float& accum) { accum = next + accum; }
+    __device__ dtype MapOut(float accum) { return cumo_float2half(accum); }
+<% else %>
     __device__ dtype Identity(int64_t /*index*/) { return m_zero; }
     __device__ dtype MapIn(dtype x, dtype y, int64_t /*index*/) {
         dtype z = m_zero;
@@ -24,6 +36,7 @@ struct <%="cumo_#{type_name}_#{name}#{nan}_impl"%> {
     }
     __device__ void Reduce(dtype next, dtype& accum) { accum = m_add(next, accum); }
     __device__ dtype MapOut(dtype accum) { return accum; }
+<% end %>
 };
 //<% end %>
 

data/ext/cumo/narray/gen/tmpl/accum_index.c

@@ -21,7 +21,7 @@ static void
         o_ptr = CUMO_NDL_PTR(lp,2);
 
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%><%=nan%>", "<%=type_name%>");
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
         idx = f_<%=name%><%=nan%>(n,d_ptr,d_step);
         *(idx_t*)o_ptr = *(idx_t*)(i_ptr + i_step * idx);
     }

data/ext/cumo/narray/gen/tmpl/accum_index_kernel.cu

@@ -33,7 +33,7 @@ struct cumo_<%=type_name%>_min_index_int<%=i%>_impl {
             return;
         }
 <% end %>
-        if (accum.value > next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_gt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };
@@ -56,7 +56,7 @@ struct cumo_<%=type_name%>_min_index_nan_int<%=i%>_impl {
             if (next_nan && (!accum_nan || next.index < accum.index)) { accum = next; }
             return;
         }
-        if (accum.value > next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_gt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };
@@ -83,7 +83,7 @@ struct cumo_<%=type_name%>_max_index_int<%=i%>_impl {
             return;
         }
 <% end %>
-        if (accum.value < next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_lt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };
@@ -97,7 +97,7 @@ struct cumo_<%=type_name%>_max_index_nan_int<%=i%>_impl {
         dtype value;
         idx_t index;
     };
-    __device__ ValueAndIndex Identity(idx_t /*index*/) { return {-(dtype)INFINITY, INT<%=i%>_MAX}; }
+    __device__ ValueAndIndex Identity(idx_t /*index*/) { return {(dtype)(-INFINITY), INT<%=i%>_MAX}; }
     __device__ ValueAndIndex MapIn(dtype in, idx_t index) { return {in, index}; }
     __device__ void Reduce(ValueAndIndex next, ValueAndIndex& accum) {
         bool accum_nan = !not_nan(accum.value);
@@ -106,7 +106,7 @@ struct cumo_<%=type_name%>_max_index_nan_int<%=i%>_impl {
             if (next_nan && (!accum_nan || next.index < accum.index)) { accum = next; }
             return;
         }
-        if (accum.value < next.value || (accum.value == next.value && next.index < accum.index)) { accum = next; }
+        if (m_lt(accum.value, next.value) || (m_eq(accum.value, next.value) && next.index < accum.index)) { accum = next; }
     }
     __device__ idx_t MapOut(ValueAndIndex accum) { return accum.index; }
 };

data/ext/cumo/narray/gen/tmpl/alloc_func.c

@@ -5,7 +5,7 @@ static size_t
         return 0;
     }
   <% if is_bit %>
-    return ((na->base.size-1)/8/sizeof(CUMO_BIT_DIGIT)+1)*sizeof(CUMO_BIT_DIGIT);
+    return CUMO_BIT_BYTES(na->base.size);
   <% else %>
     return na->base.size * sizeof(dtype);
   <% end %>
@@ -38,14 +38,12 @@ static void
     assert(na->base.type == CUMO_NARRAY_DATA_T);
 
     if (na->ptr != NULL) {
-        if (na->owned) {
   <% if is_object %>
-            xfree(na->ptr);
+        xfree(na->ptr);
   <% else %>
-            cumo_cuda_runtime_free_no_raise(na->ptr);
-            rb_gc_adjust_memory_usage(-(ssize_t)<%=type_name%>_data_bytes(na));
+        cumo_cuda_runtime_free_no_raise(na->ptr);
+        rb_gc_adjust_memory_usage(-(ssize_t)<%=type_name%>_data_bytes(na));
   <% end %>
-        }
         na->ptr = NULL;
     }
     if (na->base.shape != NULL && na->base.shape != &(na->base.size)) {
@@ -117,6 +115,5 @@ static VALUE
     na->base.shape = NULL;
     na->base.reduce = INT2FIX(0);
     na->ptr = NULL;
-    na->owned = FALSE;
     return TypedData_Wrap_Struct(klass, &<%=type_name%>_data_type, (void*)na);
 }

data/ext/cumo/narray/gen/tmpl/allocate.c

@@ -29,7 +29,6 @@ static VALUE
             ptr = cumo_cuda_runtime_malloc(sizeof(dtype) * na->size);
             <% end %>
             CUMO_NA_DATA_PTR(na) = ptr;
-            CUMO_NA_DATA_OWNED(na) = TRUE;
             <% unless is_object %>
             // Device memory never passes through ruby_xmalloc, so the GC sees a
             // few-byte object holding an arbitrarily large buffer and does not
@@ -48,7 +47,7 @@ static VALUE
                 // Callers that read on the host synchronize before they ask for
                 // the pointer, which is before this fill was even launched.
                 CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("allocate", "<%=type_name%>");
-                cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+                cumo_cuda_runtime_device_synchronize();
             }
         }
         <% end %>

data/ext/cumo/narray/gen/tmpl/aref_cpu.c

@@ -44,7 +44,7 @@ static VALUE
     } else {
         ptr = cumo_na_get_pointer_for_read(self) + pos;
         CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>");
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
         return m_extract(ptr);
     }
 }

data/ext/cumo/narray/gen/tmpl/batch_norm.c

@@ -1,17 +1,5 @@
 #ifdef CUDNN_FOUND
 
-<%
-  cudnn_dtype =
-    case type_name
-    when 'sfloat'
-      'CUDNN_DATA_FLOAT'
-    when 'dfloat'
-      'CUDNN_DATA_DOUBLE'
-    else
-      # CUDNN_DATA_HALF
-      raise 'not supported'
-    end
-%>
 
 // y = x.batch_norm(gamma, beta, running_mean:, running_var:, eps:, decay:, axis:, mean:, inv_std:)
 static VALUE
@@ -20,8 +8,8 @@ static VALUE
     cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
     cudnnStatus_t status = 0;
     cudnnHandle_t handle = 0;
-    dtype coef_one = 1;
-    dtype coef_zero = 0;
+    <%=cudnn_scalar_t%> coef_one = 1;
+    <%=cudnn_scalar_t%> coef_zero = 0;
 
     VALUE x=self, gamma, beta, running_mean, running_var, eps, decay, axis, mean, inv_std, y;
     VALUE kw_hash = Qnil;
@@ -126,12 +114,12 @@ static VALUE
     }
 
     CUMO_CHECK_NARRAY_TYPE(x, cT);
-    CUMO_CHECK_NARRAY_TYPE(gamma, cT);
-    CUMO_CHECK_NARRAY_TYPE(beta, cT);
-    if (running_mean != Qnil) CUMO_CHECK_NARRAY_TYPE(running_mean, cT);
-    if (running_var != Qnil) CUMO_CHECK_NARRAY_TYPE(running_var, cT);
-    if (mean != Qnil) CUMO_CHECK_NARRAY_TYPE(mean, cT);
-    if (inv_std != Qnil) CUMO_CHECK_NARRAY_TYPE(inv_std, cT);
+    cumo_cuda_cudnn_check_param_type(gamma, <%=cudnn_param_class%>, "gamma");
+    cumo_cuda_cudnn_check_param_type(beta, <%=cudnn_param_class%>, "beta");
+    if (running_mean != Qnil) cumo_cuda_cudnn_check_param_type(running_mean, <%=cudnn_param_class%>, "running_mean");
+    if (running_var != Qnil) cumo_cuda_cudnn_check_param_type(running_var, <%=cudnn_param_class%>, "running_var");
+    if (mean != Qnil) cumo_cuda_cudnn_check_param_type(mean, <%=cudnn_param_class%>, "mean");
+    if (inv_std != Qnil) cumo_cuda_cudnn_check_param_type(inv_std, <%=cudnn_param_class%>, "inv_std");
 
     x_cont = cumo_na_as_contiguous_array(x);
     gamma_cont = cumo_na_as_contiguous_array(gamma);
@@ -166,9 +154,9 @@ static VALUE
     mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
     status = cumo_cuda_cudnn_CreateBNTensorDescriptor(&bn_desc, x_desc, mode);
     if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
-    // The derived descriptor carries x's own type -- cuDNN widens only for
-    // half, which these templates are never generated for -- and the checks
-    // above hold every parameter to x's class, so none of them needs a cast.
+    // cuDNN derives the parameter descriptor as float for a half x and as x's
+    // own type otherwise, and the checks above hold every parameter to exactly
+    // that class, so none of them needs a cast.
 
     handle = cumo_cuda_cudnn_handle();