cumo 0.7.0 → 0.8.0

data/ext/cumo/narray/gen/spec.rb

@@ -59,7 +59,7 @@ if (is_float || is_complex) && !is_object
   def_id "gemm"
 end
 # cudnn
-if is_float && !is_complex && !is_object
+if is_float && !is_complex && !is_object && !cudnn_dtype.empty?
   def_id "conv"
   def_id "conv_transpose"
   def_id "conv_grad_w"
@@ -128,6 +128,7 @@ def_method "store" do
     store_from "SComplex", "cumo_scomplex", "m_from_scomplex"
   end
   store_from "HFloat", "cumo_half", "m_from_half"
+  store_from "BFloat", "cumo_bfloat", "m_from_bfloat"
   store_from "DFloat", "double",   "m_from_real"
   store_from "SFloat", "float",    "m_from_real"
   store_from "Int64", "int64_t",  "m_from_int64"
@@ -367,7 +368,7 @@ if (is_float || is_complex) && !is_object
 end
 
 # cudnn
-if is_float && !is_complex && !is_object
+if is_float && !is_complex && !is_object && !cudnn_dtype.empty?
   def_method "conv"
   def_method "conv_transpose" # conv_backward_data
   def_method "conv_grad_w" # conv_backward_filter
@@ -378,6 +379,12 @@ if is_float && !is_complex && !is_object
   def_method "pooling_backward"
 end
 
+# fused
+if is_float && !is_complex && !is_object
+  def_method "layer_norm"
+  def_method "softmax"
+end
+
 # rmsdev
 # prod
 
@@ -400,19 +407,7 @@ end
 def_method "poly"
 
 if is_comparable && !is_object
-  if is_float
-    qsort type_name, "dtype", "*(dtype*)", "_prnan"
-    qsort type_name, "dtype", "*(dtype*)", "_ignan"
-  else
-    qsort type_name, "dtype", "*(dtype*)"
-  end
   def_method "sort"
-  if is_float
-    qsort type_name + "_index", "dtype*", "**(dtype**)", "_prnan"
-    qsort type_name + "_index", "dtype*", "**(dtype**)", "_ignan"
-  else
-    qsort type_name + "_index", "dtype*", "**(dtype**)"
-  end
   def_method "sort_index"
   def_method "median"
 end
@@ -461,6 +456,8 @@ def_module do
     math "hypot", 2
     math "erf"
     math "erfc"
+    math "gelu"
+    math "gelu_tanh"
     math "log1p"
     math "expm1"
     math "ldexp", 2

data/ext/cumo/narray/gen/tmpl/accum_arg.c

@@ -38,7 +38,7 @@ static void
   <%=name%>. Returns an index of the <%=name[3..5]%>imum value along the axis. See also `<%=name[3..5]%>_index`.
 <% if is_float %>
   @overload <%=name%>(axis:nil, nan:false)
-  @param [TrueClass] nan  If true, apply NaN-aware algorithm (return NaN posision if exist).
+  @param [TrueClass] nan  If true, apply NaN-aware algorithm (return NaN position if exist).
 <% else %>
   @overload <%=name%>(axis:nil)
 <% end %>

data/ext/cumo/narray/gen/tmpl/accum_binary_kernel.cu

@@ -14,19 +14,19 @@
 
 // The product is never an array of its own: MapIn takes one element of each
 // operand, which is what a zip reduction hands it. The accumulator stays dtype
-// rather than widening, as the host loop it replaces did, except for half,
-// which cannot carry a sum of more than a couple of thousand terms.
+// rather than widening, as the host loop it replaces did, except for the
+// types that set an accumulator of their own.
 struct <%="cumo_#{type_name}_#{name}#{nan}_impl"%> {
-<% if is_half %>
-    __device__ float Identity(int64_t /*index*/) { return 0.0f; }
-    __device__ float MapIn(dtype x, dtype y, int64_t /*index*/) {
+<% unless acc_type.empty? %>
+    __device__ <%=acc_type%> Identity(int64_t /*index*/) { return <%=acc_zero%>; }
+    __device__ <%=acc_type%> MapIn(dtype x, dtype y, int64_t /*index*/) {
 <% if nan == '_nan' %>
-        if (!not_nan(x) || !not_nan(y)) { return 0.0f; }
+        if (!not_nan(x) || !not_nan(y)) { return <%=acc_zero%>; }
 <% end %>
-        return cumo_half2float(x) * cumo_half2float(y);
+        return <%=to_acc%>(x) * <%=to_acc%>(y);
     }
-    __device__ void Reduce(float next, float& accum) { accum = next + accum; }
-    __device__ dtype MapOut(float accum) { return cumo_float2half(accum); }
+    __device__ void Reduce(<%=acc_type%> next, <%=acc_type%>& accum) { accum = next + accum; }
+    __device__ dtype MapOut(<%=acc_type%> accum) { return <%=from_acc%>(accum); }
 <% else %>
     __device__ dtype Identity(int64_t /*index*/) { return m_zero; }
     __device__ dtype MapIn(dtype x, dtype y, int64_t /*index*/) {

data/ext/cumo/narray/gen/tmpl/accum_index.c

@@ -39,7 +39,7 @@ static void
   <%=name%>. Return an index of result.
 <% if is_float %>
   @overload <%=name%>(axis:nil, nan:false)
-  @param [TrueClass] nan  If true, apply NaN-aware algorithm (return NaN posision if exist).
+  @param [TrueClass] nan  If true, apply NaN-aware algorithm (return NaN position if exist).
 <% else %>
   @overload <%=name%>(axis:nil)
 <% end %>

data/ext/cumo/narray/gen/tmpl/batch_norm.c

@@ -148,18 +148,18 @@ static VALUE
     }
     y_ptr = cumo_na_get_offset_pointer_for_write(y);
 
+    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
+    handle = cumo_cuda_cudnn_handle();
+
     status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
     if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
 
-    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
     status = cumo_cuda_cudnn_CreateBNTensorDescriptor(&bn_desc, x_desc, mode);
     if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_ERROR;
     // cuDNN derives the parameter descriptor as float for a half x and as x's
     // own type otherwise, and the checks above hold every parameter to exactly
     // that class, so none of them needs a cast.
 
-    handle = cumo_cuda_cudnn_handle();
-
     status = cudnnBatchNormalizationForwardTraining(
             handle,
             mode,

data/ext/cumo/narray/gen/tmpl/batch_norm_backward.c

@@ -133,18 +133,18 @@ static VALUE
     }
     gbeta_ptr = cumo_na_get_offset_pointer_for_write(gbeta);
 
+    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
+    handle = cumo_cuda_cudnn_handle();
+
     status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
     if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_BACKWARD_ERROR;
 
-    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
     status = cumo_cuda_cudnn_CreateBNTensorDescriptor(&bn_desc, x_desc, mode);
     if (status != CUDNN_STATUS_SUCCESS) goto BATCH_NORM_BACKWARD_ERROR;
     // cuDNN derives the parameter descriptor as float for a half x and as x's
     // own type otherwise, and the checks above hold every parameter to exactly
     // that class, so none of them needs a cast.
 
-    handle = cumo_cuda_cudnn_handle();
-
     status = cudnnBatchNormalizationBackward(
             handle,
             mode,

data/ext/cumo/narray/gen/tmpl/complex_accum_kernel.cu

@@ -43,11 +43,21 @@ struct cumo_<%=type_name%>_moments_impl {
     }
 };
 
-struct cumo_<%=type_name%>_var_impl : cumo_<%=type_name%>_moments_impl {
+// An infinity has no deviation from a mean that is itself infinite, so a row
+// holding one has a variance of NaN wherever the infinity falls in the tree.
+// It sits below the accumulator so that the forms that cannot use it, as the
+// real layer_norm cannot, do not pay for it.
+struct cumo_<%=type_name%>_spread_impl : cumo_<%=type_name%>_moments_impl {
+    __device__ Moments MapIn(dtype in, int64_t /*index*/) {
+        return {1, in, c_isinf(in) ? (rtype)NAN : (rtype)0};
+    }
+};
+
+struct cumo_<%=type_name%>_var_impl : cumo_<%=type_name%>_spread_impl {
     __device__ rtype MapOut(Moments accum) { return accum.m2 / (accum.n - 1); }
 };
 
-struct cumo_<%=type_name%>_stddev_impl : cumo_<%=type_name%>_moments_impl {
+struct cumo_<%=type_name%>_stddev_impl : cumo_<%=type_name%>_spread_impl {
     __device__ rtype MapOut(Moments accum) { return r_sqrt(accum.m2 / (accum.n - 1)); }
 };
 
@@ -87,10 +97,10 @@ struct cumo_<%=type_name%>_prod_nan_impl {
     __device__ <%=dtype%> MapOut(<%=dtype%> accum) { return accum; }
 };
 
-struct cumo_<%=type_name%>_moments_nan_impl : cumo_<%=type_name%>_moments_impl {
-    __device__ Moments MapIn(dtype in, int64_t /*index*/) {
+struct cumo_<%=type_name%>_moments_nan_impl : cumo_<%=type_name%>_spread_impl {
+    __device__ Moments MapIn(dtype in, int64_t index) {
         if (!not_nan(in)) { return {0, m_zero, 0}; }
-        return {1, in, 0};
+        return cumo_<%=type_name%>_spread_impl::MapIn(in, index);
     }
 };
 

data/ext/cumo/narray/gen/tmpl/conv.c

@@ -1,17 +1,79 @@
 #ifdef CUDNN_FOUND
 
 
+// Everything here holds a descriptor or the workspace, and the two allocations
+// below raise on their own when the device is full.
+static VALUE
+<%=c_iter%>_run(VALUE v)
+{
+    cumo_cuda_cudnn_conv_run_t *r = (cumo_cuda_cudnn_conv_run_t*)v;
+    cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
+    <%=cudnn_scalar_t%> alpha = 1;
+    <%=cudnn_scalar_t%> beta = 0;
+    cudnnConvolutionFwdAlgoPerf_t perf_result;
+    size_t workspace_size;
+
+    r->status = cumo_cuda_cudnn_CreateTensorDescriptor(&r->held.x_desc, r->x_cont, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateTensorDescriptor(&r->held.y_desc, r->y, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateFilterDescriptor(&r->held.w_desc, r->w_cont, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateConvolutionDescriptor(&r->held.conv_desc, r->ndim, r->int_stride, r->int_pad, <%=cudnn_compute_dtype%>, <%=cudnn_math_type%>);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    // auto tune
+    r->status = cumo_cuda_cudnn_FindConvolutionForwardAlgorithm(
+            &perf_result,
+            r->handle,
+            r->held.x_desc,
+            r->x_cont,
+            r->held.w_desc,
+            r->w_cont,
+            r->held.conv_desc,
+            r->held.y_desc,
+            r->y,
+            cumo_cuda_cudnn_max_workspace_size(),
+            r->int_stride,
+            r->int_pad,
+            r->ndim,
+            cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    // The descriptor asked for a math type; use the one the search settled on.
+    r->status = cudnnSetConvolutionMathType(r->held.conv_desc, perf_result.mathType);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    // The search may look at algorithms needing up to max_workspace_size,
+    // but only the one it picked has to be paid for.
+    workspace_size = perf_result.memory;
+    if (workspace_size > 0) r->held.workspace = cumo_cuda_runtime_malloc(workspace_size);
+    r->status = cudnnConvolutionForward(
+            r->handle,
+            (void*)&alpha,
+            r->held.x_desc,
+            (void*)r->x_cont_ptr,
+            r->held.w_desc,
+            (void*)r->w_cont_ptr,
+            r->held.conv_desc,
+            perf_result.algo,
+            (void*)r->held.workspace,
+            workspace_size,
+            (void*)&beta,
+            r->held.y_desc,
+            (void*)r->y_ptr);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    r->status = cumo_cuda_cudnn_AddBias(r, cudnn_dtype, &alpha);
+    return Qnil;
+}
+
 // cover_all=true is not supported with CUDNN
 // dilation > 1 is not supported yet
 // x.conv(w, b: nil, stride: 1, pad: 0, y: nil)
 static VALUE
 <%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
 {
-    cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
-    cudnnStatus_t status = 0;
-    cudnnHandle_t handle = 0;
-    <%=cudnn_scalar_t%> alpha = 1;
-    <%=cudnn_scalar_t%> beta = 0;
+    cumo_cuda_cudnn_conv_run_t r;
 
     VALUE x=self, w, b, stride, pad, y;
     VALUE kw_hash = Qnil;
@@ -24,22 +86,10 @@ static VALUE
     size_t out_channels, batch_size;
 
     VALUE x_cont, w_cont;
-    cudnnTensorDescriptor_t x_desc = 0;
-    cudnnTensorDescriptor_t y_desc = 0;
-    cudnnTensorDescriptor_t b_desc = 0;
-    cudnnFilterDescriptor_t w_desc = 0;
-    cudnnConvolutionDescriptor_t conv_desc = 0;
-    char *x_cont_ptr, *w_cont_ptr, *y_ptr;
-
-    cudnnConvolutionFwdAlgoPerf_t perf_result;
-    cudnnConvolutionFwdAlgo_t algo;
-    size_t max_workspace_size = cumo_cuda_cudnn_max_workspace_size();
-    size_t workspace_size;
-    char* workspace = 0;
-
     int int_stride[CUMO_NA_MAX_DIMENSION];
     int int_pad[CUMO_NA_MAX_DIMENSION];
 
+    memset(&r, 0, sizeof(r));
     rb_scan_args(argc, argv, "1:", &w, &kw_hash);
     rb_get_kwargs(kw_hash, kw_table, 0, 4, opts);
     stride = cumo_option_value(opts[0], Qnil);
@@ -91,112 +141,29 @@ static VALUE
     x_cont = cumo_na_as_contiguous_array(x);
     w_cont = cumo_na_as_contiguous_array(w);
 
-    x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
-    w_cont_ptr = cumo_na_get_offset_pointer_for_read(w_cont);
-    y_ptr = cumo_na_get_offset_pointer_for_write(y);
-
-    status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-    status = cumo_cuda_cudnn_CreateTensorDescriptor(&y_desc, y, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-    status = cumo_cuda_cudnn_CreateFilterDescriptor(&w_desc, w_cont, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-    status = cumo_cuda_cudnn_CreateConvolutionDescriptor(&conv_desc, ndim, int_stride, int_pad, <%=cudnn_compute_dtype%>, <%=cudnn_math_type%>);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-
-    handle = cumo_cuda_cudnn_handle();
-
-    // auto tune
-    status = cumo_cuda_cudnn_FindConvolutionForwardAlgorithm(
-            &perf_result,
-            handle,
-            x_desc,
-            x_cont,
-            w_desc,
-            w_cont,
-            conv_desc,
-            y_desc,
-            y,
-            max_workspace_size,
-            int_stride,
-            int_pad,
-            ndim,
-            cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-    // The descriptor asked for a math type; use the one the search settled on.
-    status = cudnnSetConvolutionMathType(conv_desc, perf_result.mathType);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-
-    algo = perf_result.algo;
-    workspace_size = perf_result.memory;
-
-    // The search may look at algorithms needing up to max_workspace_size,
-    // but only the one it picked has to be paid for.
-    if (workspace_size > 0) workspace = cumo_cuda_runtime_malloc(workspace_size);
-    status = cudnnConvolutionForward(
-            handle,
-            (void*)&alpha,
-            x_desc,
-            (void*)x_cont_ptr,
-            w_desc,
-            (void*)w_cont_ptr,
-            conv_desc,
-            algo,
-            (void*)workspace,
-            workspace_size,
-            (void*)&beta,
-            y_desc,
-            (void*)y_ptr);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-
+    r.x_cont = x_cont;
+    r.w_cont = w_cont;
+    r.y = y;
+    r.x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
+    r.w_cont_ptr = cumo_na_get_offset_pointer_for_read(w_cont);
+    r.y_ptr = cumo_na_get_offset_pointer_for_write(y);
+    r.ndim = ndim;
+    r.int_stride = int_stride;
+    r.int_pad = int_pad;
+
+    // Taking the bias runs Ruby, which can raise, so it is settled before the
+    // ensure below has anything to give back.
     if (b != Qnil) {
-        size_t new_shape[CUMO_NA_MAX_DIMENSION];
-        VALUE b_cont;
-        char* b_cont_ptr;
-        cumo_narray_t *nb, *nb_cont;
-        size_t *b_shape;
-        int b_ndim;
-
-        CUMO_CHECK_NARRAY_TYPE(b, cT);
-        CumoGetNArray(b, nb);
-        new_shape[0] = 1;
-        new_shape[1] = nb->size;
-        for (size_t i = 0; i < ndim; ++i) {
-            new_shape[i + 2] = 1;
-        }
-        b_cont =  cumo_na_as_contiguous_array(b);
-        b_cont_ptr = cumo_na_get_offset_pointer_for_read(b_cont);
-        CumoGetNArray(b_cont, nb_cont);
-        b_shape = nb_cont->shape;
-        b_ndim = nb_cont->ndim;
-        // reshape b
-        nb_cont->ndim = ndim + 2;
-        nb_cont->shape = new_shape;
-        status = cumo_cuda_cudnn_CreateTensorDescriptor(&b_desc, b_cont, cudnn_dtype);
-        // restore b.shape
-        nb_cont->ndim = b_ndim;
-        nb_cont->shape = b_shape;
-        if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
-
-        status = cudnnAddTensor(
-                    handle,
-                    (void*)&alpha,
-                    b_desc,
-                    (void*)b_cont_ptr,
-                    (void*)&alpha,
-                    y_desc,
-                    (void*)y_ptr);
-        if (status != CUDNN_STATUS_SUCCESS) goto CONV_ERROR;
+        cumo_cuda_cudnn_check_input(b, cT, 1, &out_channels);
+        r.b_cont = cumo_na_as_contiguous_array(b);
+        r.b_cont_ptr = cumo_na_get_offset_pointer_for_read(r.b_cont);
     }
+    r.handle = cumo_cuda_cudnn_handle();
 
-CONV_ERROR:
-    if (x_desc) cudnnDestroyTensorDescriptor(x_desc);
-    if (y_desc) cudnnDestroyTensorDescriptor(y_desc);
-    if (b_desc) cudnnDestroyTensorDescriptor(b_desc);
-    if (w_desc) cudnnDestroyFilterDescriptor(w_desc);
-    if (conv_desc) cudnnDestroyConvolutionDescriptor(conv_desc);
-    if (workspace) cumo_cuda_runtime_free(workspace);
-    cumo_cuda_cudnn_check_status(status);
+    rb_ensure(<%=c_iter%>_run, (VALUE)&r, cumo_cuda_cudnn_release_conv_held, (VALUE)&r.held);
+    RB_GC_GUARD(r.b_cont);
+    cumo_cuda_cudnn_check_status(r.status);
+    cumo_cuda_runtime_check_status(r.held.wait_status);
 
     return y;
 }

data/ext/cumo/narray/gen/tmpl/conv_grad_w.c

@@ -11,16 +11,85 @@ cumo_cuda_cudnn_get_sizet_ary(size_t *sizet_ary, VALUE ary, size_t ndim)
     }
 }
 
-// cover_all=true is not supported with CUDNN
-// gw = x.conv_grad_w(gy, w_shape, stride: 1, pad: 0, gw: nil)
+typedef struct {
+    cumo_cuda_cudnn_conv_held_t held;
+    cudnnHandle_t handle;
+    VALUE   x_cont, gy_cont, gw;
+    char   *x_cont_ptr, *gy_cont_ptr, *gw_ptr;
+    size_t  ndim;
+    int    *int_stride, *int_pad;
+    cudnnStatus_t status;
+} <%=c_iter%>_run_t;
+
+// Everything here holds a descriptor or the workspace, and the two allocations
+// below raise on their own when the device is full.
 static VALUE
-<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+<%=c_iter%>_run(VALUE v)
 {
+    <%=c_iter%>_run_t *r = (<%=c_iter%>_run_t*)v;
     cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
-    cudnnStatus_t status = 0;
-    cudnnHandle_t handle = 0;
     <%=cudnn_scalar_t%> one = 1;
     <%=cudnn_scalar_t%> zero = 0;
+    cudnnConvolutionBwdFilterAlgoPerf_t perf_result;
+    size_t workspace_size;
+
+    r->status = cumo_cuda_cudnn_CreateTensorDescriptor(&r->held.x_desc, r->x_cont, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateTensorDescriptor(&r->held.y_desc, r->gy_cont, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateFilterDescriptor(&r->held.w_desc, r->gw, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateConvolutionDescriptor(&r->held.conv_desc, r->ndim, r->int_stride, r->int_pad, <%=cudnn_compute_dtype%>, <%=cudnn_math_type%>);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    // auto tune
+    r->status = cumo_cuda_cudnn_FindConvolutionBackwardFilterAlgorithm(
+            &perf_result,
+            r->handle,
+            r->held.x_desc,
+            r->x_cont,
+            r->held.y_desc,
+            r->gy_cont,
+            r->held.conv_desc,
+            r->held.w_desc,
+            r->gw,
+            cumo_cuda_cudnn_max_workspace_size(),
+            r->int_stride,
+            r->int_pad,
+            r->ndim,
+            cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    // The descriptor asked for a math type; use the one the search settled on.
+    r->status = cudnnSetConvolutionMathType(r->held.conv_desc, perf_result.mathType);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    // The search may look at algorithms needing up to max_workspace_size,
+    // but only the one it picked has to be paid for.
+    workspace_size = perf_result.memory;
+    if (workspace_size > 0) r->held.workspace = cumo_cuda_runtime_malloc(workspace_size);
+    r->status = cudnnConvolutionBackwardFilter(
+            r->handle,
+            (void*)&one,
+            r->held.x_desc,
+            (void*)r->x_cont_ptr,
+            r->held.y_desc,
+            (void*)r->gy_cont_ptr,
+            r->held.conv_desc,
+            perf_result.algo,
+            (void*)r->held.workspace,
+            workspace_size,
+            (void*)&zero,
+            r->held.w_desc,
+            (void*)r->gw_ptr);
+    return Qnil;
+}
+
+// cover_all=true is not supported with CUDNN
+// gw = x.conv_grad_w(gy, w_shape, stride: 1, pad: 0, gw: nil)
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+{
+    <%=c_iter%>_run_t r;
 
     VALUE x=self, gy, w_shape, stride, pad, gw;
     VALUE kw_hash = Qnil;
@@ -31,22 +100,12 @@ static VALUE
     cumo_narray_t *nx, *ngy;
 
     VALUE x_cont, gy_cont;
-    cudnnTensorDescriptor_t x_desc = 0;
-    cudnnTensorDescriptor_t gy_desc = 0;
-    cudnnConvolutionDescriptor_t conv_desc = 0;
-    cudnnFilterDescriptor_t gw_desc = 0;
-    char *x_cont_ptr, *gy_cont_ptr, *gw_ptr;
-
-    cudnnConvolutionBwdFilterAlgoPerf_t perf_result;
-    cudnnConvolutionBwdFilterAlgo_t algo;
-    size_t max_workspace_size = cumo_cuda_cudnn_max_workspace_size();
-    size_t workspace_size;
-    char* workspace = 0;
 
     size_t sizet_w_shape[CUMO_NA_MAX_DIMENSION];
     int int_stride[CUMO_NA_MAX_DIMENSION];
     int int_pad[CUMO_NA_MAX_DIMENSION];
 
+    memset(&r, 0, sizeof(r));
     rb_scan_args(argc, argv, "2:", &gy, &w_shape, &kw_hash);
     rb_get_kwargs(kw_hash, kw_table, 0, 3, opts);
     stride = cumo_option_value(opts[0], Qnil);
@@ -102,71 +161,20 @@ static VALUE
     x_cont = cumo_na_as_contiguous_array(x);
     gy_cont = cumo_na_as_contiguous_array(gy);
 
-    x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
-    gy_cont_ptr = cumo_na_get_offset_pointer_for_read(gy_cont);
-    gw_ptr = cumo_na_get_offset_pointer_for_write(gw);
-
-    status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-    status = cumo_cuda_cudnn_CreateTensorDescriptor(&gy_desc, gy_cont, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-    status = cumo_cuda_cudnn_CreateFilterDescriptor(&gw_desc, gw, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-    status = cumo_cuda_cudnn_CreateConvolutionDescriptor(&conv_desc, ndim, int_stride, int_pad, <%=cudnn_compute_dtype%>, <%=cudnn_math_type%>);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-
-    handle = cumo_cuda_cudnn_handle();
-
-    // auto tune
-    status = cumo_cuda_cudnn_FindConvolutionBackwardFilterAlgorithm(
-            &perf_result,
-            handle,
-            x_desc,
-            x_cont,
-            gy_desc,
-            gy_cont,
-            conv_desc,
-            gw_desc,
-            gw,
-            max_workspace_size,
-            int_stride,
-            int_pad,
-            ndim,
-            cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-    // The descriptor asked for a math type; use the one the search settled on.
-    status = cudnnSetConvolutionMathType(conv_desc, perf_result.mathType);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-
-    algo = perf_result.algo;
-    workspace_size = perf_result.memory;
-
-    // The search may look at algorithms needing up to max_workspace_size,
-    // but only the one it picked has to be paid for.
-    if (workspace_size > 0) workspace = cumo_cuda_runtime_malloc(workspace_size);
-    status = cudnnConvolutionBackwardFilter(
-            handle,
-            (void*)&one,
-            x_desc,
-            (void*)x_cont_ptr,
-            gy_desc,
-            (void*)gy_cont_ptr,
-            conv_desc,
-            algo,
-            (void*)workspace,
-            workspace_size,
-            (void*)&zero,
-            gw_desc,
-            (void*)gw_ptr);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_GRAD_W_ERROR;
-
-CONV_GRAD_W_ERROR:
-    if (x_desc) cudnnDestroyTensorDescriptor(x_desc);
-    if (gy_desc) cudnnDestroyTensorDescriptor(gy_desc);
-    if (gw_desc) cudnnDestroyFilterDescriptor(gw_desc);
-    if (conv_desc) cudnnDestroyConvolutionDescriptor(conv_desc);
-    if (workspace) cumo_cuda_runtime_free(workspace);
-    cumo_cuda_cudnn_check_status(status);
+    r.x_cont = x_cont;
+    r.gy_cont = gy_cont;
+    r.gw = gw;
+    r.x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
+    r.gy_cont_ptr = cumo_na_get_offset_pointer_for_read(gy_cont);
+    r.gw_ptr = cumo_na_get_offset_pointer_for_write(gw);
+    r.ndim = ndim;
+    r.int_stride = int_stride;
+    r.int_pad = int_pad;
+    r.handle = cumo_cuda_cudnn_handle();
+
+    rb_ensure(<%=c_iter%>_run, (VALUE)&r, cumo_cuda_cudnn_release_conv_held, (VALUE)&r.held);
+    cumo_cuda_cudnn_check_status(r.status);
+    cumo_cuda_runtime_check_status(r.held.wait_status);
 
     return gw;
 }