cumo 0.7.0 → 0.8.0

data/ext/cumo/narray/gen/tmpl/conv_transpose.c

@@ -26,17 +26,79 @@ get_int_out_size(int* int_out_size, VALUE out_size, size_t ndim, size_t* x_shape
     }
 }
 
+// Everything here holds a descriptor or the workspace, and the two allocations
+// below raise on their own when the device is full.
+static VALUE
+<%=c_iter%>_run(VALUE v)
+{
+    cumo_cuda_cudnn_conv_run_t *r = (cumo_cuda_cudnn_conv_run_t*)v;
+    cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
+    <%=cudnn_scalar_t%> alpha = 1;
+    <%=cudnn_scalar_t%> beta = 0;
+    cudnnConvolutionBwdDataAlgoPerf_t perf_result;
+    size_t workspace_size;
+
+    r->status = cumo_cuda_cudnn_CreateTensorDescriptor(&r->held.x_desc, r->x_cont, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateTensorDescriptor(&r->held.y_desc, r->y, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateFilterDescriptor(&r->held.w_desc, r->w_cont, cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    r->status = cumo_cuda_cudnn_CreateConvolutionDescriptor(&r->held.conv_desc, r->ndim, r->int_stride, r->int_pad, <%=cudnn_compute_dtype%>, <%=cudnn_math_type%>);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    // auto tune
+    r->status = cumo_cuda_cudnn_FindConvolutionBackwardDataAlgorithm(
+            &perf_result,
+            r->handle,
+            r->held.w_desc,
+            r->w_cont,
+            r->held.x_desc,
+            r->x_cont,
+            r->held.conv_desc,
+            r->held.y_desc,
+            r->y,
+            cumo_cuda_cudnn_max_workspace_size(),
+            r->int_stride,
+            r->int_pad,
+            r->ndim,
+            cudnn_dtype);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+    // The descriptor asked for a math type; use the one the search settled on.
+    r->status = cudnnSetConvolutionMathType(r->held.conv_desc, perf_result.mathType);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    // The search may look at algorithms needing up to max_workspace_size,
+    // but only the one it picked has to be paid for.
+    workspace_size = perf_result.memory;
+    if (workspace_size > 0) r->held.workspace = cumo_cuda_runtime_malloc(workspace_size);
+    r->status = cudnnConvolutionBackwardData(
+            r->handle,
+            (void*)&alpha,
+            r->held.w_desc,
+            (void*)r->w_cont_ptr,
+            r->held.x_desc,
+            (void*)r->x_cont_ptr,
+            r->held.conv_desc,
+            perf_result.algo,
+            (void*)r->held.workspace,
+            workspace_size,
+            (void*)&beta,
+            r->held.y_desc,
+            (void*)r->y_ptr);
+    if (r->status != CUDNN_STATUS_SUCCESS) return Qnil;
+
+    r->status = cumo_cuda_cudnn_AddBias(r, cudnn_dtype, &alpha);
+    return Qnil;
+}
+
 // cover_all=true is not supported with CUDNN
 // dilation > 1 is not supported yet
 // x.conv(w, b: nil, stride: 1, pad: 0, out_size: nil, y: nil)
 static VALUE
 <%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
 {
-    cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
-    cudnnStatus_t status = 0;
-    cudnnHandle_t handle = 0;
-    <%=cudnn_scalar_t%> alpha = 1;
-    <%=cudnn_scalar_t%> beta = 0;
+    cumo_cuda_cudnn_conv_run_t r;
 
     VALUE x=self, w, b, stride, pad, out_size, y;
     VALUE kw_hash = Qnil;
@@ -49,23 +111,11 @@ static VALUE
     size_t out_channels, batch_size;
 
     VALUE x_cont, w_cont;
-    cudnnTensorDescriptor_t x_desc = 0;
-    cudnnTensorDescriptor_t y_desc = 0;
-    cudnnTensorDescriptor_t b_desc = 0;
-    cudnnFilterDescriptor_t w_desc = 0;
-    cudnnConvolutionDescriptor_t conv_desc = 0;
-    char *x_cont_ptr, *w_cont_ptr, *y_ptr;
-
-    cudnnConvolutionBwdDataAlgoPerf_t perf_result;
-    cudnnConvolutionBwdDataAlgo_t algo;
-    size_t max_workspace_size = cumo_cuda_cudnn_max_workspace_size();
-    size_t workspace_size;
-    char* workspace = 0;
-
     int int_stride[CUMO_NA_MAX_DIMENSION];
     int int_pad[CUMO_NA_MAX_DIMENSION];
     int int_out_size[CUMO_NA_MAX_DIMENSION];
 
+    memset(&r, 0, sizeof(r));
     rb_scan_args(argc, argv, "1:", &w, &kw_hash);
     rb_get_kwargs(kw_hash, kw_table, 0, 5, opts);
     b = cumo_option_value(opts[0], Qnil);
@@ -118,112 +168,29 @@ static VALUE
     x_cont = cumo_na_as_contiguous_array(x);
     w_cont = cumo_na_as_contiguous_array(w);
 
-    x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
-    w_cont_ptr = cumo_na_get_offset_pointer_for_read(w_cont);
-    y_ptr = cumo_na_get_offset_pointer_for_write(y);
-
-    status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-    status = cumo_cuda_cudnn_CreateTensorDescriptor(&y_desc, y, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-    status = cumo_cuda_cudnn_CreateFilterDescriptor(&w_desc, w_cont, cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-    status = cumo_cuda_cudnn_CreateConvolutionDescriptor(&conv_desc, ndim, int_stride, int_pad, <%=cudnn_compute_dtype%>, <%=cudnn_math_type%>);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-
-    handle = cumo_cuda_cudnn_handle();
-
-    // auto tune
-    status = cumo_cuda_cudnn_FindConvolutionBackwardDataAlgorithm(
-            &perf_result,
-            handle,
-            w_desc,
-            w_cont,
-            x_desc,
-            x_cont,
-            conv_desc,
-            y_desc,
-            y,
-            max_workspace_size,
-            int_stride,
-            int_pad,
-            ndim,
-            cudnn_dtype);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-    // The descriptor asked for a math type; use the one the search settled on.
-    status = cudnnSetConvolutionMathType(conv_desc, perf_result.mathType);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-
-    algo = perf_result.algo;
-    workspace_size = perf_result.memory;
-
-    // The search may look at algorithms needing up to max_workspace_size,
-    // but only the one it picked has to be paid for.
-    if (workspace_size > 0) workspace = cumo_cuda_runtime_malloc(workspace_size);
-    status = cudnnConvolutionBackwardData(
-            handle,
-            (void*)&alpha,
-            w_desc,
-            (void*)w_cont_ptr,
-            x_desc,
-            (void*)x_cont_ptr,
-            conv_desc,
-            algo,
-            (void*)workspace,
-            workspace_size,
-            (void*)&beta,
-            y_desc,
-            (void*)y_ptr);
-    if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-
+    r.x_cont = x_cont;
+    r.w_cont = w_cont;
+    r.y = y;
+    r.x_cont_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
+    r.w_cont_ptr = cumo_na_get_offset_pointer_for_read(w_cont);
+    r.y_ptr = cumo_na_get_offset_pointer_for_write(y);
+    r.ndim = ndim;
+    r.int_stride = int_stride;
+    r.int_pad = int_pad;
+
+    // Taking the bias runs Ruby, which can raise, so it is settled before the
+    // ensure below has anything to give back.
     if (b != Qnil) {
-        size_t new_shape[CUMO_NA_MAX_DIMENSION];
-        VALUE b_cont;
-        char* b_cont_ptr;
-        cumo_narray_t *nb, *nb_cont;
-        size_t *b_shape;
-        int b_ndim;
-
-        CUMO_CHECK_NARRAY_TYPE(b, cT);
-        CumoGetNArray(b, nb);
-        new_shape[0] = 1;
-        new_shape[1] = nb->size;
-        for (size_t i = 0; i < ndim; ++i) {
-            new_shape[i + 2] = 1;
-        }
-        b_cont =  cumo_na_as_contiguous_array(b);
-        b_cont_ptr = cumo_na_get_offset_pointer_for_read(b_cont);
-        CumoGetNArray(b_cont, nb_cont);
-        b_shape = nb_cont->shape;
-        b_ndim = nb_cont->ndim;
-        // reshape b
-        nb_cont->ndim = ndim + 2;
-        nb_cont->shape = new_shape;
-        status = cumo_cuda_cudnn_CreateTensorDescriptor(&b_desc, b_cont, cudnn_dtype);
-        // restore b.shape
-        nb_cont->ndim = b_ndim;
-        nb_cont->shape = b_shape;
-        if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
-
-        status = cudnnAddTensor(
-                    handle,
-                    (void*)&alpha,
-                    b_desc,
-                    (void*)b_cont_ptr,
-                    (void*)&alpha,
-                    y_desc,
-                    (void*)y_ptr);
-        if (status != CUDNN_STATUS_SUCCESS) goto CONV_TRANSPOSE_ERROR;
+        cumo_cuda_cudnn_check_input(b, cT, 1, &out_channels);
+        r.b_cont = cumo_na_as_contiguous_array(b);
+        r.b_cont_ptr = cumo_na_get_offset_pointer_for_read(r.b_cont);
     }
+    r.handle = cumo_cuda_cudnn_handle();
 
-CONV_TRANSPOSE_ERROR:
-    if (x_desc) cudnnDestroyTensorDescriptor(x_desc);
-    if (y_desc) cudnnDestroyTensorDescriptor(y_desc);
-    if (b_desc) cudnnDestroyTensorDescriptor(b_desc);
-    if (w_desc) cudnnDestroyFilterDescriptor(w_desc);
-    if (conv_desc) cudnnDestroyConvolutionDescriptor(conv_desc);
-    if (workspace) cumo_cuda_runtime_free(workspace);
-    cumo_cuda_cudnn_check_status(status);
+    rb_ensure(<%=c_iter%>_run, (VALUE)&r, cumo_cuda_cudnn_release_conv_held, (VALUE)&r.held);
+    RB_GC_GUARD(r.b_cont);
+    cumo_cuda_cudnn_check_status(r.status);
+    cumo_cuda_runtime_check_status(r.held.wait_status);
 
     return y;
 }

data/ext/cumo/narray/gen/tmpl/cum.c

@@ -28,24 +28,25 @@ static void
     CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%><%=j%>", "<%=type_name%>");
     cumo_cuda_runtime_device_synchronize();
 
-<% if is_half && name == 'cumsum' %>
-    // The scan this stands in for carries float, and a running sum of halves
-    // stops moving once it passes 2048, so the accumulator is float here too.
+<% if !acc_type.empty? && name == 'cumsum' %>
+    // The scan this stands in for carries the accumulator type, and a running
+    // sum in the element type stops moving once the partial outgrows it, so this
+    // path carries it too.
     {
-        float acc, fy;
+        <%=acc_type%> acc, fy;
 
         CUMO_GET_DATA_STRIDE(p1,s1,dtype,x);
-        acc = cumo_half2float(x);
+        acc = <%=to_acc%>(x);
         CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
         for (i--; i--;) {
             CUMO_GET_DATA_STRIDE(p1,s1,dtype,y);
-            fy = cumo_half2float(y);
+            fy = <%=to_acc%>(y);
   <% if j == '_nan' %>
             if (acc != acc) { acc = fy; } else if (fy == fy) { acc += fy; }
   <% else %>
             acc += fy;
   <% end %>
-            x = cumo_float2half(acc);
+            x = <%=from_acc%>(acc);
             CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
         }
     }

data/ext/cumo/narray/gen/tmpl/cum_kernel.cu

@@ -1,18 +1,19 @@
 <% unless type_name == 'robject' %>
-<% widen = is_half && name == 'cumsum' %>
-<% cum_t = widen ? 'float' : 'dtype' %>
+<% widen = !acc_type.empty? && name == 'cumsum' %>
+<% cum_t = widen ? acc_type : 'dtype' %>
 <% if widen %>
 
-// A running sum of halves stops moving once it passes 2048, so the scan carries
-// float and each element is rounded as it is written.
+// A running sum in the element type stops moving once the partial outgrows it,
+// so the scan carries the accumulator type and each element is rounded as it is
+// written.
 struct <%="cumo_thrust_#{name}_widen"%>
 {
-    __host__ __device__ float operator()(dtype x) const { return cumo_half2float(x); }
+    __host__ __device__ <%=acc_type%> operator()(dtype x) const { return <%=to_acc%>(x); }
 };
 
 struct <%="cumo_thrust_#{name}_narrow"%>
 {
-    __host__ __device__ dtype operator()(float x) const { return cumo_float2half(x); }
+    __host__ __device__ dtype operator()(<%=acc_type%> x) const { return <%=from_acc%>(x); }
 };
 <% end %>
 <% (is_float ? ["","_nan"] : [""]).each do |j| %>

data/ext/cumo/narray/gen/tmpl/fixed_batch_norm.c

@@ -97,18 +97,18 @@ static VALUE
     }
     y_ptr = cumo_na_get_offset_pointer_for_write(y);
 
+    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
+    handle = cumo_cuda_cudnn_handle();
+
     status = cumo_cuda_cudnn_CreateTensorDescriptor(&x_desc, x_cont, cudnn_dtype);
     if (status != CUDNN_STATUS_SUCCESS) goto FIXED_BATCH_NORM_ERROR;
 
-    mode = cumo_cuda_cudnn_GetBatchNormMode(axis_ndim, int_axis);
     status = cumo_cuda_cudnn_CreateBNTensorDescriptor(&bn_desc, x_desc, mode);
     if (status != CUDNN_STATUS_SUCCESS) goto FIXED_BATCH_NORM_ERROR;
     // cuDNN derives the parameter descriptor as float for a half x and as x's
     // own type otherwise, and the checks above hold every parameter to exactly
     // that class, so none of them needs a cast.
 
-    handle = cumo_cuda_cudnn_handle();
-
     status = cudnnBatchNormalizationForwardInference(
             handle,
             mode,

data/ext/cumo/narray/gen/tmpl/float_accum_kernel.cu

@@ -1,7 +1,5 @@
 <%= load_erb('real_accum').result(binding) %>
-<% acc = is_half ? 'float' : 'dtype' %>
-<% to_acc = is_half ? 'cumo_half2float' : '' %>
-<% from_acc = is_half ? 'cumo_float2half' : '' %>
+<% acc = acc_type.empty? ? 'dtype' : acc_type %>
 
 
 #if defined(__cplusplus)
@@ -27,19 +25,33 @@ struct cumo_<%=type_name%>_moments_impl {
     __device__ void Reduce(Moments next, Moments& accum) {
         if (next.n == 0) { return; }
         if (accum.n == 0) { accum = next; return; }
+        // The two quotients share a denominator, and one reciprocal is much
+        // cheaper than two double divides on a GeForce card.
         <%=acc%> n = accum.n + next.n;
+        <%=acc%> r = <%=acc%>(1) / n;
         <%=acc%> delta = next.mean - accum.mean;
-        accum.mean = accum.mean + delta * (next.n / n);
-        accum.m2 += next.m2 + delta * delta * accum.n * next.n / n;
+        accum.mean = accum.mean + delta * (next.n * r);
+        accum.m2 += next.m2 + delta * delta * accum.n * next.n * r;
         accum.n = n;
     }
 };
 
-struct cumo_<%=type_name%>_var_impl : cumo_<%=type_name%>_moments_impl {
+// An infinity has no deviation from a mean that is itself infinite, so a row
+// holding one has a variance of NaN wherever the infinity falls in the tree.
+// layer_norm shares the accumulator above and cannot use this, so it sits here
+// rather than there.
+struct cumo_<%=type_name%>_spread_impl : cumo_<%=type_name%>_moments_impl {
+    __device__ Moments MapIn(dtype in, int64_t /*index*/) {
+        <%=acc%> x = <%=to_acc%>(in);
+        return {1, x, isinf(x) ? <%=acc%>(NAN) : <%=acc%>(0)};
+    }
+};
+
+struct cumo_<%=type_name%>_var_impl : cumo_<%=type_name%>_spread_impl {
     __device__ rtype MapOut(Moments accum) { return <%=from_acc%>(accum.m2 / (accum.n - 1)); }
 };
 
-struct cumo_<%=type_name%>_stddev_impl : cumo_<%=type_name%>_moments_impl {
+struct cumo_<%=type_name%>_stddev_impl : cumo_<%=type_name%>_spread_impl {
     __device__ rtype MapOut(Moments accum) { return <%=from_acc%>(sqrt(accum.m2 / (accum.n - 1))); }
 };
 
@@ -65,10 +77,10 @@ struct cumo_<%=type_name%>_rms_impl {
 // Reduce above already returns early on a zero count, so the moments never see
 // it, and mean and rms carry the count of what was left rather than the length
 // of the reduce axis.
-struct cumo_<%=type_name%>_moments_nan_impl : cumo_<%=type_name%>_moments_impl {
-    __device__ Moments MapIn(dtype in, int64_t /*index*/) {
+struct cumo_<%=type_name%>_moments_nan_impl : cumo_<%=type_name%>_spread_impl {
+    __device__ Moments MapIn(dtype in, int64_t index) {
         if (!not_nan(in)) { return {0, 0, 0}; }
-        return {1, <%=to_acc%>(in), 0};
+        return cumo_<%=type_name%>_spread_impl::MapIn(in, index);
     }
 };
 

data/ext/cumo/narray/gen/tmpl/gemm.c

@@ -1,34 +1,10 @@
 <%
-  func_prefix =
-    case type_name
-    when 'sfloat'
-      'S'
-    when 'dfloat'
-      'D'
-    when 'scomplex'
-      'C'
-    when 'dcomplex'
-      'Z'
-    end
-  cutype =
-    case type_name
-    when 'hfloat'
-      'cumo_half'
-    when 'sfloat'
-      'float'
-    when 'dfloat'
-      'double'
-    when 'scomplex'
-      'cuComplex'
-    when 'dcomplex'
-      'cuDoubleComplex'
-    end
-  # cublasGemmStridedBatchedEx multiplies in half and accumulates in float, and
-  # takes its scalars in the accumulator's type.
-  scalar_t = is_half ? 'float' : 'dtype'
-  num_to_scalar = is_half ? '(float)NUM2DBL' : 'm_num_to_data'
-  scalar_one = is_half ? '1.0f' : 'm_one'
-  scalar_zero = is_half ? '0.0f' : 'm_zero'
+  # cublasGemmStridedBatchedEx multiplies in the element type and accumulates
+  # in the wider one, and takes its scalars in the accumulator's type.
+  scalar_t = acc_type.empty? ? 'dtype' : acc_type
+  num_to_scalar = acc_type.empty? ? 'm_num_to_data' : "(#{acc_type})NUM2DBL"
+  scalar_one = acc_one
+  scalar_zero = acc_zero
 %>
 
 #define ROW_SIZE(na) ((na)->shape[(na)->ndim-2])
@@ -298,7 +274,7 @@ static void
 
     if (cumo_na_debug_flag) print_gemm_args(g, &a_layout, &b_layout, stridec, batch_count);
     handle = cumo_cuda_cublas_handle();
-<% if is_half %>
+<% unless cublas_dtype.empty? %>
     status = cublasGemmStridedBatchedEx(
             handle,
             b_layout.trans,
@@ -308,23 +284,23 @@ static void
             g->k,
             &g->alpha,
             (const void*)(cumo_na_get_pointer_for_read(b_layout.a) + cumo_na_get_offset(b_layout.a)),
-            CUDA_R_16F,
+            <%=cublas_dtype%>,
             b_layout.ld,
             b_layout.stride,
             (const void*)(cumo_na_get_pointer_for_read(a_layout.a) + cumo_na_get_offset(a_layout.a)),
-            CUDA_R_16F,
+            <%=cublas_dtype%>,
             a_layout.ld,
             a_layout.stride,
             &g->beta,
             (void*)(cumo_na_get_pointer_for_write(c) + cumo_na_get_offset(c)),
-            CUDA_R_16F,
+            <%=cublas_dtype%>,
             g->n,
             stridec,
             batch_count,
             CUBLAS_COMPUTE_32F,
             CUBLAS_GEMM_DEFAULT);
 <% else %>
-    status = cublas<%=func_prefix%>gemmStridedBatched(
+    status = cublas<%=cublas_prefix%>gemmStridedBatched(
             handle,
             b_layout.trans,
             a_layout.trans,

data/ext/cumo/narray/gen/tmpl/layer_norm.c

@@ -0,0 +1,91 @@
+void <%="cumo_#{c_iter}_kernel_launch"%>(char *px, char *pg, char *pb, char *py, uint64_t rows, uint64_t cols, double eps);
+
+/*
+  Normalizes self along its last axis, then scales by gamma and shifts by beta.
+  Each row is taken on its own: y = (x - mean) / sqrt(var + eps) * gamma + beta,
+  where mean and var are of that row and var divides by the row length.
+  @overload layer_norm(gamma, beta, eps:1e-5)
+  @param [Cumo::<%=class_name%>] gamma  scale, one-dimensional and as long as the last axis.
+  @param [Cumo::<%=class_name%>] beta  shift, one-dimensional and as long as the last axis.
+  @param [Float] eps  added to the variance before the square root.
+  Answers a new array: inplace! is not honoured.
+  @return [Cumo::<%=class_name%>] returns the normalized array, shaped like self.
+*/
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE argv[], VALUE self)
+{
+    VALUE x=self, gamma, beta, eps, y;
+    VALUE kw_hash = Qnil;
+    ID kw_table[] = {rb_intern("eps")};
+    VALUE opts[] = {Qundef};
+    VALUE x_cont, gamma_cont, beta_cont;
+    cumo_narray_t *nx, *ngamma, *nbeta, *ny;
+    char *x_ptr, *gamma_ptr, *beta_ptr, *y_ptr;
+    size_t cols, rows;
+    double double_eps = 1e-5;
+
+    rb_scan_args(argc, argv, "2:", &gamma, &beta, &kw_hash);
+    rb_get_kwargs(kw_hash, kw_table, 0, 1, opts);
+    eps = cumo_option_value(opts[0], Qnil);
+    if (eps != Qnil) {
+        double_eps = NUM2DBL(eps);
+    }
+
+    CUMO_CHECK_NARRAY_TYPE(x, cT);
+    CUMO_CHECK_NARRAY_TYPE(gamma, cT);
+    CUMO_CHECK_NARRAY_TYPE(beta, cT);
+
+    x_cont = cumo_na_as_contiguous_array(x);
+    gamma_cont = cumo_na_as_contiguous_array(gamma);
+    beta_cont = cumo_na_as_contiguous_array(beta);
+
+    CumoGetNArray(x_cont, nx);
+    if (nx->ndim < 1) {
+        rb_raise(cumo_na_eShapeError, "layer_norm needs an axis to normalize along");
+    }
+    y = cumo_na_new(cT, nx->ndim, nx->shape);
+
+    // Taking a pointer runs allocate, which is Ruby, so all four are taken
+    // before anything is measured and the checks below run over what is left.
+    // See row_method.h for why the pointers themselves have to be looked at
+    // again.
+    x_ptr = cumo_na_get_offset_pointer_for_read(x_cont);
+    gamma_ptr = cumo_na_get_offset_pointer_for_read(gamma_cont);
+    beta_ptr = cumo_na_get_offset_pointer_for_read(beta_cont);
+    y_ptr = cumo_na_get_offset_pointer_for_write(y);
+
+    // cumo_na_as_contiguous_array answers what dup gave it, which a class is
+    // free to define, so what these carry is not what was checked above.
+    CUMO_CHECK_NARRAY_TYPE(x_cont, cT);
+    CUMO_CHECK_NARRAY_TYPE(gamma_cont, cT);
+    CUMO_CHECK_NARRAY_TYPE(beta_cont, cT);
+    CumoGetNArray(x_cont, nx);
+    CumoGetNArray(gamma_cont, ngamma);
+    CumoGetNArray(beta_cont, nbeta);
+    CumoGetNArray(y, ny);
+
+    if (nx->ndim < 1) {
+        rb_raise(cumo_na_eShapeError, "layer_norm needs an axis to normalize along");
+    }
+    cols = nx->shape[nx->ndim - 1];
+    if (ngamma->ndim != 1 || ngamma->shape[0] != cols) {
+        rb_raise(cumo_na_eShapeError, "gamma must be 1-dimensional and %"SZF"u long", cols);
+    }
+    if (nbeta->ndim != 1 || nbeta->shape[0] != cols) {
+        rb_raise(cumo_na_eShapeError, "beta must be 1-dimensional and %"SZF"u long", cols);
+    }
+    CUMO_ROW_CHECK_SAME_SHAPE(ny, nx);
+    CUMO_ROW_CHECK_READ_BUFFER(x_cont, x_ptr, "self");
+    CUMO_ROW_CHECK_READ_BUFFER(gamma_cont, gamma_ptr, "gamma");
+    CUMO_ROW_CHECK_READ_BUFFER(beta_cont, beta_ptr, "beta");
+    CUMO_ROW_CHECK_WRITE_BUFFER(y, y_ptr, "the result");
+
+    if (nx->size == 0) {
+        return y;
+    }
+    rows = nx->size / cols;
+
+    <%="cumo_#{c_iter}_kernel_launch"%>(x_ptr, gamma_ptr, beta_ptr, y_ptr, (uint64_t)rows, (uint64_t)cols, double_eps);
+
+    return y;
+}

data/ext/cumo/narray/gen/tmpl/layer_norm_kernel.cu

@@ -0,0 +1,54 @@
+<%
+  acc = acc_type.empty? ? 'dtype' : acc_type
+%>
+
+#if defined(__cplusplus)
+#if 0
+{ /* satisfy cc-mode */
+#endif
+}  /* extern "C" { */
+#endif
+
+struct <%="cumo_#{c_iter}_stats"%> {
+    <%=acc%> mean;
+    <%=acc%> rstd;
+};
+
+// Layer norm divides the sum of squared deviations by n, where cumo's var
+// divides by n - 1, so the two cannot share a MapOut however much of the
+// accumulation they share.
+struct <%="cumo_#{c_iter}_impl"%> : cumo_<%=type_name%>_moments_impl {
+    <%=acc%> eps;
+    __device__ <%="cumo_#{c_iter}_stats"%> MapOut(Moments accum) {
+        return {accum.mean, <%=acc%>(1) / sqrt(accum.m2 / accum.n + eps)};
+    }
+};
+
+struct <%="cumo_#{c_iter}_apply"%> {
+    const dtype* g;
+    const dtype* b;
+    __device__ dtype operator()(dtype x, uint64_t col, <%="cumo_#{c_iter}_stats"%> st) const {
+        <%=acc%> v = (<%=to_acc%>(x) - st.mean) * st.rstd;
+        return <%=from_acc%>(v * <%=to_acc%>(g[col]) + <%=to_acc%>(b[col]));
+    }
+};
+
+#if defined(__cplusplus)
+extern "C" {
+#if 0
+} /* satisfy cc-mode */
+#endif
+#endif
+
+void <%="cumo_#{c_iter}_kernel_launch"%>(
+        char *px, char *pg, char *pb, char *py,
+        uint64_t rows, uint64_t cols, double eps)
+{
+    <%="cumo_#{c_iter}_impl"%> impl;
+    <%="cumo_#{c_iter}_apply"%> apply;
+
+    impl.eps = (<%=acc%>)eps;
+    apply.g = (const dtype*)pg;
+    apply.b = (const dtype*)pb;
+    cumo_row_reduce_apply<dtype>(px, py, rows, cols, impl, apply);
+}

data/ext/cumo/narray/gen/tmpl/lib.c

@@ -15,6 +15,7 @@
 #include "SFMT.h"
 #include "cumo/cuda/memory_pool.h"
 #include "cumo/cuda/runtime.h"
+#include "cumo/row_method.h"
 <% unless type_name == 'robject' %>
 #include "cumo/indexer.h"
 <% end %>

data/ext/cumo/narray/gen/tmpl/lib_kernel.cu

@@ -2,6 +2,7 @@
 <% unless type_name == 'robject' %>
 #include "cumo/indexer.h"
 #include "cumo/reduce_kernel.h"
+#include "cumo/row_kernel.h"
 <% end %>
 #include <<%="cumo/types/#{type_name}_kernel.h"%>>
 

data/ext/cumo/narray/gen/tmpl/median.c

@@ -28,7 +28,7 @@ static void
   <%=name%> of self.
 <% if is_float %>
   @overload <%=name%>(axis:nil, keepdims:false, nan:false)
-  @param [TrueClass] nan (keyword) If true, propagete NaN. If false, ignore NaN.
+  @param [TrueClass] nan (keyword) If true, propagate NaN. If false, ignore NaN.
 <% else %>
   @overload <%=name%>(axis:nil, keepdims:false)
 <% end %>

data/ext/cumo/narray/gen/tmpl/poly.c

@@ -1,8 +1,6 @@
 <% unless type_name == 'robject' %>
 void <%="cumo_#{c_iter}_kernel_launch"%>(cumo_na_iarray_t* x, cumo_na_iarray_t* coef, int ncoef, cumo_na_iarray_t* z, cumo_na_indexer_t* indexer);
 
-// Horner over every element in one launch. The host loop this replaces ran
-// once per element, each time behind a cudaDeviceSynchronize.
 static void
 <%=c_iter%>_kernel(cumo_na_loop_t *const lp)
 {
@@ -27,7 +25,6 @@ static void
     dtype  x, y, a;
 
     CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
-    cumo_cuda_runtime_device_synchronize();
     n = lp->narg - 2;
     x = *(dtype*)CUMO_NDL_PTR(lp,0);
     y = *(dtype*)CUMO_NDL_PTR(lp,n);

data/ext/cumo/narray/gen/tmpl/poly_kernel.cu

@@ -40,7 +40,7 @@ void <%="cumo_#{c_iter}_kernel_launch"%>(cumo_na_iarray_t* x, cumo_na_iarray_t*
         <%="cumo_#{c_iter}_kernel_dim"%><<<grid_dim, block_dim>>>(*x, d_coef, ncoef, *z, *indexer);
         break;
     }
-    cumo_cuda_runtime_check_kernel_launch();
+    cumo_check_launch_holding(d_coef);
     cumo_cuda_runtime_free((char*)d_coef);
 }
 <% end %>