cumo 0.6.0 → 0.7.0

data/ext/cumo/narray/gen/tmpl/pooling_forward.c

@@ -1,17 +1,5 @@
 #ifdef CUDNN_FOUND
 
-<%
-  cudnn_dtype =
-    case type_name
-    when 'sfloat'
-      'CUDNN_DATA_FLOAT'
-    when 'dfloat'
-      'CUDNN_DATA_DOUBLE'
-    else
-      # CUDNN_DATA_HALF
-      raise 'not supported'
-    end
-%>
 
 // cover_all=true is not supported with CUDNN
 // x.pooling_forward(mode, kernel_size, stride: 1, pad: 0, y: nil)
@@ -25,8 +13,8 @@ static VALUE
     cudnnDataType_t cudnn_dtype = <%= cudnn_dtype %>;
     cudnnStatus_t status = 0;
     cudnnHandle_t handle = 0;
-    dtype alpha = 1;
-    dtype beta = 0;
+    <%=cudnn_scalar_t%> alpha = 1;
+    <%=cudnn_scalar_t%> beta = 0;
 
     VALUE x=self, mode, kernel_size, stride, pad, y;
     VALUE kw_hash = Qnil;

data/ext/cumo/narray/gen/tmpl/qsort.c

@@ -126,7 +126,6 @@ static void
         r,
         swaptype,
         presorted;
-    cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
 
  loop:SWAPINIT(a, es);
     if (n < 7)

data/ext/cumo/narray/gen/tmpl/rand.c

@@ -58,6 +58,11 @@ inline static dtype m_rand(uint32_t max, int shift)
 }
 <% end %>
 <%
+elsif is_half
+  m_rand = "m_rand(max)"
+  shift_def = ""
+  shift_set = ""
+  rand_type = "float"
 else
   m_rand = "m_rand(max)"
   shift_def = ""
@@ -102,7 +107,7 @@ static void
         dtype x;
 
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
         if (idx1) {
             for (; i--;) {
                 x = m_add(<%=m_rand%>,low);
@@ -157,7 +162,13 @@ static VALUE
 {
     rand_opt_t g;
     VALUE v1=Qnil, v2=Qnil;
+    <% if is_half %>
+    // The span is read in float: half cannot hold one wider than 65504, and
+    // subtracting the bounds in half would answer infinity for it.
+    float high;
+    <% else %>
     dtype high;
+    <% end %>
     cumo_ndfunc_arg_in_t ain[1] = {{CUMO_OVERWRITE,0}};
     cumo_ndfunc_t ndf = {<%=c_iter%>, CUMO_FULL_LOOP, 1,0, ain,0};
 
@@ -173,17 +184,24 @@ static VALUE
         if (v1==Qnil) {
             <% if is_complex %>
             g.max = high = c_new(1,1);
+            <% elsif is_half %>
+            g.max = high = 1.0f;
             <% else %>
             g.max = high = m_one;
             <% end %>
         } else {
-            g.max = high = m_num_to_data(v1);
+            g.max = high = <% if is_half %>(float)NUM2DBL(v1)<% else %>m_num_to_data(v1)<% end %>;
         }
     <% end %>
     } else {
         g.low = m_num_to_data(v1);
+        <% if is_half %>
+        high = (float)NUM2DBL(v2);
+        g.max = high - cumo_half2float(g.low);
+        <% else %>
         high = m_num_to_data(v2);
         g.max = m_sub(high,g.low);
+        <% end %>
     }
     <% if is_int && !is_object %>
     if (high <= g.low) {

data/ext/cumo/narray/gen/tmpl/rand_kernel.cu

@@ -2,6 +2,8 @@
 if is_int && !is_object
   rand_bit = (/Int64$/ =~ class_name) ? 64 : 32
   rand_type = "uint#{rand_bit}_t"
+elsif is_half
+  rand_type = "float"
 else
   rand_type = "dtype"
 end
@@ -45,6 +47,17 @@ __device__ static dtype
     CUMO_REAL(z) = cumo_rand_uniform(st) * CUMO_REAL(max) + CUMO_REAL(low);
     CUMO_IMAG(z) = cumo_rand_uniform(st) * CUMO_IMAG(max) + CUMO_IMAG(low);
     return z;
+    //<% elsif is_half %>
+    // Rounding the float into a half can land on the upper bound, which rand
+    // does not include, and the neighbour below it is the nearest value that
+    // does. A bound half cannot hold has no neighbour to step to.
+    float flow = cumo_half2float(low);
+    float high = flow + max;
+    dtype h = cumo_float2half(cumo_rand_uniform(st) * max + flow);
+    if (isfinite(high) && cumo_half2float(h) >= high) {
+        h = cumo_half_step_down(h);
+    }
+    return h;
     //<% else %>
     return (dtype)(cumo_rand_uniform(st) * max) + low;
     //<% end %>

data/ext/cumo/narray/gen/tmpl/rand_norm.c

@@ -1,12 +1,12 @@
 typedef struct {
     dtype mu;
-    rtype sigma;
+    <% if is_half %>float<% else %>rtype<% end %> sigma;
     u_int64_t seed;
     u_int64_t offset;
 } randn_opt_t;
 
-void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, uint64_t seed, uint64_t offset, dtype mu, rtype sigma, uint64_t n);
-void <%="cumo_#{c_iter}_stride_kernel_launch"%>(char *p1, ssize_t s1, uint64_t seed, uint64_t offset, dtype mu, rtype sigma, uint64_t n);
+void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, uint64_t seed, uint64_t offset, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma, uint64_t n);
+void <%="cumo_#{c_iter}_stride_kernel_launch"%>(char *p1, ssize_t s1, uint64_t seed, uint64_t offset, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma, uint64_t n);
 
 static void
 <%=c_iter%>(cumo_na_loop_t *const lp)
@@ -16,7 +16,7 @@ static void
     ssize_t  s1;
     size_t  *idx1;
     dtype    mu;
-    rtype    sigma;
+    <% if is_half %>float<% else %>rtype<% end %>    sigma;
     randn_opt_t *g;
 
     CUMO_INIT_COUNTER(lp, i);

data/ext/cumo/narray/gen/tmpl/rand_norm_kernel.cu

@@ -16,19 +16,21 @@
 <% end %>
 
 __device__ static dtype
-<%="cumo_#{c_iter}_value"%>(curandStatePhilox4_32_10_t *st, dtype mu, rtype sigma)
+<%="cumo_#{c_iter}_value"%>(curandStatePhilox4_32_10_t *st, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma)
 {
     //<% if is_complex %>
     dtype z;
     CUMO_REAL(z) = cumo_rand_normal(st) * sigma + CUMO_REAL(mu);
     CUMO_IMAG(z) = cumo_rand_normal(st) * sigma + CUMO_IMAG(mu);
     return z;
+    //<% elsif is_half %>
+    return cumo_float2half(cumo_rand_normal(st) * sigma + cumo_half2float(mu));
     //<% else %>
     return cumo_rand_normal(st) * sigma + mu;
     //<% end %>
 }
 
-__global__ void <%="cumo_#{c_iter}_index_kernel"%>(char *p1, size_t *idx1, uint64_t seed, uint64_t offset, dtype mu, rtype sigma, uint64_t n)
+__global__ void <%="cumo_#{c_iter}_index_kernel"%>(char *p1, size_t *idx1, uint64_t seed, uint64_t offset, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma, uint64_t n)
 {
     for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) {
         curandStatePhilox4_32_10_t st;
@@ -37,7 +39,7 @@ __global__ void <%="cumo_#{c_iter}_index_kernel"%>(char *p1, size_t *idx1, uint6
     }
 }
 
-__global__ void <%="cumo_#{c_iter}_stride_kernel"%>(char *p1, ssize_t s1, uint64_t seed, uint64_t offset, dtype mu, rtype sigma, uint64_t n)
+__global__ void <%="cumo_#{c_iter}_stride_kernel"%>(char *p1, ssize_t s1, uint64_t seed, uint64_t offset, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma, uint64_t n)
 {
     for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) {
         curandStatePhilox4_32_10_t st;
@@ -55,7 +57,7 @@ extern "C" {
 #endif
 #endif
 
-void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, uint64_t seed, uint64_t offset, dtype mu, rtype sigma, uint64_t n)
+void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, uint64_t seed, uint64_t offset, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma, uint64_t n)
 {
     size_t grid_dim = cumo_get_grid_dim(n);
     size_t block_dim = cumo_get_block_dim(n);
@@ -63,7 +65,7 @@ void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, uint64_t
     cumo_cuda_runtime_check_kernel_launch();
 }
 
-void <%="cumo_#{c_iter}_stride_kernel_launch"%>(char *p1, ssize_t s1, uint64_t seed, uint64_t offset, dtype mu, rtype sigma, uint64_t n)
+void <%="cumo_#{c_iter}_stride_kernel_launch"%>(char *p1, ssize_t s1, uint64_t seed, uint64_t offset, dtype mu, <% if is_half %>float<% else %>rtype<% end %> sigma, uint64_t n)
 {
     size_t grid_dim = cumo_get_grid_dim(n);
     size_t block_dim = cumo_get_block_dim(n);

data/ext/cumo/narray/gen/tmpl/real_accum_kernel.cu

@@ -5,102 +5,103 @@
 }  /* extern "C" { */
 #endif
 
+<% acc = is_half ? 'float' : dtype %>
+<% to_acc = is_half ? 'cumo_half2float' : '' %>
+<% from_acc = is_half ? 'cumo_float2half' : '' %>
+<% acc_zero = is_half ? '0.0f' : 'm_zero' %>
+
+
 // Reduce also combines two accumulators, so next has to be the accumulator type:
-// the integer types widen to 64 bits and taking dtype there truncates every
-// partial the shared-memory tree merges.
+// the integer types widen to 64 bits and half to float, and taking dtype there
+// truncates every partial the shared-memory tree merges.
 struct cumo_<%=type_name%>_sum_impl {
-    __device__ <%=dtype%> Identity(int64_t /*index*/) { return m_zero; }
-    __device__ <%=dtype%> MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(<%=dtype%> next, <%=dtype%>& accum) { accum += next; }
-    __device__ <%=dtype%> MapOut(<%=dtype%> accum) { return accum; }
+    __device__ <%=acc%> Identity(int64_t /*index*/) { return <%=acc_zero%>; }
+    __device__ <%=acc%> MapIn(dtype in, int64_t /*index*/) { return <%=to_acc%>(in); }
+    __device__ void Reduce(<%=acc%> next, <%=acc%>& accum) { accum = accum + next; }
+    __device__ <%=dtype%> MapOut(<%=acc%> accum) { return <%=from_acc%>(accum); }
 };
 
 struct cumo_<%=type_name%>_prod_impl {
     __device__ <%=dtype%> Identity(int64_t /*index*/) { return m_one; }
     __device__ <%=dtype%> MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(<%=dtype%> next, <%=dtype%>& accum) { accum *= next; }
+    __device__ void Reduce(<%=dtype%> next, <%=dtype%>& accum) { accum = m_mul(accum, next); }
     __device__ <%=dtype%> MapOut(<%=dtype%> accum) { return accum; }
 };
 
-struct cumo_<%=type_name%>_min_impl {
+// One set of rules answers for both components, and min, max, minmax and ptp
+// all name a whole set rather than a rule each. A pair that mixes two sets --
+// the nan-aware min beside the default max -- is then not expressible, which
+// is the shape #372 had: #219 changed the identity in min and max and left the
+// copies inside the fused pair behind.
+struct cumo_<%=type_name%>_extremum_rules {
+    struct Min {
 <% if is_float %>
-    // A NaN loses the comparison, so it is skipped unless the accumulator is
-    // still the identity: the reduction then answers NaN only when every
-    // element was NaN, and equal elements keep the earlier one as numo does.
-    __device__ dtype Identity(int64_t /*index*/) { return (dtype)nan(""); }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(dtype next, dtype& accum) { if (next < accum || !not_nan(accum)) { accum = next; } }
+        // A NaN loses the comparison, so it is skipped unless the accumulator
+        // is still the identity: the reduction then answers NaN only when every
+        // element was NaN, and equal elements keep the earlier one as numo does.
+        __device__ static dtype Identity() { return (dtype)nan(""); }
+        __device__ static void Reduce(dtype next, dtype& accum) { if (m_lt(next, accum) || !not_nan(accum)) { accum = next; } }
 <% else %>
-    __device__ dtype Identity(int64_t /*index*/) { return DATA_MAX; }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(dtype next, dtype& accum) { accum = next < accum ? next : accum; }
+        __device__ static dtype Identity() { return DATA_MAX; }
+        __device__ static void Reduce(dtype next, dtype& accum) { accum = m_lt(next, accum) ? next : accum; }
 <% end %>
-    __device__ dtype MapOut(dtype accum) { return accum; }
-};
-
-struct cumo_<%=type_name%>_max_impl {
+    };
+    struct Max {
 <% if is_float %>
-    __device__ dtype Identity(int64_t /*index*/) { return (dtype)nan(""); }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(dtype next, dtype& accum) { if (accum < next || !not_nan(accum)) { accum = next; } }
+        __device__ static dtype Identity() { return (dtype)nan(""); }
+        __device__ static void Reduce(dtype next, dtype& accum) { if (m_lt(accum, next) || !not_nan(accum)) { accum = next; } }
 <% else %>
-    __device__ dtype Identity(int64_t /*index*/) { return DATA_MIN; }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(dtype next, dtype& accum) { accum = next < accum ? accum : next; }
+        __device__ static dtype Identity() { return DATA_MIN; }
+        __device__ static void Reduce(dtype next, dtype& accum) { accum = m_lt(next, accum) ? accum : next; }
 <% end %>
+    };
+};
+
+template <typename Rule>
+struct cumo_<%=type_name%>_extremum_of {
+    __device__ dtype Identity(int64_t /*index*/) { return Rule::Identity(); }
+    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
+    __device__ void Reduce(dtype next, dtype& accum) { Rule::Reduce(next, accum); }
     __device__ dtype MapOut(dtype accum) { return accum; }
 };
 
-// min and max fused, so that minmax reads the input once. The rules per
-// component are the two above verbatim: minmax must not answer differently
-// from min and max.
-struct cumo_<%=type_name%>_minmax_impl {
+// What minmax and ptp accumulate, and how it goes in and out. No Identity and
+// no Reduce: those come from the rules, so a form that names none does not
+// compile.
+struct cumo_<%=type_name%>_minmax_pair {
     struct MinAndMax {
         dtype min;
         dtype max;
     };
-<% if is_float %>
-    __device__ MinAndMax Identity(int64_t /*index*/) { return {(dtype)nan(""), (dtype)nan("")}; }
-    __device__ MinAndMax MapIn(dtype in, int64_t /*index*/) { return {in, in}; }
-    __device__ void Reduce(MinAndMax next, MinAndMax& accum) {
-        if (next.min < accum.min || !not_nan(accum.min)) { accum.min = next.min; }
-        if (accum.max < next.max || !not_nan(accum.max)) { accum.max = next.max; }
-    }
-<% else %>
-    __device__ MinAndMax Identity(int64_t /*index*/) { return {DATA_MAX, DATA_MIN}; }
     __device__ MinAndMax MapIn(dtype in, int64_t /*index*/) { return {in, in}; }
-    __device__ void Reduce(MinAndMax next, MinAndMax& accum) {
-        accum.min = next.min < accum.min ? next.min : accum.min;
-        accum.max = next.max < accum.max ? accum.max : next.max;
-    }
-<% end %>
     __device__ void MapOut(MinAndMax accum, dtype* out_min, dtype* out_max) {
         *out_min = accum.min;
         *out_max = accum.max;
     }
 };
 
-struct cumo_<%=type_name%>_ptp_impl {
-    struct MinAndMax {
-        dtype min;
-        dtype max;
-    };
-    __device__ MinAndMax Identity(int64_t /*index*/) { return {DATA_MAX, DATA_MIN}; }
-    __device__ MinAndMax MapIn(dtype in, int64_t /*index*/) { return {in, in}; }
+// min and max fused, so that minmax reads the input once.
+template <typename Rules>
+struct cumo_<%=type_name%>_minmax_of : cumo_<%=type_name%>_minmax_pair {
+    __device__ MinAndMax Identity(int64_t /*index*/) { return {Rules::Min::Identity(), Rules::Max::Identity()}; }
     __device__ void Reduce(MinAndMax next, MinAndMax& accum) {
-        if (next.min < accum.min) { accum.min = next.min; }
-        if (accum.max < next.max) { accum.max = next.max; }
-    }
-    __device__ dtype MapOut(MinAndMax accum) {
-    <% if is_float %>
-        // A NaN loses both comparisons above, so every element being NaN leaves
-        // the identity untouched. An empty reduction raises before it gets here.
-        if (accum.max < accum.min) { return (dtype)nan(""); }
-    <% end %>
-        return m_sub(accum.max, accum.min);
+        Rules::Min::Reduce(next.min, accum.min);
+        Rules::Max::Reduce(next.max, accum.max);
     }
 };
 
+// ptp is its minmax with the pair subtracted. The one-argument MapOut hides
+// the two-output one it inherits.
+template <typename Rules>
+struct cumo_<%=type_name%>_ptp_of : cumo_<%=type_name%>_minmax_of<Rules> {
+    __device__ dtype MapOut(cumo_<%=type_name%>_minmax_pair::MinAndMax accum) { return m_sub(accum.max, accum.min); }
+};
+
+using cumo_<%=type_name%>_min_impl = cumo_<%=type_name%>_extremum_of<cumo_<%=type_name%>_extremum_rules::Min>;
+using cumo_<%=type_name%>_max_impl = cumo_<%=type_name%>_extremum_of<cumo_<%=type_name%>_extremum_rules::Max>;
+using cumo_<%=type_name%>_minmax_impl = cumo_<%=type_name%>_minmax_of<cumo_<%=type_name%>_extremum_rules>;
+using cumo_<%=type_name%>_ptp_impl = cumo_<%=type_name%>_ptp_of<cumo_<%=type_name%>_extremum_rules>;
+
 <% unless is_float %>
 // mean, var, stddev and rms answer in double for the integer types, so they
 // accumulate in double too rather than in dtype.
@@ -155,11 +156,15 @@ struct cumo_<%=type_name%>_rms_impl {
 // NaN, so it maps to the identity and the tree never sees it. min and max go
 // the other way: numo answers NaN as soon as one element is NaN, and a NaN
 // absorbs in Reduce because it loses every comparison.
+//
+// min and max take an infinity for their identity rather than the largest
+// finite value, so that an element which is itself an infinity still wins.
+// numo seeds from the first element and never faces the question.
 struct cumo_<%=type_name%>_sum_nan_impl {
-    __device__ dtype Identity(int64_t /*index*/) { return m_zero; }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return not_nan(in) ? in : m_zero; }
-    __device__ void Reduce(dtype next, dtype& accum) { accum = m_add(next, accum); }
-    __device__ dtype MapOut(dtype accum) { return accum; }
+    __device__ <%=acc%> Identity(int64_t /*index*/) { return <%=acc_zero%>; }
+    __device__ <%=acc%> MapIn(dtype in, int64_t /*index*/) { return not_nan(in) ? <%=to_acc%>(in) : <%=acc_zero%>; }
+    __device__ void Reduce(<%=acc%> next, <%=acc%>& accum) { accum = next + accum; }
+    __device__ dtype MapOut(<%=acc%> accum) { return <%=from_acc%>(accum); }
 };
 
 struct cumo_<%=type_name%>_prod_nan_impl {
@@ -169,40 +174,21 @@ struct cumo_<%=type_name%>_prod_nan_impl {
     __device__ dtype MapOut(dtype accum) { return accum; }
 };
 
-struct cumo_<%=type_name%>_min_nan_impl {
-    __device__ dtype Identity(int64_t /*index*/) { return DATA_MAX; }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(dtype next, dtype& accum) { if (!not_nan(next) || next < accum) { accum = next; } }
-    __device__ dtype MapOut(dtype accum) { return accum; }
-};
-
-struct cumo_<%=type_name%>_max_nan_impl {
-    __device__ dtype Identity(int64_t /*index*/) { return DATA_MIN; }
-    __device__ dtype MapIn(dtype in, int64_t /*index*/) { return in; }
-    __device__ void Reduce(dtype next, dtype& accum) { if (!not_nan(next) || accum < next) { accum = next; } }
-    __device__ dtype MapOut(dtype accum) { return accum; }
-};
-
-struct cumo_<%=type_name%>_minmax_nan_impl {
-    struct MinAndMax {
-        dtype min;
-        dtype max;
+struct cumo_<%=type_name%>_extremum_nan_rules {
+    struct Min {
+        __device__ static dtype Identity() { return (dtype)INFINITY; }
+        __device__ static void Reduce(dtype next, dtype& accum) { if (!not_nan(next) || m_lt(next, accum)) { accum = next; } }
+    };
+    struct Max {
+        __device__ static dtype Identity() { return (dtype)(-INFINITY); }
+        __device__ static void Reduce(dtype next, dtype& accum) { if (!not_nan(next) || m_lt(accum, next)) { accum = next; } }
     };
-    __device__ MinAndMax Identity(int64_t /*index*/) { return {DATA_MAX, DATA_MIN}; }
-    __device__ MinAndMax MapIn(dtype in, int64_t /*index*/) { return {in, in}; }
-    __device__ void Reduce(MinAndMax next, MinAndMax& accum) {
-        if (!not_nan(next.min) || next.min < accum.min) { accum.min = next.min; }
-        if (!not_nan(next.max) || accum.max < next.max) { accum.max = next.max; }
-    }
-    __device__ void MapOut(MinAndMax accum, dtype* out_min, dtype* out_max) {
-        *out_min = accum.min;
-        *out_max = accum.max;
-    }
 };
 
-struct cumo_<%=type_name%>_ptp_nan_impl : cumo_<%=type_name%>_minmax_nan_impl {
-    __device__ dtype MapOut(MinAndMax accum) { return m_sub(accum.max, accum.min); }
-};
+using cumo_<%=type_name%>_min_nan_impl = cumo_<%=type_name%>_extremum_of<cumo_<%=type_name%>_extremum_nan_rules::Min>;
+using cumo_<%=type_name%>_max_nan_impl = cumo_<%=type_name%>_extremum_of<cumo_<%=type_name%>_extremum_nan_rules::Max>;
+using cumo_<%=type_name%>_minmax_nan_impl = cumo_<%=type_name%>_minmax_of<cumo_<%=type_name%>_extremum_nan_rules>;
+using cumo_<%=type_name%>_ptp_nan_impl = cumo_<%=type_name%>_ptp_of<cumo_<%=type_name%>_extremum_nan_rules>;
 <% end %>
 
 #if defined(__cplusplus)

data/ext/cumo/narray/gen/tmpl/sort.c

@@ -42,7 +42,7 @@ static void
     CUMO_INIT_COUNTER(lp, n);
     CUMO_INIT_PTR(lp, 0, ptr, step);
     CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
-    cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+    cumo_cuda_runtime_device_synchronize();
     <%=type_name%>_qsort<%=j%>(ptr, n, step);
 }
 <% end %>

data/ext/cumo/narray/gen/tmpl/sort_index.c

@@ -50,6 +50,12 @@ static void
 
     ptr = (char**)(lp->opt_ptr);
 
+    // The row is managed memory that ndloop may have just filled with a copy
+    // kernel, and everything below reads it on the host. Per row rather than
+    // once for the call, because that copy happens per row.
+    CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
+    cumo_cuda_runtime_device_synchronize();
+
     //printf("(ptr=%lx, d_ptr=%lx,d_step=%ld, i_ptr=%lx,i_step=%ld, o_ptr=%lx,o_step=%ld)\n",(size_t)ptr,(size_t)d_ptr,(ssize_t)d_step,(size_t)i_ptr,(ssize_t)i_step,(size_t)o_ptr,(ssize_t)o_step);
 
     if (n==1) {
@@ -153,9 +159,6 @@ static VALUE
     size = na->size*sizeof(void*); // max capa
     buf = rb_alloc_tmp_buffer(&tmp, size);
 
-    CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
-    cudaDeviceSynchronize();
-
     res = cumo_na_ndloop3(&ndf, buf, 3, self, idx, reduce);
     rb_free_tmp_buffer(&tmp);
     return res;

data/ext/cumo/narray/gen/tmpl/store_array.c

@@ -24,29 +24,29 @@ static void
     v1 = lp->args[1].value;
     i = 0;
 
-    if (lp->args[1].ptr) {
-        if (v1 == Qtrue) {
-            // The loop counter is the destination length, but the sub-narray
-            // may be shorter. Copy only what the source actually holds, or the
-            // kernel would read past its end; the rest is zero-filled below.
-            i = lp->args[1].shape[0];
-            if (i > n) {
-                i = n;
-            }
+    if (CumoIsNArray(v1)) {
+        // shape[0] is how many elements the sub-narray offers at this row,
+        // which is zero where it does not reach. The loop counter is the
+        // destination length, so copying more would read past the row's end.
+        i = lp->args[1].shape[0];
+        if (i > n) {
+            i = n;
+        }
+        if (i > 0) {
             CUMO_NDL_CNT(lp) = i;
             iter_<%=type_name%>_store_<%=type_name%>(lp);
             CUMO_NDL_CNT(lp) = n;
-            //<% if c_iter.include? 'robject' %>
-            // The zero fill below walks from here. The kernel the other branch
-            // launches takes the offset of the first element left to fill as an
-            // argument instead, so advancing for it would count i twice.
-            if (idx1) {
-                idx1 += i;
-            } else {
-                p1 += s1 * i;
-            }
-            //<% end %>
         }
+        //<% if c_iter.include? 'robject' %>
+        // The zero fill below walks from here. The kernel the other branch
+        // launches takes the offset of the first element left to fill as an
+        // argument instead, so advancing for it would count i twice.
+        if (idx1) {
+            idx1 += i;
+        } else {
+            p1 += s1 * i;
+        }
+        //<% end %>
         goto loop_end;
     }
 
@@ -55,8 +55,7 @@ static void
         n1 = RARRAY_LEN(v1);
         break;
     case T_NIL:
-        n1 = 0;
-        break;
+        goto loop_end;
     default:
         n1 = 1;
     }
@@ -64,7 +63,7 @@ static void
     //<% if c_iter.include? 'robject' %>
     {
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("store_<%=name%>", "<%=type_name%>");
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
 
         if (idx1) {
             for (i=i1=0; i1<n1 && i<n; i1++) {

data/ext/cumo/narray/gen/tmpl/store_bit.c

@@ -23,7 +23,7 @@ static void
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
         // The index arrays are filled by a kernel; wait for it before reading
         // them from here.
-        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+        cumo_cuda_runtime_device_synchronize();
         if (idx2) {
             if (idx1) {
                 for (; i--;) {

data/ext/cumo/narray/gen/tmpl/store_from.c

@@ -34,7 +34,7 @@ static void
     // until that has finished. Cumo::RObject is the only dtype whose loop runs
     // on the host and still sees the index; every other one hands it to a
     // kernel of its own, which the stream already orders.
-    cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+    cumo_cuda_runtime_device_synchronize();
     {
         <%=dtype%> x;
         dtype y;

data/ext/cumo/narray/gen/tmpl/store_from_kernel.cu

@@ -40,10 +40,45 @@ void <%="cumo_#{c_iter}_stridx_kernel_launch"%>(cumo_na_iarray_stridx_t* a1, cum
     cumo_cuda_runtime_check_kernel_launch();
 }
 
+__global__ void <%="cumo_#{c_iter}_transpose_kernel"%>(char *p1, char *p2, uint64_t rows, uint64_t cols)
+{
+    CUMO_TRANSPOSE_TILE_DECL(dtype, tile);
+
+    CUMO_TRANSPOSE_TILE_LOOP(tile, rows, cols,
+        <%=macro%>(*(<%=dtype%>*)(p2 + cumo_tile_src * sizeof(<%=dtype%>))),
+        *(dtype*)(p1 + cumo_tile_dst * sizeof(dtype)) = cumo_tile_val;
+    );
+}
+
+// True when the destination runs along its rows and the source along its
+// columns, which is what a copy of a transposed view looks like. A side shorter
+// than a tile leaves most of each warp idle, and the plain loop already reads or
+// writes such a side in one go, so the tiles only pay off once both sides reach
+// one.
+static int
+<%="cumo_#{c_iter}_is_transpose"%>(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_indexer_t* indexer)
+{
+    return CUMO_TRANSPOSE_TILE_FITS(indexer) &&
+        a1->step[1] == (ssize_t)sizeof(dtype) &&
+        a1->step[0] == (ssize_t)(sizeof(dtype) * indexer->shape[1]) &&
+        a2->step[0] == (ssize_t)sizeof(<%=dtype%>) &&
+        a2->step[1] == (ssize_t)(sizeof(<%=dtype%>) * indexer->shape[0]);
+}
+
 void <%="cumo_#{c_iter}_kernel_launch"%>(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_indexer_t* indexer)
 {
-    size_t grid_dim = cumo_get_grid_dim(indexer->total_size);
-    size_t block_dim = cumo_get_block_dim(indexer->total_size);
+    size_t grid_dim, block_dim;
+
+    if (<%="cumo_#{c_iter}_is_transpose"%>(a1, a2, indexer)) {
+        CUMO_TRANSPOSE_LAUNCH(<%="cumo_#{c_iter}_transpose_kernel"%>,
+                              indexer->shape[0], indexer->shape[1],
+                              a1->ptr, a2->ptr);
+        cumo_cuda_runtime_check_kernel_launch();
+        return;
+    }
+
+    grid_dim = cumo_get_grid_dim(indexer->total_size);
+    block_dim = cumo_get_block_dim(indexer->total_size);
     switch (indexer->ndim) {
     <% (0..opt_indexer_ndim).each do |idim| %>
     case <%=idim%>:

data/ext/cumo/narray/gen/tmpl/to_a.c

@@ -38,6 +38,6 @@ static VALUE
     cumo_ndfunc_arg_out_t aout[1] = {{rb_cArray,0}}; // dummy?
     cumo_ndfunc_t ndf = { <%=c_iter%>, CUMO_FULL_LOOP_NIP, 3, 1, ain, aout };
     CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>");
-    cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+    cumo_cuda_runtime_device_synchronize();
     return cumo_na_ndloop_cast_narray_to_rarray(&ndf, self, Qnil);
 }