cumo 0.5.11 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b345c4e036212d334600041165fe79e533ef0e1dd450e22b1288f5e6d6214ffa
-  data.tar.gz: af60f78cdd2f39a96e8a183eff02a26c26d8977560ba148643c55df5961c2bf2
+  metadata.gz: 0d6b3e0decf69d07199a7930dc6a23d1a05a90ed044cf007203c78c28a8e9ea1
+  data.tar.gz: dec0ca1f195b302919d30085fcc139911299ba075193b67cab9965907210092f
 SHA512:
-  metadata.gz: 361042e17c2f498b8e4f5ddc76bbd119688a6a3a3ac176307973fd51c75b332d4d831a82c0fc0e8a3b28be98bfbe50d507ec67d517c609c2342caf57d94d9c41
-  data.tar.gz: 53da42c447fe31812654e1ab0566f5e553f781daaf75b97bb1c36630b8dc3ed195704002990bc4a2a9f4fdd6e26e92c2d5905ff64da0e0f058b01dae20ae3dec
+  metadata.gz: e4f8d6204fca59288efad1ab5dec228d474de02ee3465f89afe7fc7440b456b0d46b78c92fb1e48e92063a2b3c385dedb8955aed3ddfa4411c13beb2f32a5417
+  data.tar.gz: a397b427534b829b13f8bd4a4d6f8616d918bde834dc5c69a5611671ec942796d8615bd74d2564a2c3cd6cdaf9e572f4599e0704071f21a3c40394d0236b148d

data/CHANGELOG.md

@@ -1,3 +1,36 @@
+# 0.6.0 (2026/09/06)
+
+Breaking changes:
+
+* `divmod` and `%` floor the quotient and give the remainder the divisor's sign, as Ruby does, on every signed type: `Cumo::Int32[-7].divmod(3)` answers `[-3, 2]` where it answered `[-2, -1]`, and `-7 % 3` answers `2` rather than `-1`. Dividing a float by zero answers a quotient of `NaN` where it answered `Infinity`. Unsigned types are unchanged (PR #364)
+* A view whose base has since been given a smaller shape raises `RuntimeError` instead of reading past what the base holds (PR #363)
+* `marshal_load` and `initialize` leave an array unallocated when they take a new shape, so it reads as unallocated until something is stored in it, and both refuse a view. `initialize` also refuses a frozen array, as `marshal_load` already did (PR #361, PR #360, PR #355)
+* A range subscript longer than 2**31 answers the length it names instead of an empty view, and one longer than 2**32 no longer answers with the low 32 bits of it (PR #354)
+* `cov` rejects a 3-dimensional receiver or `y`, 2-dimensional or non-integer `fweights`, and any `ddof` but 0 or 1, where it answered from them; it warns and answers `nan` when the degrees of freedom come out at zero or below (PR #343)
+* `Cumo::Bit#extract` answers with a zero-dimensional `Cumo::Bit`, as every other dtype does, rather than a Ruby Integer read back through a synchronize (PR #341)
+* `batch_norm` and its backward reject an axis that names nothing in `x`, repeats a dimension, or is out of order, where they answered as though a different axis had been given (PR #336)
+
+Fixes:
+
+* Fix `RObject#divmod` reading the pair an element's `divmod` answered without looking at it: a bare Integer or a String took the process down, and a shorter Array handed back the words past its end as the quotient and the remainder (PR #359)
+* Fix an out-of-bounds read when `initialize` is given a new shape through `send`, which left the buffer sized for the old one: 4096 elements were read out of 8, and the same on an `Cumo::RObject` crashed at the next collection (PR #361)
+* Fix a memory leak when an Array subscript is rejected, the host buffer it is staged in being freed only after the loop that fills it, and the memory a subscript piles up before the collector takes it back (PR #357, PR #358)
+* Fix `marshal_load` writing past a buffer sized for the shape it replaces, and the double free that came of releasing a buffer noted down before the shape's own `to_int` had run (PR #355, PR #356)
+* Fix reading past a Ruby array or string that a conversion shrank underneath the walk: a subscript, a shape, an axis list, a marshal array and the string behind `from_binary` were each measured once and walked afterwards, and `to_int` is free to empty any of them (PR #353)
+* Fix `Cumo::Bit#swap_byte` walking one byte per bit, eight times past the end of the packed buffer; `hton`, `to_network` and `to_swapped` reach the same loop (PR #352)
+
+Changes:
+
+* Add `argsort`, which answers positions along an axis rather than into the flattened array as `sort_index` does (PR #342)
+* Add `mean`, `var`, `stddev` and `rms` to the integer types and to `Cumo::Bit`, which had only the float ones (PR #344, PR #345)
+* Walk an elementwise function along the memory of the operand it writes rather than the shape it was given; on a transposed 1024x1024 SFloat view `a.inplace + 1.0` goes from 1180.6us to 78.3us (PR #351)
+* Reduce along the contiguous axis with a contiguous group of threads, keep a reduction's operands out of local memory, and give a short strided row more elements per thread; `[1024, 4, 1024].sum(axis: 1)` goes from 142.0us to 23.4us and the same `mulsum` from 675.3us to 30.9us (PR #350, PR #347)
+* Split a flat index in 32 bits where the array fits in them, the 64-bit division being a software routine on the GPU; a 4M SFloat `a[true, 1..-2] * 2.0` goes from 60.6us to 38.7us (PR #348)
+* Reverse the bytes of an array in a kernel instead of on the host; a 4M SFloat `swap_byte` goes from 5617.0us to 13.0us (PR #335)
+* Add `bench/cumo_shape_probe.rb`, which reads each shape of an operation against the same bytes moved along the best path it has (PR #349)
+* Make every object depend on the headers it is built from, a header-only change having left stale objects behind (PR #346)
+* Answer the four cuDNN TODOs left in the tree, and say what the memory pool's `TestRemoveFromFreeList` holds rather than what it waits for (PR #340, PR #339, PR #338, PR #337)
+
 # 0.5.11 (2026/08/29)
 
 Breaking changes:

data/bench/cumo_shape_probe.rb

@@ -0,0 +1,264 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# ---------------------------------------------------------------------------
+# Cumo shape sweep
+#
+# cumo_probe.rb runs every kernel family at one shape and reads each member
+# against the fastest of its group. What that cannot see is a family whose
+# speed depends on the shape it is handed: a reduction along a short row, an
+# operand that is broadcast or strided, a copy that transposes. This sweep
+# holds the bytes fixed, varies the shape and the layout, and reads every case
+# against the same bytes moved down the best path there is for them: the flat
+# reduction, the add of two contiguous arrays, the contiguous copy.
+#
+#   GPU=1 ruby cumo_shape_probe.rb                     # the default sweep
+#   GPU=1 GROUP=reduce ruby cumo_shape_probe.rb        # reduce, elementwise or copy
+#   GPU=1 ALL=1 ruby cumo_shape_probe.rb               # every row, not just the flagged
+#   GPU=1 FLAG=3 ruby cumo_shape_probe.rb              # how far off the reference is called out
+#   GPU=1 DTYPE=DFloat ELEMENTS=67108864 ruby cumo_shape_probe.rb
+#   GPU=1 JSON=before.json ruby cumo_shape_probe.rb    # keep this run's rows ...
+#   GPU=1 COMPARE=before.json ruby cumo_shape_probe.rb # ... and read a later build against them
+#   ruby cumo_shape_probe.rb                           # the same sweep on Numo
+#
+# Every case queues LAUNCHES calls and synchronizes once, and the run opens
+# with a few seconds of load. On a laptop the memory clock climbs in steps,
+# and a best-of-N that synchronizes between calls can settle on a step below
+# the top and read a third low without anything being wrong.
+#
+# The sweep runs in one process. The cases work in place or reduce, so the
+# only results allocated are the reductions', a thousandth of their input.
+#
+# The default 4M elements of SFloat is 16 MB, inside the L2 of this class of
+# GPU, which is where the activations of a model live. ELEMENTS=67108864 moves
+# the sweep to DRAM, where a cost paid per element hides behind the bandwidth.
+# ---------------------------------------------------------------------------
+
+require 'json'
+
+GPU = !%w[0 false].include?(ENV['GPU'].to_s.downcase) && !ENV['GPU'].to_s.empty?
+
+if GPU
+  require 'cumo/narray'
+  XM = Cumo
+else
+  require 'numo/narray'
+  XM = Numo
+end
+
+GROUP    = ENV['GROUP']
+FLAG     = (ENV['FLAG'] || 2.0).to_f
+SHOW_ALL = !ENV['ALL'].to_s.empty?
+LAUNCHES = (ENV['LAUNCHES'] || 10).to_i
+ROUNDS   = (ENV['ROUNDS'] || 4).to_i
+ELEMENTS = (ENV['ELEMENTS'] || (1 << 22)).to_i
+DTYPE    = XM.const_get(ENV['DTYPE'] || 'SFloat')
+JSON_OUT = ENV['JSON']
+COMPARE  = ENV['COMPARE']
+
+def sync
+  XM::CUDA::Runtime.cudaDeviceSynchronize if GPU
+end
+
+def now
+  Process.clock_gettime(Process::CLOCK_MONOTONIC)
+end
+
+def elmsz
+  DTYPE::ELEMENT_BYTE_SIZE
+end
+
+def warm(seconds)
+  a = DTYPE.new(1 << 22).fill(0)
+  t = now
+  a.inplace + 1 while now - t < seconds
+  sync
+end
+
+# Seconds for one call, best of ROUNDS batches of LAUNCHES calls.
+def time_one
+  2.times { yield }
+  sync
+  best = Float::INFINITY
+  ROUNDS.times do
+    sync
+    t = now
+    LAUNCHES.times { yield }
+    sync
+    best = [best, (now - t) / LAUNCHES].min
+  end
+  best
+end
+
+Row = Struct.new(:group, :name, :shape, :us, :gbs, :ref, keyword_init: true) do
+  def key
+    "#{group}/#{name}"
+  end
+end
+
+ROWS = []
+
+# bytes is what the case moves, and the reference row is the one this case is
+# read against.
+def probe(group, name, shape, bytes, ref: nil, &blk)
+  return if GROUP && !group.start_with?(GROUP)
+
+  s = time_one(&blk)
+  ROWS << Row.new(group: group, name: name, shape: shape.inspect, us: s * 1e6, gbs: bytes / s / 1e9, ref: ref || name)
+end
+
+def rand_array(*shape)
+  a = DTYPE.new(*shape)
+  DTYPE.name.include?('Int') ? a.rand(100) : a.rand
+end
+
+# --- reductions ---------------------------------------------------------------
+#
+# The same ELEMENTS reduced along the last axis, the first axis and a middle
+# axis, over rows of every length, all read against the flat reduction of the
+# same array. Each output of every case is a reduction over len elements.
+def sweep_reduce
+  n = ELEMENTS
+  es = elmsz
+  a = rand_array(n)
+  b = rand_array(n)
+  methods = %i[sum max mean argmax mulsum].select { |m| a.respond_to?(m) }
+  methods.each do |m|
+    bytes = (m == :mulsum ? 2 : 1) * n * es
+    probe('reduce', "#{m} flat", [n], bytes) { m == :mulsum ? a.mulsum(b) : a.send(m) }
+  end
+  [4, 16, 64, 256, 1024, 4096].each do |len|
+    next if len > n
+
+    rows = n / len
+    r1 = 1 << (Math.log2(rows).to_i / 2)
+    r2 = rows / r1
+    last = a.reshape(rows, len)
+    last_b = b.reshape(rows, len)
+    first = a.reshape(len, rows)
+    first_b = b.reshape(len, rows)
+    mid = a.reshape(r1, len, r2)
+    mid_b = b.reshape(r1, len, r2)
+    methods.each do |m|
+      bytes = (m == :mulsum ? 2 : 1) * n * es
+      ref = "#{m} flat"
+      call = lambda do |x, y, axis|
+        m == :mulsum ? x.mulsum(y, axis: axis) : x.send(m, axis: axis)
+      end
+      probe('reduce', "#{m} len #{len} last", [rows, len], bytes, ref: ref) { call.call(last, last_b, 1) }
+      probe('reduce', "#{m} len #{len} first", [len, rows], bytes, ref: ref) { call.call(first, first_b, 0) }
+      probe('reduce', "#{m} len #{len} middle", [r1, len, r2], bytes, ref: ref) { call.call(mid, mid_b, 1) }
+    end
+  end
+end
+
+# --- elementwise --------------------------------------------------------------
+#
+# One [R, C] array added to in place, its operand contiguous, a scalar, a
+# broadcast row or column, a slice, a stepped view or a transpose, plus the
+# same broadcast through three and four dimensions. The reference is the add
+# of two contiguous arrays, which the loop walks as one run.
+def sweep_elementwise
+  n = ELEMENTS
+  es = elmsz
+  r = 1024
+  c = n / r
+  a = rand_array(r, c)
+  b = rand_array(r, c)
+  row = rand_array(c)
+  col = rand_array(r, 1)
+  probe('elementwise', 'a + b contiguous', [r, c], 3 * n * es) { a.inplace + b }
+  probe('elementwise', 'a * scalar', [r, c], 2 * n * es, ref: 'a + b contiguous') { a.inplace * 2 }
+  probe('elementwise', 'a + row broadcast', [r, c], 2 * n * es, ref: 'a + b contiguous') { a.inplace + row }
+  probe('elementwise', 'a - column broadcast', [r, c], 2 * n * es, ref: 'a + b contiguous') { a.inplace - col }
+  a3 = a.reshape(16, r / 16, c)
+  b3 = rand_array(1, r / 16, 1)
+  probe('elementwise', 'a + broadcast 3d', [16, r / 16, c], 2 * n * es, ref: 'a + b contiguous') { a3.inplace + b3 }
+  a4 = a.reshape(4, 4, r / 16, c)
+  b4 = rand_array(1, 4, 1, c)
+  probe('elementwise', 'a + broadcast 4d', [4, 4, r / 16, c], 2 * n * es, ref: 'a + b contiguous') { a4.inplace + b4 }
+  half = 0...(c / 2)
+  probe('elementwise', 'a + b column slice', [r, c / 2], 3 * n * es / 2, ref: 'a + b contiguous') { a[true, half].inplace + b[true, half] }
+  every_other = (0..-1) % 2
+  probe('elementwise', 'a * scalar stepped rows', [r / 2, c], n * es, ref: 'a + b contiguous') { a[every_other, true].inplace * 2 }
+  probe('elementwise', 'a + b transposed views', [c, r], 3 * n * es, ref: 'a + b contiguous') { a.transpose.inplace + b.transpose }
+  probe('elementwise', 'a.gt(b) to Bit', [r, c], 2 * n * es, ref: 'a + b contiguous') { a.gt(b) }
+  probe('elementwise', 'a.gt(row) to Bit', [r, c], n * es, ref: 'a + b contiguous') { a.gt(row) }
+end
+
+# --- copies -------------------------------------------------------------------
+#
+# A store from a view of every kind into contiguous memory, read against the
+# contiguous copy of the same bytes.
+def sweep_copy
+  n = ELEMENTS
+  es = elmsz
+  r = 1024
+  c = n / r
+  a = rand_array(r, c)
+  dst = DTYPE.new(r, c).fill(0)
+  dst_t = DTYPE.new(c, r).fill(0)
+  probe('copy', 'contiguous', [r, c], 2 * n * es) { dst.store(a) }
+  half = 0...(c / 2)
+  probe('copy', 'column slice', [r, c / 2], n * es, ref: 'contiguous') { dst[true, half].store(a[true, half]) }
+  every_other = (0..-1) % 2
+  probe('copy', 'stepped rows', [r / 2, c], n * es, ref: 'contiguous') { dst[every_other, true].store(a[every_other, true]) }
+  probe('copy', 'reversed rows', [r, c], 2 * n * es, ref: 'contiguous') { dst.store(a.reverse(0)) }
+  probe('copy', 'transposed', [c, r], 2 * n * es, ref: 'contiguous') { dst_t.store(a.transpose) }
+  idx = XM::Int32.new(r / 2).seq * 2
+  probe('copy', 'index-backed rows', [r / 2, c], n * es, ref: 'contiguous') { dst[0...(r / 2), true].store(a[idx, true]) }
+end
+
+# --- run ----------------------------------------------------------------------
+
+version = GPU ? Cumo::NArray::VERSION : Numo::NArray::VERSION
+puts "backend   : #{XM} #{version}"
+puts "dtype     : #{DTYPE}, #{ELEMENTS} elements (#{(ELEMENTS * elmsz / 1e6).round(1)} MB), #{LAUNCHES} launches per sync, best of #{ROUNDS}"
+warm(3)
+sweep_reduce
+sweep_elementwise
+sweep_copy
+
+refs = ROWS.to_h { |row| [row.key, row] }
+flagged = []
+puts
+puts format('  %-11s %-26s %-18s %9s %8s %8s', 'group', 'case', 'shape', 'us', 'GB/s', 'vs ref')
+ROWS.each do |row|
+  ref = refs["#{row.group}/#{row.ref}"]
+  ratio = ref ? ref.gbs / row.gbs : 1.0
+  slow = ratio > FLAG
+  flagged << row if slow
+  next unless slow || SHOW_ALL || row.ref == row.name
+
+  puts format('  %-11s %-26s %-18s %9.1f %8.1f %7.1fx%s', row.group, row.name, row.shape, row.us, row.gbs, ratio, slow ? '  <--' : '')
+end
+puts
+if flagged.empty?
+  puts format('  nothing was more than %.1fx off its reference', FLAG)
+else
+  puts format('  %d of %d cases more than %.1fx off their reference', flagged.size, ROWS.size, FLAG)
+end
+
+if COMPARE
+  before = JSON.parse(File.read(COMPARE))
+  puts
+  puts "  against #{File.basename(COMPARE)} (#{before['meta']['backend']} #{before['meta']['version']}, saved #{before['meta']['saved']})"
+  puts format('  %-11s %-26s %9s %9s %8s', 'group', 'case', 'before us', 'after us', 'speedup')
+  ROWS.each do |row|
+    was = before['rows'][row.key]
+    next unless was
+
+    speedup = was['us'] / row.us
+    next unless SHOW_ALL || speedup > 1.25 || speedup < 0.8
+
+    puts format('  %-11s %-26s %9.1f %9.1f %7.2fx', row.group, row.name, was['us'], row.us, speedup)
+  end
+end
+
+if JSON_OUT
+  out = { 'meta' => { 'backend' => XM.to_s, 'version' => version, 'dtype' => DTYPE.to_s, 'elements' => ELEMENTS,
+                      'saved' => Time.now.strftime('%Y-%m-%d %H:%M') },
+          'rows' => ROWS.to_h { |row| [row.key, { 'shape' => row.shape, 'us' => row.us, 'gbs' => row.gbs }] } }
+  File.write(JSON_OUT, JSON.pretty_generate(out))
+  puts "  wrote #{ROWS.size} rows to #{JSON_OUT}"
+end

data/ext/cumo/cuda/cudnn_impl.cpp

@@ -552,7 +552,15 @@ cumo_cuda_cudnn_GetBatchNormMode(size_t ndim, int* axis) {
     }
     if ((ndim == 3 && axis[0] == 0 && axis[1] == 2 && axis[2] == 3) ||
         (ndim == 4 && axis[0] == 0 && axis[1] == 2 && axis[2] == 3 && axis[3] == 4)) {  // (1, channels, (1, )1, 1)
-        // TODO: Consider CUDNN_BATCHNORM_SPATIAL_PERSISTENT if we can afford to check for overflow, with or without blocking.
+        // CUDNN_BATCHNORM_SPATIAL_PERSISTENT is not worth taking. The overflow
+        // check it asks for is affordable -- cudnnQueryRuntimeError costs 0.1us
+        // either way -- but the mode is 1.3 to 1.8x faster only where the
+        // spatial extent is large, and 1.6x slower on 32x256x14x14 backward.
+        // Worse, on 32x64x56x56 it raises CUDNN_STATUS_RUNTIME_FP_OVERFLOW for x
+        // in [-2, 2] with gamma 1 and beta 0, answering bit for bit what SPATIAL
+        // does; honouring that means running SPATIAL again on top. The mode, and
+        // the batch normalization API it belongs to, are deprecated as of cuDNN
+        // 9.0.0 besides.
         return CUDNN_BATCHNORM_SPATIAL;
     }
     rb_raise(rb_eRuntimeError, "Invalid axis for BatchNorm using cuDNN. Expected 1, 3 or 4 dimensions.");

data/ext/cumo/cuda/memory_pool_impl_test.cpp

@@ -337,7 +337,10 @@ public:
         assert(pool_->GetFreeBytes() == kRoundSize * 8);
     }
 
-    // TODO(sonots): Fix after implementing compaction
+    // RemoveFromFreeList leaves an emptied bin in the arena -- it erases the
+    // chunk and returns, where Malloc drops one through CompactIndex. The
+    // assertions below spell that out, so they describe the arena as it is
+    // rather than wait on a compaction this path is not getting.
     void TestRemoveFromFreeList() {
         Arena& arena = pool_->GetArena(stream_ptr_);
         ArenaIndexMap& arena_index_map = pool_->GetArenaIndexMap(stream_ptr_);

data/ext/cumo/cuda/nvrtc.c

@@ -63,7 +63,7 @@ push_strings(VALUE strings, VALUE ary, int n)
     int i;
 
     for (i = 0; i < n; i++) {
-        VALUE str = rb_str_to_str(RARRAY_AREF(ary, i));
+        VALUE str = rb_str_to_str(rb_ary_entry(ary, i));
         StringValueCStr(str);
         rb_ary_push(strings, str);
     }

data/ext/cumo/depend.erb

@@ -49,6 +49,13 @@ list_type_rb.each do |type_rb|
 
 src : <%= list_type_cu.join(" ") %> <%= list_type_c.join(" ") %>
 
+# mkmf writes no rule that makes an object depend on a header, so editing one
+# and rebuilding was a no-op that kept the previous cumo.so. Rebuilding all of
+# them is a minute and a half, so the list does not have to be per-object.
+CUMO_HDRS = <%= Dir.glob("#{__dir__}/**/*.{h,hpp}").sort.join(" ") %>
+
+$(OBJS): $(CUMO_HDRS)
+
 build-ctest : <%= __dir__ %>/cuda/memory_pool_impl_test.exe
 
 run-ctest : <%= __dir__ %>/cuda/memory_pool_impl_test.exe

data/ext/cumo/include/cumo/bit_reduce_kernel.h

@@ -7,6 +7,11 @@
 #include "cumo/indexer.h"
 #include "cumo/reduce_kernel.h"
 
+#define CUMO_BIT_STAT_MEAN   0
+#define CUMO_BIT_STAT_VAR    1
+#define CUMO_BIT_STAT_STDDEV 2
+#define CUMO_BIT_STAT_RMS    3
+
 namespace cumo_bit_detail {
 
 // A Bit reduction addresses its operands the way a numeric one does -- the
@@ -169,7 +174,7 @@ __device__ static inline uint64_t bit_count_axis(
 }
 
 __global__ static void bit_count_reduction_kernel(
-        cumo_na_bit_reduction_arg_t arg, cumo_detail::cumo_reduce_addr_t ad, cumo_bit_word_addr_t wa, int invert,
+        CUMO_GRID_CONSTANT cumo_na_bit_reduction_arg_t arg, CUMO_GRID_CONSTANT cumo_detail::cumo_reduce_addr_t ad, CUMO_GRID_CONSTANT cumo_bit_word_addr_t wa, int invert,
         int out_block_size, int reduce_block_size, int64_t unit_total_size) {
     extern __shared__ __align__(8) char sdata_raw[];
     uint64_t* sdata = reinterpret_cast<uint64_t*>(sdata_raw);
@@ -179,8 +184,8 @@ __global__ static void bit_count_reduction_kernel(
     int64_t out_total_size = arg.out_indexer.total_size;
     int64_t reduce_total_size = arg.in_indexer.total_size / out_total_size;
 
-    int64_t reduce_offset = tid / out_block_size;
-    int64_t out_offset = tid % out_block_size;
+    int64_t reduce_offset, out_offset;
+    cumo_detail::reduce_thread_split(ad, tid, out_block_size, reduce_block_size, &reduce_offset, &out_offset);
     int64_t out_base = blockIdx.x * out_block_size;
     int64_t out_stride = gridDim.x * out_block_size;
 
@@ -191,7 +196,7 @@ __global__ static void bit_count_reduction_kernel(
         uint64_t accum = bit_count_axis(arg, ad, wa, invert, in_out_off, i_in, reduce_total_size,
                                         0, unit_total_size, reduce_offset, reduce_block_size);
 
-        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, impl);
+        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, reduce_block_size, !ad.out_inner, impl);
         if (reduce_offset == 0) {
             *reinterpret_cast<uint64_t*>(arg.out.ptr + bit_out_offset(arg, ad, i_out)) = accum;
         }
@@ -202,7 +207,7 @@ __global__ static void bit_count_reduction_kernel(
 // grid a handful of blocks however long the reduce axis is.
 template <typename TArg>
 __global__ static void bit_count_partial_kernel(
-        TArg arg, cumo_detail::cumo_reduce_addr_t ad, cumo_bit_word_addr_t wa, int invert,
+        CUMO_GRID_CONSTANT TArg arg, CUMO_GRID_CONSTANT cumo_detail::cumo_reduce_addr_t ad, CUMO_GRID_CONSTANT cumo_bit_word_addr_t wa, int invert,
         uint64_t* partial, int64_t n_split, int64_t chunk,
         int out_block_size, int reduce_block_size, int64_t unit_total_size) {
     extern __shared__ __align__(8) char sdata_raw[];
@@ -214,8 +219,8 @@ __global__ static void bit_count_partial_kernel(
     int64_t reduce_total_size = arg.in_indexer.total_size / out_total_size;
     int64_t partial_total_size = out_total_size * n_split;
 
-    int64_t reduce_offset = tid / out_block_size;
-    int64_t out_offset = tid % out_block_size;
+    int64_t reduce_offset, out_offset;
+    cumo_detail::reduce_thread_split(ad, tid, out_block_size, reduce_block_size, &reduce_offset, &out_offset);
     int64_t out_base = blockIdx.x * out_block_size;
     int64_t out_stride = gridDim.x * out_block_size;
 
@@ -231,7 +236,7 @@ __global__ static void bit_count_partial_kernel(
         uint64_t accum = bit_count_axis(arg, ad, wa, invert, in_out_off, i_in, reduce_total_size,
                                         begin, end, reduce_offset, reduce_block_size);
 
-        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, impl);
+        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, reduce_block_size, !ad.out_inner, impl);
         if (reduce_offset == 0) {
             partial[i_out * n_split + i_split] = accum;
         }
@@ -287,6 +292,71 @@ static inline bit_reduce_plan make_bit_reduce_plan(const TArg& arg) {
     return p;
 }
 
+// mean, var, stddev and rms of a bit array are functions of how many bits are
+// set and how long the axis is, because every element is 0 or 1. They reduce
+// the way count does and differ only in what the store computes.
+__device__ static inline double bit_stat_of_count(uint64_t count, int64_t reduce_total_size, int stat) {
+    double k = (double)count;
+    double n = (double)reduce_total_size;
+    double mean = k / n;
+    switch (stat) {
+        case CUMO_BIT_STAT_MEAN: return mean;
+        case CUMO_BIT_STAT_RMS: return sqrt(mean);
+        // x squared is x for a bit, so the squared deviations sum to k - k * mean.
+        case CUMO_BIT_STAT_VAR: return (k - k * mean) / (n - 1);
+        default: return sqrt((k - k * mean) / (n - 1));
+    }
+}
+
+__global__ static void bit_stat_reduction_kernel(
+        CUMO_GRID_CONSTANT cumo_na_bit_reduction_arg_t arg, CUMO_GRID_CONSTANT cumo_detail::cumo_reduce_addr_t ad, CUMO_GRID_CONSTANT cumo_bit_word_addr_t wa, int stat,
+        int out_block_size, int reduce_block_size, int64_t unit_total_size) {
+    extern __shared__ __align__(8) char sdata_raw[];
+    uint64_t* sdata = reinterpret_cast<uint64_t*>(sdata_raw);
+    unsigned int tid = threadIdx.x;
+    BitCountImpl impl;
+
+    int64_t out_total_size = arg.out_indexer.total_size;
+    int64_t reduce_total_size = arg.in_indexer.total_size / out_total_size;
+
+    int64_t reduce_offset, out_offset;
+    cumo_detail::reduce_thread_split(ad, tid, out_block_size, reduce_block_size, &reduce_offset, &out_offset);
+    int64_t out_base = blockIdx.x * out_block_size;
+    int64_t out_stride = gridDim.x * out_block_size;
+
+    for (int64_t i_out = out_base + out_offset; i_out < out_total_size; i_out += out_stride) {
+        ssize_t in_out_off = bit_in_out_offset(arg, ad, i_out);
+        int64_t i_in = i_out * reduce_total_size + reduce_offset;
+
+        uint64_t accum = bit_count_axis(arg, ad, wa, 0, in_out_off, i_in, reduce_total_size,
+                                        0, unit_total_size, reduce_offset, reduce_block_size);
+
+        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, reduce_block_size, !ad.out_inner, impl);
+        if (reduce_offset == 0) {
+            *reinterpret_cast<double*>(arg.out.ptr + bit_out_offset(arg, ad, i_out)) =
+                bit_stat_of_count(accum, reduce_total_size, stat);
+        }
+    }
+}
+
+// Second pass of a split statistic, laid out like bit_pred_combine_kernel.
+__global__ static void bit_stat_combine_kernel(
+        CUMO_GRID_CONSTANT cumo_na_bit_reduction_arg_t arg, CUMO_GRID_CONSTANT cumo_detail::cumo_reduce_addr_t ad, int stat,
+        const uint64_t* partial, int64_t n_split) {
+    int64_t out_total_size = arg.out_indexer.total_size;
+    int64_t reduce_total_size = arg.in_indexer.total_size / out_total_size;
+
+    for (int64_t i_out = blockIdx.x * blockDim.x + threadIdx.x; i_out < out_total_size;
+         i_out += (int64_t)blockDim.x * gridDim.x) {
+        uint64_t count = 0;
+        for (int64_t i_split = 0; i_split < n_split; ++i_split) {
+            count += partial[i_out * n_split + i_split];
+        }
+        *reinterpret_cast<double*>(arg.out.ptr + bit_out_offset(arg, ad, i_out)) =
+            bit_stat_of_count(count, reduce_total_size, stat);
+    }
+}
+
 // all? and any? are the count of set bits against the length of the axis and
 // against zero, so they reduce the same way and differ only here.
 __device__ static inline CUMO_BIT_DIGIT bit_pred_of_count(uint64_t count, int64_t reduce_total_size, int all) {
@@ -294,7 +364,7 @@ __device__ static inline CUMO_BIT_DIGIT bit_pred_of_count(uint64_t count, int64_
 }
 
 __global__ static void bit_pred_reduction_kernel(
-        cumo_na_bit_pred_reduction_arg_t arg, cumo_detail::cumo_reduce_addr_t ad, cumo_bit_word_addr_t wa, int all,
+        CUMO_GRID_CONSTANT cumo_na_bit_pred_reduction_arg_t arg, CUMO_GRID_CONSTANT cumo_detail::cumo_reduce_addr_t ad, CUMO_GRID_CONSTANT cumo_bit_word_addr_t wa, int all,
         int out_block_size, int reduce_block_size, int64_t unit_total_size) {
     extern __shared__ __align__(8) char sdata_raw[];
     uint64_t* sdata = reinterpret_cast<uint64_t*>(sdata_raw);
@@ -304,8 +374,8 @@ __global__ static void bit_pred_reduction_kernel(
     int64_t out_total_size = arg.out_indexer.total_size;
     int64_t reduce_total_size = arg.in_indexer.total_size / out_total_size;
 
-    int64_t reduce_offset = tid / out_block_size;
-    int64_t out_offset = tid % out_block_size;
+    int64_t reduce_offset, out_offset;
+    cumo_detail::reduce_thread_split(ad, tid, out_block_size, reduce_block_size, &reduce_offset, &out_offset);
     int64_t out_base = blockIdx.x * out_block_size;
     int64_t out_stride = gridDim.x * out_block_size;
 
@@ -316,7 +386,7 @@ __global__ static void bit_pred_reduction_kernel(
         uint64_t accum = bit_count_axis(arg, ad, wa, 0, in_out_off, i_in, reduce_total_size,
                                         0, unit_total_size, reduce_offset, reduce_block_size);
 
-        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, impl);
+        accum = cumo_detail::reduce_in_block(accum, sdata, tid, out_block_size, reduce_block_size, !ad.out_inner, impl);
         if (reduce_offset == 0) {
             size_t pos = (size_t)((ssize_t)arg.out.pos + bit_out_offset(arg, ad, i_out));
             CUMO_STORE_BIT(arg.out.ptr, pos, bit_pred_of_count(accum, reduce_total_size, all));
@@ -327,7 +397,7 @@ __global__ static void bit_pred_reduction_kernel(
 // Second pass of a split all? or any?. There are few outputs whenever the axis
 // was worth splitting, so one thread per output walks its chunks.
 __global__ static void bit_pred_combine_kernel(
-        cumo_na_bit_pred_reduction_arg_t arg, cumo_detail::cumo_reduce_addr_t ad, int all,
+        CUMO_GRID_CONSTANT cumo_na_bit_pred_reduction_arg_t arg, CUMO_GRID_CONSTANT cumo_detail::cumo_reduce_addr_t ad, int all,
         const uint64_t* partial, int64_t n_split) {
     int64_t out_total_size = arg.out_indexer.total_size;
     int64_t reduce_total_size = arg.in_indexer.total_size / out_total_size;
@@ -387,6 +457,42 @@ static inline void cumo_bit_count_reduce(cumo_na_bit_reduction_arg_t arg, int in
     cumo_cuda_runtime_free(reinterpret_cast<char*>(partial));
 }
 
+// The same for the four statistics, whose result is a DFloat of the same width
+// as the count it is computed from.
+static inline void cumo_bit_stat_reduce(cumo_na_bit_reduction_arg_t arg, int stat) {
+    if (arg.out_indexer.total_size == 0) {
+        return;
+    }
+
+    cumo_bit_detail::bit_reduce_plan p = cumo_bit_detail::make_bit_reduce_plan(arg);
+    int64_t block_size = cumo_detail::max_block_size;
+    int64_t shared_mem_size = sizeof(uint64_t) * block_size;
+
+    if (p.n_split < 2) {
+        int64_t grid_size = std::min(cumo_detail::max_grid_size, p.out_block_num);
+        cumo_bit_detail::bit_stat_reduction_kernel<<<grid_size, block_size, shared_mem_size>>>(
+            arg, p.ad, p.wa, stat, (int)p.out_block_size, (int)p.reduce_block_size, p.unit_total_size);
+        cumo_cuda_runtime_check_kernel_launch();
+        return;
+    }
+
+    int64_t partial_total_size = p.out_total_size * p.n_split;
+    uint64_t* partial = reinterpret_cast<uint64_t*>(cumo_cuda_runtime_malloc(sizeof(uint64_t) * partial_total_size));
+
+    int64_t grid_size = std::min(cumo_detail::max_grid_size, p.partial_block_num);
+    cumo_bit_detail::bit_count_partial_kernel<<<grid_size, block_size, shared_mem_size>>>(
+        arg, p.ad, p.wa, 0, partial, p.n_split, p.chunk,
+        (int)p.split_out_block_size, (int)p.split_reduce_block_size, p.unit_total_size);
+    cumo_cuda_runtime_check_kernel_launch();
+
+    int64_t combine_grid = (p.out_total_size + block_size - 1) / block_size;
+    if (combine_grid > cumo_detail::max_grid_size) combine_grid = cumo_detail::max_grid_size;
+    cumo_bit_detail::bit_stat_combine_kernel<<<combine_grid, block_size>>>(arg, p.ad, stat, partial, p.n_split);
+    cumo_cuda_runtime_check_kernel_launch();
+
+    cumo_cuda_runtime_free(reinterpret_cast<char*>(partial));
+}
+
 // The same for all? (all non-zero) and any? (any non-zero), whose result is a
 // Bit rather than a count.
 static inline void cumo_bit_pred_reduce(cumo_na_bit_pred_reduction_arg_t arg, int all) {

data/ext/cumo/include/cumo/check.h

@@ -0,0 +1,51 @@
+#ifndef CUMO_CHECK_H
+#define CUMO_CHECK_H
+
+#include <ruby.h>
+#include "cumo/narray.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#if 0
+} /* satisfy cc-mode */
+#endif
+#endif
+
+#define CUMO_CHECK_NARRAY_TYPE(x,t)                            \
+    if (rb_obj_class(x)!=(t)) {                                \
+        rb_raise(rb_eTypeError,"invalid NArray type (class)"); \
+    }
+
+#define CUMO_CHECK_SIZE_EQ(sz1,sz2)                  \
+    if ((sz1) != (sz2)) {                            \
+        rb_raise(cumo_na_eShapeError,                \
+                 "size mismatch: %d != %d",          \
+                 (int)(sz1), (int)(sz2));            \
+    }
+
+#define CUMO_CHECK_DIM_EQ(nd1,nd2)                   \
+    if ((nd1) != (nd2)) {                            \
+        rb_raise(cumo_na_eShapeError,                \
+                 "dimension mismatch: %d != %d",     \
+                 (int)(nd1), (int)(nd2));            \
+    }
+
+static inline VALUE
+cumo_option_value(VALUE value, VALUE default_value)
+{
+    switch(TYPE(value)) {
+    case T_NIL:
+    case T_UNDEF:
+        return default_value;
+    }
+    return value;
+}
+
+#if defined(__cplusplus)
+#if 0
+{ /* satisfy cc-mode */
+#endif
+}  /* extern "C" { */
+#endif
+
+#endif /* CUMO_CHECK_H */