cumo 0.7.0 → 0.9.0

data/ext/cumo/cuda/memory_pool_impl.cpp

@@ -108,7 +108,16 @@ bool SingleDeviceMemoryPool::RemoveFromFreeList(size_t size, std::shared_ptr<Chu
     }
     assert(arena.size() > arena_index);
     FreeList& free_list = arena[arena_index];
-    return EraseFromFreeList(free_list, chunk);
+    if (!EraseFromFreeList(free_list, chunk)) {
+        return false;
+    }
+    // A bin left empty here would be walked by every later search, and the
+    // merge path below Free empties one whenever it takes a neighbour out.
+    if (free_list.empty()) {
+        arena.erase(arena.begin() + arena_index);
+        arena_index_map.erase(arena_index_map.begin() + arena_index);
+    }
+    return true;
 }
 
 intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
@@ -132,6 +141,7 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
 
         // find best-fit, or a smallest larger allocation
         Arena& arena = GetArena(stream_ptr);
+        ArenaIndexMap& arena_index_map = GetArenaIndexMap(stream_ptr);
         size_t arena_index = GetArenaIndex(size, stream_ptr);
         size_t arena_length = arena.size();
         for (size_t i = arena_index; i < arena_length; ++i) {
@@ -141,10 +151,15 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
             }
             chunk = PopFromFreeList(free_list);
             if (free_list.empty()) {
-                // An emptied bin stays in the arena otherwise, and every later
-                // search walks it. Dropping it here invalidates arena and
-                // free_list, so nothing below this loop may touch them.
-                CompactIndex(stream_ptr, false);
+                // Drop just this bin. Rebuilding the whole arena instead
+                // copied every other bin's free list, and a free list holds
+                // shared pointers, so a fragmented pool touched the reference
+                // count of every chunk it held on every allocation.
+                // This leaves free_list, the elements of arena_index_map and
+                // every iterator from i on dangling, so nothing below may
+                // touch them.
+                arena.erase(arena.begin() + i);
+                arena_index_map.erase(arena_index_map.begin() + i);
             }
             break;
         }
@@ -252,7 +267,7 @@ bool SingleDeviceMemoryPool::Free(intptr_t ptr) {
     return true;
 }
 
-void SingleDeviceMemoryPool::CompactIndex(cudaStream_t stream_ptr, bool free) {
+void SingleDeviceMemoryPool::CompactIndex(cudaStream_t stream_ptr) {
     // need lock ouside this function
     if (!HasArena(stream_ptr)) return;
 
@@ -266,22 +281,17 @@ void SingleDeviceMemoryPool::CompactIndex(cudaStream_t stream_ptr, bool free) {
         if (free_list.empty()) {
             continue;
         }
-        if (free) {
-            FreeList keep_list;
-            for (auto chunk : free_list) {
-                if (chunk->prev() != nullptr || chunk->next() != nullptr) {
-                    keep_list.emplace_back(chunk);
-                }
-            }
-            if (keep_list.size() == 0) {
-                continue;
+        FreeList keep_list;
+        for (auto chunk : free_list) {
+            if (chunk->prev() != nullptr || chunk->next() != nullptr) {
+                keep_list.emplace_back(chunk);
             }
-            new_arena_index_map.emplace_back(arena_index_map[arena_index]);
-            new_arena.emplace_back(keep_list);
-        } else {
-            new_arena_index_map.emplace_back(arena_index_map[arena_index]);
-            new_arena.emplace_back(free_list);
         }
+        if (keep_list.size() == 0) {
+            continue;
+        }
+        new_arena_index_map.emplace_back(arena_index_map[arena_index]);
+        new_arena.emplace_back(keep_list);
     }
     if (new_arena.empty()) {
         index_.erase(stream_ptr);
@@ -299,7 +309,7 @@ void SingleDeviceMemoryPool::FreeAllBlocks() {
     std::vector<cudaStream_t> keys(free_.size());
     transform(free_.begin(), free_.end(), keys.begin(), [](auto pair) { return pair.first; });
     for (cudaStream_t stream_ptr : keys) {
-        CompactIndex(stream_ptr, true);
+        CompactIndex(stream_ptr);
     }
 }
 
@@ -307,7 +317,7 @@ void SingleDeviceMemoryPool::FreeAllBlocks() {
 void SingleDeviceMemoryPool::FreeAllBlocks(cudaStream_t stream_ptr) {
     std::lock_guard<std::recursive_mutex> lock{mutex_};
 
-    CompactIndex(stream_ptr, true);
+    CompactIndex(stream_ptr);
 }
 
 size_t SingleDeviceMemoryPool::GetNumFreeBlocks() {

data/ext/cumo/cuda/memory_pool_impl.hpp

@@ -284,7 +284,9 @@ public:
     //         be found in the free list as the chunk is allocated.)
     bool RemoveFromFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr);
 
-    void CompactIndex(cudaStream_t stream_ptr, bool free);
+    // Drops every chunk that was never split from a larger one, and every
+    // bin left empty by that.
+    void CompactIndex(cudaStream_t stream_ptr);
 };
 
 // Memory pool for all GPU devices on the host.

data/ext/cumo/cuda/memory_pool_impl_test.cpp

@@ -337,10 +337,10 @@ public:
         assert(pool_->GetFreeBytes() == kRoundSize * 8);
     }
 
-    // RemoveFromFreeList leaves an emptied bin in the arena -- it erases the
-    // chunk and returns, where Malloc drops one through CompactIndex. The
-    // assertions below spell that out, so they describe the arena as it is
-    // rather than wait on a compaction this path is not getting.
+    // RemoveFromFreeList drops a bin its erase leaves empty, the same as
+    // Malloc. The merge path below Free empties one whenever it takes a
+    // neighbour out, so a bin left behind here would be walked by every later
+    // search.
     void TestRemoveFromFreeList() {
         Arena& arena = pool_->GetArena(stream_ptr_);
         ArenaIndexMap& arena_index_map = pool_->GetArenaIndexMap(stream_ptr_);
@@ -372,36 +372,30 @@ public:
         assert(arena_index_map[1] == 3);
         assert(arena_index_map[2] == 4);
 
-        // remove two from two
+        // remove two from two, which empties that bin
         pool_->RemoveFromFreeList(chunk2->size(), chunk2, stream_ptr_);
-        assert(arena.size() == 3);
+        assert(arena.size() == 2);
         assert(arena[0].size() == 1);
-        assert(arena[1].size() == 0);
-        assert(arena[2].size() == 1);
-        assert(arena_index_map.size() == 3);
+        assert(arena[1].size() == 1);
+        assert(arena_index_map.size() == 2);
         assert(arena_index_map[0] == 2);
-        assert(arena_index_map[1] == 3);
-        assert(arena_index_map[2] == 4);
+        assert(arena_index_map[1] == 4);
 
         pool_->RemoveFromFreeList(chunk3->size(), chunk3, stream_ptr_);
-        assert(arena.size() == 3);
+        assert(arena.size() == 1);
         assert(arena[0].size() == 1);
-        assert(arena[1].size() == 0);
-        assert(arena[2].size() == 0);
-        assert(arena_index_map.size() == 3);
+        assert(arena_index_map.size() == 1);
         assert(arena_index_map[0] == 2);
-        assert(arena_index_map[1] == 3);
-        assert(arena_index_map[2] == 4);
 
         pool_->RemoveFromFreeList(chunk4->size(), chunk4, stream_ptr_);
-        assert(arena.size() == 3);
-        assert(arena[0].size() == 0);
-        assert(arena[1].size() == 0);
-        assert(arena[2].size() == 0);
-        assert(arena_index_map.size() == 3);
-        assert(arena_index_map[0] == 2);
-        assert(arena_index_map[1] == 3);
-        assert(arena_index_map[2] == 4);
+        assert(arena.size() == 0);
+        assert(arena_index_map.size() == 0);
+
+        // A chunk that is not in the free list leaves the arena alone.
+        pool_->AppendToFreeList(chunk1->size(), chunk1, stream_ptr_);
+        assert(!pool_->RemoveFromFreeList(chunk2->size(), chunk2, stream_ptr_));
+        assert(arena.size() == 1);
+        assert(arena[0].size() == 1);
     }
 
     void TestMalloc() {

data/ext/cumo/cuda/runtime.c

@@ -2,6 +2,8 @@
 #include <assert.h>
 #include <cuda_runtime.h>
 #include "cumo/cuda/runtime.h"
+#include "cumo/cuda/memory_pool.h"
+#include "cumo/template_kernel.h"
 
 VALUE cumo_cuda_eRuntimeError;
 VALUE cumo_cuda_mRuntime;
@@ -20,6 +22,60 @@ cumo_cuda_runtime_check_kernel_launch(void)
     check_status(cudaGetLastError());
 }
 
+// A pointer given twice would be freed twice, and the second free would reach
+// past the pool to a block it still owns, so each one goes back once.
+static void
+release_held(char *p[5])
+{
+    int i, j;
+    for (i = 0; i < 5; ++i) {
+        if (p[i] == NULL) { continue; }
+        for (j = 0; j < i; ++j) {
+            if (p[j] == p[i]) { break; }
+        }
+        if (j == i) { cumo_cuda_runtime_free_no_raise(p[i]); }
+    }
+}
+
+// What a launcher holding scratch has to call instead of the plain check, for a
+// status it already took out of the slot: a library call in the window takes it
+// first otherwise, and whoever reads it first is the only one who sees it.
+// Raising is a longjmp, so the frees written below the check never run, and
+// unwinding frees through the form that cannot raise so one failure cannot hide
+// another.
+void
+cumo_cuda_runtime_check_taken_status_holding(int status, char *p0, char *p1, char *p2, char *p3, char *p4)
+{
+    if ((cudaError_t)status != cudaSuccess) {
+        char *held[5];
+        held[0] = p0; held[1] = p1; held[2] = p2; held[3] = p3; held[4] = p4;
+        release_held(held);
+    }
+    check_status((cudaError_t)status);
+}
+
+// The slot is read once here rather than peeked and read again: the frees below
+// can put a status of their own in it, and the launch is the one to report.
+void
+cumo_cuda_runtime_check_kernel_launch_holding(char *p0, char *p1, char *p2, char *p3, char *p4)
+{
+    cumo_cuda_runtime_check_taken_status_holding((int)cudaGetLastError(), p0, p1, p2, p3, p4);
+}
+
+// How scratch goes back from a handler that may be unwinding: a free that
+// raises there replaces the exception being carried, and leaves every free
+// below it unrun. wait_for_stream asks for the wait a caller needs when the
+// buffer is still being read, since the pool hands a freed chunk straight out.
+void
+cumo_cuda_runtime_return_scratch(char *ptr, int wait_for_stream, cudaError_t *status)
+{
+    if (wait_for_stream) {
+        cudaError_t wait = cudaStreamSynchronize(0);
+        if (status != NULL && *status == cudaSuccess) { *status = wait; }
+    }
+    if (ptr != NULL) { cumo_cuda_runtime_free_no_raise(ptr); }
+}
+
 int*
 cumo_cuda_runtime_error_flag_ptr(void)
 {

data/ext/cumo/cumo.c

@@ -21,6 +21,7 @@ void Init_cumo_uint16();
 void Init_cumo_uint32();
 void Init_cumo_uint64();
 void Init_cumo_hfloat();
+void Init_cumo_bfloat();
 void Init_cumo_sfloat();
 void Init_cumo_scomplex();
 void Init_cumo_dfloat();
@@ -31,6 +32,7 @@ void Init_cumo_dfloat_upcast();
 void Init_cumo_scomplex_upcast();
 void Init_cumo_sfloat_upcast();
 void Init_cumo_hfloat_upcast();
+void Init_cumo_bfloat_upcast();
 void Init_cumo_int64_upcast();
 void Init_cumo_uint64_upcast();
 void Init_cumo_int32_upcast();
@@ -167,6 +169,7 @@ Init_cumo()
     Init_cumo_scomplex();
     Init_cumo_sfloat();
     Init_cumo_hfloat();
+    Init_cumo_bfloat();
 
     Init_cumo_int64();
     Init_cumo_uint64();
@@ -186,6 +189,7 @@ Init_cumo()
     Init_cumo_scomplex_upcast();
     Init_cumo_sfloat_upcast();
     Init_cumo_hfloat_upcast();
+    Init_cumo_bfloat_upcast();
     Init_cumo_int64_upcast();
     Init_cumo_uint64_upcast();
     Init_cumo_int32_upcast();

data/ext/cumo/extconf.rb

@@ -65,6 +65,7 @@ narray/types/uint16
 narray/types/uint32
 narray/types/uint64
 narray/types/hfloat
+narray/types/bfloat
 narray/types/sfloat
 narray/types/dfloat
 narray/types/scomplex
@@ -80,6 +81,7 @@ narray/types/uint16_kernel
 narray/types/uint32_kernel
 narray/types/uint64_kernel
 narray/types/hfloat_kernel
+narray/types/bfloat_kernel
 narray/types/sfloat_kernel
 narray/types/dfloat_kernel
 narray/types/scomplex_kernel

data/ext/cumo/include/cumo/bit_reduce_kernel.h

@@ -442,7 +442,7 @@ static inline void cumo_bit_count_reduce(cumo_na_bit_reduction_arg_t arg, int in
     cumo_bit_detail::bit_count_partial_kernel<<<grid_size, block_size, shared_mem_size>>>(
         arg, p.ad, p.wa, invert, partial, p.n_split, p.chunk,
         (int)p.split_out_block_size, (int)p.split_reduce_block_size, p.unit_total_size);
-    cumo_cuda_runtime_check_kernel_launch();
+    cumo_check_launch_holding(partial);
 
     cumo_na_reduction_arg_t combine;
     combine.in.ptr = reinterpret_cast<char*>(partial);
@@ -452,7 +452,7 @@ static inline void cumo_bit_count_reduce(cumo_na_bit_reduction_arg_t arg, int in
     combine.in_indexer.total_size = partial_total_size;
     combine.out = arg.out;
     combine.out_indexer = arg.out_indexer;
-    cumo_reduce<uint64_t, uint64_t>(combine, cumo_bit_detail::BitCountImpl{});
+    cumo_reduce<uint64_t, uint64_t>(combine, cumo_bit_detail::BitCountImpl{}, reinterpret_cast<char*>(partial));
 
     cumo_cuda_runtime_free(reinterpret_cast<char*>(partial));
 }
@@ -483,12 +483,12 @@ static inline void cumo_bit_stat_reduce(cumo_na_bit_reduction_arg_t arg, int sta
     cumo_bit_detail::bit_count_partial_kernel<<<grid_size, block_size, shared_mem_size>>>(
         arg, p.ad, p.wa, 0, partial, p.n_split, p.chunk,
         (int)p.split_out_block_size, (int)p.split_reduce_block_size, p.unit_total_size);
-    cumo_cuda_runtime_check_kernel_launch();
+    cumo_check_launch_holding(partial);
 
     int64_t combine_grid = (p.out_total_size + block_size - 1) / block_size;
     if (combine_grid > cumo_detail::max_grid_size) combine_grid = cumo_detail::max_grid_size;
     cumo_bit_detail::bit_stat_combine_kernel<<<combine_grid, block_size>>>(arg, p.ad, stat, partial, p.n_split);
-    cumo_cuda_runtime_check_kernel_launch();
+    cumo_check_launch_holding(partial);
 
     cumo_cuda_runtime_free(reinterpret_cast<char*>(partial));
 }
@@ -519,12 +519,12 @@ static inline void cumo_bit_pred_reduce(cumo_na_bit_pred_reduction_arg_t arg, in
     cumo_bit_detail::bit_count_partial_kernel<<<grid_size, block_size, shared_mem_size>>>(
         arg, p.ad, p.wa, 0, partial, p.n_split, p.chunk,
         (int)p.split_out_block_size, (int)p.split_reduce_block_size, p.unit_total_size);
-    cumo_cuda_runtime_check_kernel_launch();
+    cumo_check_launch_holding(partial);
 
     int64_t combine_grid = (p.out_total_size + block_size - 1) / block_size;
     if (combine_grid > cumo_detail::max_grid_size) combine_grid = cumo_detail::max_grid_size;
     cumo_bit_detail::bit_pred_combine_kernel<<<combine_grid, block_size>>>(arg, p.ad, all, partial, p.n_split);
-    cumo_cuda_runtime_check_kernel_launch();
+    cumo_check_launch_holding(partial);
 
     cumo_cuda_runtime_free(reinterpret_cast<char*>(partial));
 }

data/ext/cumo/include/cumo/check.h

@@ -11,10 +11,18 @@ extern "C" {
 #endif
 #endif
 
-#define CUMO_CHECK_NARRAY_TYPE(x,t)                            \
-    if (rb_obj_class(x)!=(t)) {                                \
-        rb_raise(rb_eTypeError,"invalid NArray type (class)"); \
+// Two of these often sit side by side, so the message has to say which array
+// was turned down as well as what it is.
+static inline void
+cumo_na_check_narray_type(VALUE x, VALUE type, const char *name)
+{
+    if (rb_obj_class(x) != type) {
+        rb_raise(rb_eTypeError, "%s must be %" PRIsVALUE ", not %" PRIsVALUE,
+                 name, type, rb_obj_class(x));
     }
+}
+
+#define CUMO_CHECK_NARRAY_TYPE(x,t,name) cumo_na_check_narray_type((x),(t),(name))
 
 #define CUMO_CHECK_SIZE_EQ(sz1,sz2)                  \
     if ((sz1) != (sz2)) {                            \

data/ext/cumo/include/cumo/cuda/cudnn.h

@@ -38,7 +38,7 @@ cumo_cuda_cudnn_check_output(VALUE out, VALUE type, size_t ndim, size_t *shape)
 {
     cumo_narray_t *na;
 
-    CUMO_CHECK_NARRAY_TYPE(out, type);
+    CUMO_CHECK_NARRAY_TYPE(out, type, "output");
     CumoGetNArray(out, na);
     CUMO_CHECK_DIM_EQ((size_t)(na->ndim), ndim);
     for (size_t idim = 0; idim < ndim; ++idim) {
@@ -75,7 +75,7 @@ cumo_cuda_cudnn_check_input(VALUE in, VALUE type, size_t ndim, size_t *shape)
 {
     cumo_narray_t *na;
 
-    CUMO_CHECK_NARRAY_TYPE(in, type);
+    CUMO_CHECK_NARRAY_TYPE(in, type, "input");
     CumoGetNArray(in, na);
     CUMO_CHECK_DIM_EQ((size_t)(na->ndim), ndim);
     for (size_t idim = 0; idim < ndim; ++idim) {
@@ -88,18 +88,48 @@ cumo_cuda_cudnn_check_input(VALUE in, VALUE type, size_t ndim, size_t *shape)
 static inline void
 cumo_cuda_cudnn_check_param_type(VALUE param, VALUE type, const char* name)
 {
-    if (rb_obj_class(param) != type) {
-        rb_raise(rb_eTypeError, "%s must be %s, not %s",
-                 name, rb_class2name(type), rb_obj_classname(param));
-    }
+    CUMO_CHECK_NARRAY_TYPE(param, type, name);
 }
 
 void
 cumo_cuda_cudnn_check_status(cudnnStatus_t status);
 
+// Raises, so the methods that clean up through a label take it before they
+// hold a descriptor: a raise runs no label.
 cudnnHandle_t
 cumo_cuda_cudnn_handle();
 
+// What a convolution holds while it works. A raise runs no error label, so the
+// work goes under rb_ensure and the handler gives these back instead.
+typedef struct {
+    cudnnTensorDescriptor_t      x_desc;
+    cudnnTensorDescriptor_t      y_desc;
+    cudnnTensorDescriptor_t      b_desc;
+    cudnnFilterDescriptor_t      w_desc;
+    cudnnConvolutionDescriptor_t conv_desc;
+    char                        *workspace;
+    cudaError_t                  wait_status;
+} cumo_cuda_cudnn_conv_held_t;
+
+// Takes the held as its argument so it can be handed to rb_ensure as it is.
+VALUE
+cumo_cuda_cudnn_release_conv_held(VALUE held);
+
+// What conv and conv_transpose carry into the run the ensure above guards.
+typedef struct {
+    cumo_cuda_cudnn_conv_held_t held;
+    cudnnHandle_t handle;
+    VALUE   x_cont, w_cont, y, b_cont;
+    char   *x_cont_ptr, *w_cont_ptr, *y_ptr, *b_cont_ptr;
+    size_t  ndim;
+    int    *int_stride, *int_pad;
+    cudnnStatus_t status;
+} cumo_cuda_cudnn_conv_run_t;
+
+// Adds nothing when the run carries no bias, so callers need no guard.
+cudnnStatus_t
+cumo_cuda_cudnn_AddBias(cumo_cuda_cudnn_conv_run_t *r, cudnnDataType_t dtype, const void *one);
+
 // VALUE is Ruby Array
 static inline void
 cumo_cuda_cudnn_get_int_ary(int* int_ary, VALUE ary, size_t ndim, int default_value)
@@ -178,6 +208,13 @@ cumo_cuda_cudnn_CreateTensorDescriptor(
         VALUE a,
         cudnnDataType_t cudnn_dtype);
 
+cudnnStatus_t
+cumo_cuda_cudnn_CreateTensorDescriptorFromShape(
+        cudnnTensorDescriptor_t *desc,
+        int ndim,
+        size_t *shape,
+        cudnnDataType_t cudnn_dtype);
+
 cudnnStatus_t
 cumo_cuda_cudnn_CreateFilterDescriptor(
         cudnnFilterDescriptor_t *desc,

data/ext/cumo/include/cumo/cuda/runtime.h

@@ -28,6 +28,8 @@ cumo_cuda_runtime_check_status(cudaError_t status)
     }
 }
 
+void cumo_cuda_runtime_return_scratch(char *ptr, int wait_for_stream, cudaError_t *status);
+
 static inline void
 cumo_cuda_runtime_device_synchronize(void)
 {

data/ext/cumo/include/cumo/indexer.h

@@ -176,6 +176,22 @@ cumo_na_make_iarray_stridx(cumo_na_loop_args_t* arg)
     return iarray;
 }
 
+// Whether any argument of the loop is backed by an index array. With
+// INDEX_LOOP an operand can arrive carrying one, and cumo_na_iarray_t holds
+// byte steps only: an indexed dimension has a step of zero there, so a loop
+// that answers yes has to address its operands through the stridx form above.
+static int
+cumo_na_loop_has_index(cumo_na_loop_t *const lp)
+{
+    int j, i;
+    for (j = 0; j < lp->narg; ++j) {
+        for (i = 0; i < lp->args[j].ndim; ++i) {
+            if (lp->args[j].iter[i].idx) return 1;
+        }
+    }
+    return 0;
+}
+
 static cumo_na_bit_iarray_stridx_t
 cumo_na_make_bit_iarray_stridx(cumo_na_loop_args_t* arg)
 {

data/ext/cumo/include/cumo/intern.h

@@ -54,6 +54,8 @@ void cumo_na_release_lock(VALUE); // currently do nothing
 // used in reduce methods
 VALUE cumo_na_reduce_dimension(int argc, VALUE *argv, int naryc, VALUE *naryv,
                             cumo_ndfunc_t *ndf, cumo_na_iter_func_t nan_iter);
+VALUE cumo_na_get_reduce(VALUE v);
+void cumo_na_set_reduce(VALUE v, VALUE reduce);
 
 VALUE cumo_na_reduce_options(VALUE axes, VALUE *opts, int naryc, VALUE *naryv,
                           cumo_ndfunc_t *ndf);
@@ -66,6 +68,15 @@ VALUE cumo_na_ndloop4(cumo_ndfunc_t *nf, void *ptr, VALUE args);
 
 VALUE cumo_na_ndloop_cast_narray_to_rarray(cumo_ndfunc_t *nf, VALUE nary, VALUE fmt);
 VALUE cumo_na_ndloop_store_rarray(cumo_ndfunc_t *nf, VALUE nary, VALUE rary);
+
+// One device buffer for a whole walk, not one per row. A row that gives its
+// own back has to wait for the kernel still reading it, and that wait costs
+// more than everything else the row does. cumo_na_ndloop_store_rarray owns
+// one and hands it over as the loop's opt_ptr.
+// Opaque: an iterator asks for a buffer and reads nothing else.
+typedef struct cumo_na_stage_pool cumo_na_stage_pool_t;
+
+char *cumo_na_stage_pool_get(cumo_na_stage_pool_t *pool, size_t bytes);
 VALUE cumo_na_ndloop_store_rarray2(cumo_ndfunc_t *nf, VALUE nary, VALUE rary, VALUE opt);
 VALUE cumo_na_ndloop_inspect(VALUE nary, cumo_na_text_func_t func, VALUE opt);
 VALUE cumo_na_ndloop_with_index(cumo_ndfunc_t *nf, int argc, ...);

data/ext/cumo/include/cumo/narray.h

@@ -190,6 +190,7 @@ extern VALUE cumo_cBit;
 extern VALUE cumo_cDFloat;
 extern VALUE cumo_cSFloat;
 extern VALUE cumo_cHFloat;
+extern VALUE cumo_cBFloat;
 extern VALUE cumo_cDComplex;
 extern VALUE cumo_cSComplex;
 extern VALUE cumo_cInt64;