cumo 0.6.0 → 0.7.0

data/ext/cumo/include/cumo/template_kernel.h

@@ -62,6 +62,102 @@
 // limit of 2^31-1 is far above this.
 #define CUMO_MAX_GRID_DIM (4294967295ul / CUMO_MAX_BLOCK_DIM)
 
+// A transpose has one side strided whichever way it is walked, so it stages a
+// square tile in shared memory and reads and writes both sides in rows. The
+// extra column spreads a tile's rows over the banks: for a 4-byte element that
+// leaves no conflict at all, and it drops an 8-byte one to 2-way and a 16-byte
+// one to 4-way. TILE must stay a multiple of ROWS.
+#define CUMO_TRANSPOSE_TILE 32
+#define CUMO_TRANSPOSE_ROWS 8
+
+// gridDim.y and gridDim.z stop at 65535, unlike gridDim.x.
+#define CUMO_MAX_GRID_DIM_Y 65535
+
+// The tile a CUMO_TRANSPOSE_TILE_LOOP stages through. The extra column is what
+// spreads its rows over the banks, so the loop reads it back along either axis
+// without conflicting with itself.
+#define CUMO_TRANSPOSE_TILE_DECL(type, name) \
+    __shared__ type name[CUMO_TRANSPOSE_TILE][CUMO_TRANSPOSE_TILE + 1]
+
+// Tiles only pay off once both sides reach one. A shorter side leaves most of
+// each warp idle, while the plain loop already reads or writes it in one go.
+#define CUMO_TRANSPOSE_TILE_FITS(indexer)                                      \
+    ((indexer)->ndim == 2 &&                                                   \
+     (indexer)->shape[0] >= CUMO_TRANSPOSE_TILE &&                             \
+     (indexer)->shape[1] >= CUMO_TRANSPOSE_TILE)
+
+// Walks a rows-by-cols transpose one tile at a time.
+//
+// STAGE reads one element of the transposed side, which sits at cumo_tile_src
+// of that side's own layout, and lands in cumo_tile_val. The body then runs
+// once per output element, at cumo_tile_dst of the row-major layout the output
+// and any other operand share.
+//
+// The body runs inside two nested loops, after a __syncthreads(). A continue
+// there skips one output element, a break leaves the current tile but not the
+// walk, and a return deadlocks the block on the next __syncthreads().
+#define CUMO_TRANSPOSE_TILE_LOOP(tile, rows, cols, STAGE, ...)                 \
+    do {                                                                       \
+        uint64_t cumo_tile_rows = (rows);                                      \
+        uint64_t cumo_tile_cols = (cols);                                      \
+        uint64_t cumo_tile_base;                                               \
+                                                                               \
+        for (cumo_tile_base = (uint64_t)blockIdx.y * CUMO_TRANSPOSE_TILE;      \
+             cumo_tile_base < cumo_tile_rows;                                  \
+             cumo_tile_base += (uint64_t)gridDim.y * CUMO_TRANSPOSE_TILE) {    \
+            uint64_t cumo_tile_x = cumo_tile_base + threadIdx.x;               \
+            uint64_t cumo_tile_y =                                             \
+                (uint64_t)blockIdx.x * CUMO_TRANSPOSE_TILE + threadIdx.y;      \
+            int cumo_tile_j;                                                   \
+                                                                               \
+            __syncthreads();                                                   \
+            for (cumo_tile_j = 0;                                              \
+                 threadIdx.y + cumo_tile_j < CUMO_TRANSPOSE_TILE;              \
+                 cumo_tile_j += CUMO_TRANSPOSE_ROWS) {                         \
+                if (cumo_tile_x < cumo_tile_rows &&                            \
+                    cumo_tile_y + cumo_tile_j < cumo_tile_cols) {              \
+                    uint64_t cumo_tile_src =                                   \
+                        (cumo_tile_y + cumo_tile_j) * cumo_tile_rows + cumo_tile_x; \
+                    (tile)[threadIdx.y + cumo_tile_j][threadIdx.x] = STAGE;    \
+                }                                                              \
+            }                                                                  \
+            __syncthreads();                                                   \
+            cumo_tile_x = (uint64_t)blockIdx.x * CUMO_TRANSPOSE_TILE + threadIdx.x; \
+            cumo_tile_y = cumo_tile_base + threadIdx.y;                        \
+            for (cumo_tile_j = 0;                                              \
+                 threadIdx.y + cumo_tile_j < CUMO_TRANSPOSE_TILE;              \
+                 cumo_tile_j += CUMO_TRANSPOSE_ROWS) {                         \
+                if (cumo_tile_x < cumo_tile_cols &&                            \
+                    cumo_tile_y + cumo_tile_j < cumo_tile_rows) {              \
+                    uint64_t cumo_tile_dst =                                   \
+                        (cumo_tile_y + cumo_tile_j) * cumo_tile_cols + cumo_tile_x; \
+                    auto cumo_tile_val =                                       \
+                        (tile)[threadIdx.x][threadIdx.y + cumo_tile_j];        \
+                    __VA_ARGS__                                                \
+                }                                                              \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+// Launches such a kernel, which takes its own arguments and then rows and cols.
+// gridDim.y walks tile rows on its own, but gridDim.x has to cover every tile
+// column: a short grid there drops the columns past it without an error.
+#define CUMO_TRANSPOSE_LAUNCH(kernel, rows, cols, ...)                         \
+    do {                                                                       \
+        uint64_t cumo_launch_rows = (rows);                                    \
+        uint64_t cumo_launch_cols = (cols);                                    \
+        uint64_t cumo_launch_tiles_y =                                         \
+            (cumo_launch_rows + CUMO_TRANSPOSE_TILE - 1) / CUMO_TRANSPOSE_TILE; \
+        dim3 cumo_launch_grid(                                                 \
+            (unsigned int)((cumo_launch_cols + CUMO_TRANSPOSE_TILE - 1) / CUMO_TRANSPOSE_TILE), \
+            (unsigned int)(cumo_launch_tiles_y > CUMO_MAX_GRID_DIM_Y           \
+                           ? CUMO_MAX_GRID_DIM_Y : cumo_launch_tiles_y));      \
+        dim3 cumo_launch_block(CUMO_TRANSPOSE_TILE, CUMO_TRANSPOSE_ROWS);      \
+                                                                               \
+        kernel<<<cumo_launch_grid, cumo_launch_block>>>(                       \
+            __VA_ARGS__, cumo_launch_rows, cumo_launch_cols);                  \
+    } while (0)
+
 static inline size_t
 cumo_get_grid_dim(size_t n)
 {

data/ext/cumo/include/cumo/types/bit.h

@@ -1,3 +1,5 @@
+#include "half_def.h"
+
 typedef CUMO_BIT_DIGIT dtype;
 typedef CUMO_BIT_DIGIT rtype;
 #define cT  cumo_cBit
@@ -11,6 +13,7 @@ typedef CUMO_BIT_DIGIT rtype;
 
 #define m_from_double(x) (((x)==0) ? 0 : 1)
 #define m_from_real(x) (((x)==0) ? 0 : 1)
+#define m_from_half(x) m_from_real(cumo_half2float(x))
 #define m_from_sint(x) (((x)==0) ? 0 : 1)
 #define m_from_int32(x) (((x)==0) ? 0 : 1)
 #define m_from_int64(x) (((x)==0) ? 0 : 1)

data/ext/cumo/include/cumo/types/bit_kernel.h

@@ -1,6 +1,8 @@
 #ifndef CUMO_BIT_KERNEL_H
 #define CUMO_BIT_KERNEL_H
 
+#include "half_def_kernel.h"
+
 typedef CUMO_BIT_DIGIT dtype;
 typedef CUMO_BIT_DIGIT rtype;
 
@@ -12,6 +14,7 @@ typedef CUMO_BIT_DIGIT rtype;
 
 #define m_from_double(x) (((x)==0) ? 0 : 1)
 #define m_from_real(x) (((x)==0) ? 0 : 1)
+#define m_from_half(x) m_from_real(cumo_half2float(x))
 #define m_from_sint(x) (((x)==0) ? 0 : 1)
 #define m_from_int32(x) (((x)==0) ? 0 : 1)
 #define m_from_int64(x) (((x)==0) ? 0 : 1)

data/ext/cumo/include/cumo/types/complex_macro.h

@@ -1,4 +1,5 @@
 #include "float_def.h"
+#include "half_def.h"
 
 extern double round(double);
 extern double log2(double);
@@ -43,6 +44,7 @@ static inline dtype c_from_dcomplex(cumo_dcomplex x) {
 
 #define m_from_double(x) c_new(x,0)
 #define m_from_real(x)   c_new(x,0)
+#define m_from_half(x) m_from_real(cumo_half2float(x))
 #define m_from_sint(x)   c_new(x,0)
 #define m_from_int32(x)  c_new(x,0)
 #define m_from_int64(x)  c_new(x,0)

data/ext/cumo/include/cumo/types/complex_macro_kernel.h

@@ -2,6 +2,7 @@
 #define CUMO_COMPLEX_MACRO_KERNEL_H
 
 #include "float_def_kernel.h"
+#include "half_def_kernel.h"
 
 extern double round(double);
 extern double log2(double);
@@ -46,6 +47,7 @@ __host__ __device__ static inline dtype c_from_dcomplex(cumo_dcomplex x) {
 
 #define m_from_double(x) c_new(x,0)
 #define m_from_real(x)   c_new(x,0)
+#define m_from_half(x) m_from_real(cumo_half2float(x))
 #define m_from_sint(x)   c_new(x,0)
 #define m_from_int32(x)  c_new(x,0)
 #define m_from_int64(x)  c_new(x,0)

data/ext/cumo/include/cumo/types/float_macro.h

@@ -1,4 +1,5 @@
 #include "float_def.h"
+#include "half_def.h"
 
 extern double round(double);
 extern double log2(double);
@@ -17,6 +18,7 @@ extern double pow(double, double);
 
 #define m_from_double(x) (x)
 #define m_from_real(x) (x)
+#define m_from_half(x) m_from_real(cumo_half2float(x))
 #define m_from_sint(x) (x)
 #define m_from_int32(x) (x)
 #define m_from_int64(x) (x)

data/ext/cumo/include/cumo/types/float_macro_kernel.h

@@ -2,6 +2,7 @@
 #define CUMO_FLOAT_MACRO_KERNEL_H
 
 #include "float_def_kernel.h"
+#include "half_def_kernel.h"
 
 extern double round(double);
 extern double log2(double);
@@ -20,6 +21,7 @@ extern double pow(double, double);
 
 #define m_from_double(x) (x)
 #define m_from_real(x) (x)
+#define m_from_half(x) m_from_real(cumo_half2float(x))
 #define m_from_sint(x) (x)
 #define m_from_int32(x) (x)
 #define m_from_int64(x) (x)

data/ext/cumo/include/cumo/types/half_def.h

@@ -0,0 +1,129 @@
+#ifndef CUMO_HALF_DEF_H
+#define CUMO_HALF_DEF_H
+
+// A struct rather than a bare uint16, so that C code which reaches for an
+// arithmetic operator on an element fails to compile instead of computing on
+// the bit pattern. Its layout matches __half, which the kernels use.
+typedef struct { unsigned short x; } cumo_half;
+
+// GCC gained _Float16 on x86-64 only in 12, and Ubuntu 22.04 ships 11, so the
+// conversions are written out. Both round the way __float2half does, which
+// test/hfloat_test.rb checks against the GPU over every 32-bit pattern.
+static inline float cumo_half2float(cumo_half h)
+{
+    union { unsigned int u; float f; } o;
+    unsigned int m = h.x & 0x3ffu;
+    unsigned int s = (unsigned int)(h.x & 0x8000u) << 16;
+    int e = (h.x >> 10) & 0x1f;
+
+    if (e == 0) {
+        if (m == 0) {
+            o.u = s;
+            return o.f;
+        }
+        e = 1;
+        while (!(m & 0x400u)) {
+            m <<= 1;
+            e--;
+        }
+        m &= 0x3ffu;
+    } else if (e == 31) {
+        o.u = s | 0x7f800000u | (m << 13);
+        return o.f;
+    }
+    o.u = s | ((unsigned int)(e + 112) << 23) | (m << 13);
+    return o.f;
+}
+
+static inline cumo_half cumo_float2half(float f)
+{
+    union { unsigned int u; float f; } v;
+    cumo_half h;
+    unsigned int u, sign, mant, round_bits, shift, out;
+    int exp;
+
+    v.f = f;
+    u = v.u;
+    sign = (u >> 16) & 0x8000u;
+    u &= 0x7fffffffu;
+
+    if (u > 0x7f800000u) {
+        h.x = (unsigned short)(sign | 0x7e00u);
+        return h;
+    }
+    if (u >= 0x47800000u) {
+        h.x = (unsigned short)(sign | 0x7c00u);
+        return h;
+    }
+    if (u < 0x33000000u) {
+        h.x = (unsigned short)sign;
+        return h;
+    }
+
+    exp = (int)(u >> 23) - 127;
+    mant = u & 0x7fffffu;
+    if (exp >= -14) {
+        out = (unsigned int)(exp + 15) << 10;
+        shift = 13;
+    } else {
+        mant |= 0x800000u;
+        shift = (unsigned int)(-exp - 1);
+        out = 0;
+    }
+    round_bits = mant & ((1u << shift) - 1u);
+    out += mant >> shift;
+    if (round_bits > (1u << (shift - 1)) ||
+        (round_bits == (1u << (shift - 1)) && ((mant >> shift) & 1u))) {
+        out += 1;
+    }
+    h.x = (unsigned short)(sign | out);
+    return h;
+}
+
+static inline cumo_half cumo_double2half(double d)
+{
+    union { unsigned long long u; double d; } v;
+    cumo_half h;
+    unsigned long long u, sign, mant, round_bits, tie;
+    unsigned int shift, out;
+    int exp;
+
+    v.d = d;
+    u = v.u;
+    sign = (u >> 48) & 0x8000ull;
+    u &= 0x7fffffffffffffffull;
+
+    if (u > 0x7ff0000000000000ull) {
+        h.x = (unsigned short)(sign | 0x7e00u);
+        return h;
+    }
+    if (u >= 0x40f0000000000000ull) {
+        h.x = (unsigned short)(sign | 0x7c00u);
+        return h;
+    }
+    if (u < 0x3e60000000000000ull) {
+        h.x = (unsigned short)sign;
+        return h;
+    }
+
+    exp = (int)(u >> 52) - 1023;
+    mant = u & 0xfffffffffffffull;
+    if (exp >= -14) {
+        out = (unsigned int)(exp + 15) << 10;
+        shift = 42;
+    } else {
+        mant |= 0x10000000000000ull;
+        shift = (unsigned int)(28 - exp);
+        out = 0;
+    }
+    round_bits = mant & ((1ull << shift) - 1ull);
+    tie = 1ull << (shift - 1);
+    out += (unsigned int)(mant >> shift);
+    if (round_bits > tie || (round_bits == tie && ((mant >> shift) & 1ull))) {
+        out += 1;
+    }
+    h.x = (unsigned short)(sign | out);
+    return h;
+}
+
+#endif // CUMO_HALF_DEF_H

data/ext/cumo/include/cumo/types/half_def_kernel.h

@@ -0,0 +1,28 @@
+#ifndef CUMO_HALF_DEF_KERNEL_H
+#define CUMO_HALF_DEF_KERNEL_H
+
+#include <cuda_fp16.h>
+
+typedef __half cumo_half;
+
+// The arithmetic instructions on __half start at sm_53, but the conversions
+// have no such floor, so every operation is computed in float. For + - * / and
+// sqrt that answers what the native half instruction would: a float carries
+// more than twice the bits of a half, so the second rounding cannot land on a
+// tie the first one moved.
+__host__ __device__ static inline float cumo_half2float(cumo_half h)
+{
+    return __half2float(h);
+}
+
+__host__ __device__ static inline cumo_half cumo_float2half(float f)
+{
+    return __float2half(f);
+}
+
+__host__ __device__ static inline cumo_half cumo_double2half(double d)
+{
+    return __double2half(d);
+}
+
+#endif // CUMO_HALF_DEF_KERNEL_H

data/ext/cumo/include/cumo/types/half_macro.h

@@ -0,0 +1,188 @@
+#include "float_def.h"
+#include "half_def.h"
+
+#define CUMO_HALF_EPSILON 9.765625e-04f
+
+#define m_zero cumo_float2half(0.0f)
+#define m_one  cumo_float2half(1.0f)
+
+#define m_num_to_data(x) cumo_na_ruby_num_to_half(x)
+#define m_data_to_num(x) rb_float_new(cumo_half2float(x))
+
+#define m_from_double(x) cumo_double2half((double)(x))
+#define m_from_real(x)   cumo_double2half((double)(x))
+#define m_from_sint(x)   cumo_double2half((double)(x))
+#define m_from_int32(x)  cumo_double2half((double)(x))
+#define m_from_int64(x)  cumo_double2half((double)(x))
+#define m_from_uint32(x) cumo_double2half((double)(x))
+#define m_from_uint64(x) cumo_double2half((double)(x))
+#define m_from_half(x)   (x)
+
+#define m_add(x,y) cumo_float2half(cumo_half2float(x)+cumo_half2float(y))
+#define m_sub(x,y) cumo_float2half(cumo_half2float(x)-cumo_half2float(y))
+#define m_mul(x,y) cumo_float2half(cumo_half2float(x)*cumo_half2float(y))
+#define m_div(x,y) cumo_float2half(cumo_half2float(x)/cumo_half2float(y))
+#define m_div_check(x,y) (cumo_half2float(y)==0)
+
+static inline float cumo_half_floored_mod(float x, float y)
+{
+    float r = fmodf(x,y);
+    if (r != 0 && ((r < 0) != (y < 0))) {
+        r += y;
+    }
+    return r;
+}
+
+#define m_mod(x,y) cumo_float2half(cumo_half_floored_mod(cumo_half2float(x),cumo_half2float(y)))
+#define m_divmod(x,y,a,b)                                               \
+    {float cumo_h_x = cumo_half2float(x), cumo_h_y = cumo_half2float(y); \
+     float cumo_h_b = fmodf(cumo_h_x,cumo_h_y);                         \
+     float cumo_h_a = roundf((cumo_h_x-cumo_h_b)/cumo_h_y);             \
+     if (cumo_h_b != 0 && ((cumo_h_b < 0) != (cumo_h_y < 0))) {         \
+         cumo_h_b += cumo_h_y; cumo_h_a -= 1;                           \
+     }                                                                  \
+     a = cumo_float2half(cumo_h_a); b = cumo_float2half(cumo_h_b);}
+#define m_pow(x,y) cumo_float2half(powf(cumo_half2float(x),cumo_half2float(y)))
+#define m_pow_int(x,y) cumo_float2half(cumo_half_pow_int(cumo_half2float(x),(int)(y)))
+
+#define m_abs(x)     cumo_float2half(fabsf(cumo_half2float(x)))
+#define m_minus(x)   cumo_float2half(-cumo_half2float(x))
+#define m_reciprocal(x) cumo_float2half(1.0f/cumo_half2float(x))
+#define m_square(x)  cumo_float2half(cumo_half2float(x)*cumo_half2float(x))
+#define m_floor(x)   cumo_float2half(floorf(cumo_half2float(x)))
+#define m_round(x)   cumo_float2half(roundf(cumo_half2float(x)))
+#define m_ceil(x)    cumo_float2half(ceilf(cumo_half2float(x)))
+#define m_trunc(x)   cumo_float2half(truncf(cumo_half2float(x)))
+#define m_rint(x)    cumo_float2half(rintf(cumo_half2float(x)))
+#define m_sign(x)    cumo_float2half(cumo_half_sign(cumo_half2float(x)))
+#define m_copysign(x,y) cumo_float2half(copysignf(cumo_half2float(x),cumo_half2float(y)))
+#define m_signbit(x) signbit(cumo_half2float(x))
+#define m_modf(x,y,z) {float cumo_h_i; float cumo_h_f = modff(cumo_half2float(x),&cumo_h_i); y = cumo_float2half(cumo_h_f); z = cumo_float2half(cumo_h_i);}
+
+#define m_eq(x,y) (cumo_half2float(x)==cumo_half2float(y))
+#define m_ne(x,y) (cumo_half2float(x)!=cumo_half2float(y))
+#define m_gt(x,y) (cumo_half2float(x)>cumo_half2float(y))
+#define m_ge(x,y) (cumo_half2float(x)>=cumo_half2float(y))
+#define m_lt(x,y) (cumo_half2float(x)<cumo_half2float(y))
+#define m_le(x,y) (cumo_half2float(x)<=cumo_half2float(y))
+
+#define m_isnan(x) isnan(cumo_half2float(x))
+#define m_isinf(x) isinf(cumo_half2float(x))
+#define m_isposinf(x) (isinf(cumo_half2float(x)) && signbit(cumo_half2float(x))==0)
+#define m_isneginf(x) (isinf(cumo_half2float(x)) && signbit(cumo_half2float(x)))
+#define m_isfinite(x) isfinite(cumo_half2float(x))
+
+#define not_nan(x) (cumo_half2float(x)==cumo_half2float(x))
+
+#define m_mulsum_init INT2FIX(0)
+
+#define m_sprintf(s,x) sprintf(s,"%g",cumo_half2float(x))
+
+#define cmp_prnan(a,b)                                                  \
+    ((cumo_half2float(qsort_cast(a))==cumo_half2float(qsort_cast(b))) ? 0 : \
+     (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b))) ? 1 : -1)
+
+#define cmp_ignan(a,b)                                                  \
+    (m_isnan(qsort_cast(a)) ? (m_isnan(qsort_cast(b)) ? 0 : 1) :        \
+     (m_isnan(qsort_cast(b)) ? -1 :                                     \
+      ((cumo_half2float(qsort_cast(a))==cumo_half2float(qsort_cast(b))) ? 0 : \
+       (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b))) ? 1 : -1)))
+
+#define cmpgt_prnan(a,b)                        \
+    (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b)))
+
+#define cmpgt_ignan(a,b)                                                \
+    ((m_isnan(qsort_cast(a)) && !m_isnan(qsort_cast(b))) ||             \
+     (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b))))
+
+#define m_sqrt(x)    cumo_float2half(sqrtf(cumo_half2float(x)))
+#define m_cbrt(x)    cumo_float2half(cbrtf(cumo_half2float(x)))
+#define m_log(x)     cumo_float2half(logf(cumo_half2float(x)))
+#define m_log2(x)    cumo_float2half(log2f(cumo_half2float(x)))
+#define m_log10(x)   cumo_float2half(log10f(cumo_half2float(x)))
+#define m_exp(x)     cumo_float2half(expf(cumo_half2float(x)))
+#define m_exp2(x)    cumo_float2half(exp2f(cumo_half2float(x)))
+#ifdef HAVE_EXP10
+#define m_exp10(x)   cumo_float2half(exp10f(cumo_half2float(x)))
+#else
+#define m_exp10(x)   cumo_float2half(powf(10.0f,cumo_half2float(x)))
+#endif
+#define m_expm1(x)   cumo_float2half(expm1f(cumo_half2float(x)))
+#define m_log1p(x)   cumo_float2half(log1pf(cumo_half2float(x)))
+
+#define m_sin(x)     cumo_float2half(sinf(cumo_half2float(x)))
+#define m_cos(x)     cumo_float2half(cosf(cumo_half2float(x)))
+#define m_tan(x)     cumo_float2half(tanf(cumo_half2float(x)))
+#define m_asin(x)    cumo_float2half(asinf(cumo_half2float(x)))
+#define m_acos(x)    cumo_float2half(acosf(cumo_half2float(x)))
+#define m_atan(x)    cumo_float2half(atanf(cumo_half2float(x)))
+#define m_sinh(x)    cumo_float2half(sinhf(cumo_half2float(x)))
+#define m_cosh(x)    cumo_float2half(coshf(cumo_half2float(x)))
+#define m_tanh(x)    cumo_float2half(tanhf(cumo_half2float(x)))
+#define m_asinh(x)   cumo_float2half(asinhf(cumo_half2float(x)))
+#define m_acosh(x)   cumo_float2half(acoshf(cumo_half2float(x)))
+#define m_atanh(x)   cumo_float2half(atanhf(cumo_half2float(x)))
+#define m_atan2(x,y) cumo_float2half(atan2f(cumo_half2float(x),cumo_half2float(y)))
+#define m_hypot(x,y) cumo_float2half(hypotf(cumo_half2float(x),cumo_half2float(y)))
+#define m_sinc(x)    cumo_float2half(cumo_half_sinc(cumo_half2float(x)))
+
+#define m_erf(x)     cumo_float2half(erff(cumo_half2float(x)))
+#define m_erfc(x)    cumo_float2half(erfcf(cumo_half2float(x)))
+#define m_ldexp(x,y) cumo_float2half(cumo_half_ldexp(cumo_half2float(x),cumo_half2float(y)))
+#define m_frexp(x,exp) cumo_float2half(frexpf(cumo_half2float(x),exp))
+
+
+static inline float cumo_half_pow_positive_int(float x, unsigned int p)
+{
+    float r = 1.0f;
+    switch (p) {
+    case 0: return 1.0f;
+    case 1: return x;
+    case 2: return x*x;
+    case 3: return x*x*x;
+    case 4: x = x*x; return x*x;
+    }
+    if (p > 64) return powf(x, (float)p);
+    while (p) {
+        if (p & 1) r *= x;
+        x *= x;
+        p >>= 1;
+    }
+    return r;
+}
+
+static inline float cumo_half_pow_int(float x, int p)
+{
+    if (p < 0) return 1.0f / cumo_half_pow_positive_int(x, -(unsigned int)p);
+    return cumo_half_pow_positive_int(x, (unsigned int)p);
+}
+
+static inline float cumo_half_ldexp(float x, float y)
+{
+    if (!(y == y)) return y;
+    if (y > 64.0f) y = 64.0f;
+    if (y < -64.0f) y = -64.0f;
+    return ldexpf(x, (int)y);
+}
+
+static inline float cumo_half_sinc(float x)
+{
+    return (x==0) ? 1.0f : (sinf(x)/x);
+}
+
+static inline float cumo_half_sign(float x)
+{
+    return (x==0) ? 0.0f : ((x>0) ? 1.0f : ((x<0) ? -1.0f : x));
+}
+
+static inline cumo_half cumo_na_ruby_num_to_half(VALUE x)
+{
+    return cumo_double2half(NIL_P(x) ? nan("") : NUM2DBL(x));
+}
+
+static inline cumo_half f_seq(cumo_half x, cumo_half y, double c)
+{
+    return cumo_float2half(cumo_half2float(x) + cumo_half2float(y) * (float)c);
+}
+
+#include "real_accum.h"