cumo 0.7.0 → 0.9.0

data/ext/cumo/include/cumo/types/half_macro.h

@@ -1,188 +1,14 @@
 #include "float_def.h"
+#include "bf16_def.h"
 #include "half_def.h"
 
 #define CUMO_HALF_EPSILON 9.765625e-04f
 
-#define m_zero cumo_float2half(0.0f)
-#define m_one  cumo_float2half(1.0f)
+#define CUMO_F16_TO_F(x) cumo_half2float(x)
+#define CUMO_F_TO_F16(x) cumo_float2half(x)
+#define CUMO_D_TO_F16(x) cumo_double2half(x)
 
-#define m_num_to_data(x) cumo_na_ruby_num_to_half(x)
-#define m_data_to_num(x) rb_float_new(cumo_half2float(x))
-
-#define m_from_double(x) cumo_double2half((double)(x))
-#define m_from_real(x)   cumo_double2half((double)(x))
-#define m_from_sint(x)   cumo_double2half((double)(x))
-#define m_from_int32(x)  cumo_double2half((double)(x))
-#define m_from_int64(x)  cumo_double2half((double)(x))
-#define m_from_uint32(x) cumo_double2half((double)(x))
-#define m_from_uint64(x) cumo_double2half((double)(x))
 #define m_from_half(x)   (x)
+#define m_from_bfloat(x) cumo_float2half(cumo_bfloat2float(x))
 
-#define m_add(x,y) cumo_float2half(cumo_half2float(x)+cumo_half2float(y))
-#define m_sub(x,y) cumo_float2half(cumo_half2float(x)-cumo_half2float(y))
-#define m_mul(x,y) cumo_float2half(cumo_half2float(x)*cumo_half2float(y))
-#define m_div(x,y) cumo_float2half(cumo_half2float(x)/cumo_half2float(y))
-#define m_div_check(x,y) (cumo_half2float(y)==0)
-
-static inline float cumo_half_floored_mod(float x, float y)
-{
-    float r = fmodf(x,y);
-    if (r != 0 && ((r < 0) != (y < 0))) {
-        r += y;
-    }
-    return r;
-}
-
-#define m_mod(x,y) cumo_float2half(cumo_half_floored_mod(cumo_half2float(x),cumo_half2float(y)))
-#define m_divmod(x,y,a,b)                                               \
-    {float cumo_h_x = cumo_half2float(x), cumo_h_y = cumo_half2float(y); \
-     float cumo_h_b = fmodf(cumo_h_x,cumo_h_y);                         \
-     float cumo_h_a = roundf((cumo_h_x-cumo_h_b)/cumo_h_y);             \
-     if (cumo_h_b != 0 && ((cumo_h_b < 0) != (cumo_h_y < 0))) {         \
-         cumo_h_b += cumo_h_y; cumo_h_a -= 1;                           \
-     }                                                                  \
-     a = cumo_float2half(cumo_h_a); b = cumo_float2half(cumo_h_b);}
-#define m_pow(x,y) cumo_float2half(powf(cumo_half2float(x),cumo_half2float(y)))
-#define m_pow_int(x,y) cumo_float2half(cumo_half_pow_int(cumo_half2float(x),(int)(y)))
-
-#define m_abs(x)     cumo_float2half(fabsf(cumo_half2float(x)))
-#define m_minus(x)   cumo_float2half(-cumo_half2float(x))
-#define m_reciprocal(x) cumo_float2half(1.0f/cumo_half2float(x))
-#define m_square(x)  cumo_float2half(cumo_half2float(x)*cumo_half2float(x))
-#define m_floor(x)   cumo_float2half(floorf(cumo_half2float(x)))
-#define m_round(x)   cumo_float2half(roundf(cumo_half2float(x)))
-#define m_ceil(x)    cumo_float2half(ceilf(cumo_half2float(x)))
-#define m_trunc(x)   cumo_float2half(truncf(cumo_half2float(x)))
-#define m_rint(x)    cumo_float2half(rintf(cumo_half2float(x)))
-#define m_sign(x)    cumo_float2half(cumo_half_sign(cumo_half2float(x)))
-#define m_copysign(x,y) cumo_float2half(copysignf(cumo_half2float(x),cumo_half2float(y)))
-#define m_signbit(x) signbit(cumo_half2float(x))
-#define m_modf(x,y,z) {float cumo_h_i; float cumo_h_f = modff(cumo_half2float(x),&cumo_h_i); y = cumo_float2half(cumo_h_f); z = cumo_float2half(cumo_h_i);}
-
-#define m_eq(x,y) (cumo_half2float(x)==cumo_half2float(y))
-#define m_ne(x,y) (cumo_half2float(x)!=cumo_half2float(y))
-#define m_gt(x,y) (cumo_half2float(x)>cumo_half2float(y))
-#define m_ge(x,y) (cumo_half2float(x)>=cumo_half2float(y))
-#define m_lt(x,y) (cumo_half2float(x)<cumo_half2float(y))
-#define m_le(x,y) (cumo_half2float(x)<=cumo_half2float(y))
-
-#define m_isnan(x) isnan(cumo_half2float(x))
-#define m_isinf(x) isinf(cumo_half2float(x))
-#define m_isposinf(x) (isinf(cumo_half2float(x)) && signbit(cumo_half2float(x))==0)
-#define m_isneginf(x) (isinf(cumo_half2float(x)) && signbit(cumo_half2float(x)))
-#define m_isfinite(x) isfinite(cumo_half2float(x))
-
-#define not_nan(x) (cumo_half2float(x)==cumo_half2float(x))
-
-#define m_mulsum_init INT2FIX(0)
-
-#define m_sprintf(s,x) sprintf(s,"%g",cumo_half2float(x))
-
-#define cmp_prnan(a,b)                                                  \
-    ((cumo_half2float(qsort_cast(a))==cumo_half2float(qsort_cast(b))) ? 0 : \
-     (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b))) ? 1 : -1)
-
-#define cmp_ignan(a,b)                                                  \
-    (m_isnan(qsort_cast(a)) ? (m_isnan(qsort_cast(b)) ? 0 : 1) :        \
-     (m_isnan(qsort_cast(b)) ? -1 :                                     \
-      ((cumo_half2float(qsort_cast(a))==cumo_half2float(qsort_cast(b))) ? 0 : \
-       (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b))) ? 1 : -1)))
-
-#define cmpgt_prnan(a,b)                        \
-    (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b)))
-
-#define cmpgt_ignan(a,b)                                                \
-    ((m_isnan(qsort_cast(a)) && !m_isnan(qsort_cast(b))) ||             \
-     (cumo_half2float(qsort_cast(a)) > cumo_half2float(qsort_cast(b))))
-
-#define m_sqrt(x)    cumo_float2half(sqrtf(cumo_half2float(x)))
-#define m_cbrt(x)    cumo_float2half(cbrtf(cumo_half2float(x)))
-#define m_log(x)     cumo_float2half(logf(cumo_half2float(x)))
-#define m_log2(x)    cumo_float2half(log2f(cumo_half2float(x)))
-#define m_log10(x)   cumo_float2half(log10f(cumo_half2float(x)))
-#define m_exp(x)     cumo_float2half(expf(cumo_half2float(x)))
-#define m_exp2(x)    cumo_float2half(exp2f(cumo_half2float(x)))
-#ifdef HAVE_EXP10
-#define m_exp10(x)   cumo_float2half(exp10f(cumo_half2float(x)))
-#else
-#define m_exp10(x)   cumo_float2half(powf(10.0f,cumo_half2float(x)))
-#endif
-#define m_expm1(x)   cumo_float2half(expm1f(cumo_half2float(x)))
-#define m_log1p(x)   cumo_float2half(log1pf(cumo_half2float(x)))
-
-#define m_sin(x)     cumo_float2half(sinf(cumo_half2float(x)))
-#define m_cos(x)     cumo_float2half(cosf(cumo_half2float(x)))
-#define m_tan(x)     cumo_float2half(tanf(cumo_half2float(x)))
-#define m_asin(x)    cumo_float2half(asinf(cumo_half2float(x)))
-#define m_acos(x)    cumo_float2half(acosf(cumo_half2float(x)))
-#define m_atan(x)    cumo_float2half(atanf(cumo_half2float(x)))
-#define m_sinh(x)    cumo_float2half(sinhf(cumo_half2float(x)))
-#define m_cosh(x)    cumo_float2half(coshf(cumo_half2float(x)))
-#define m_tanh(x)    cumo_float2half(tanhf(cumo_half2float(x)))
-#define m_asinh(x)   cumo_float2half(asinhf(cumo_half2float(x)))
-#define m_acosh(x)   cumo_float2half(acoshf(cumo_half2float(x)))
-#define m_atanh(x)   cumo_float2half(atanhf(cumo_half2float(x)))
-#define m_atan2(x,y) cumo_float2half(atan2f(cumo_half2float(x),cumo_half2float(y)))
-#define m_hypot(x,y) cumo_float2half(hypotf(cumo_half2float(x),cumo_half2float(y)))
-#define m_sinc(x)    cumo_float2half(cumo_half_sinc(cumo_half2float(x)))
-
-#define m_erf(x)     cumo_float2half(erff(cumo_half2float(x)))
-#define m_erfc(x)    cumo_float2half(erfcf(cumo_half2float(x)))
-#define m_ldexp(x,y) cumo_float2half(cumo_half_ldexp(cumo_half2float(x),cumo_half2float(y)))
-#define m_frexp(x,exp) cumo_float2half(frexpf(cumo_half2float(x),exp))
-
-
-static inline float cumo_half_pow_positive_int(float x, unsigned int p)
-{
-    float r = 1.0f;
-    switch (p) {
-    case 0: return 1.0f;
-    case 1: return x;
-    case 2: return x*x;
-    case 3: return x*x*x;
-    case 4: x = x*x; return x*x;
-    }
-    if (p > 64) return powf(x, (float)p);
-    while (p) {
-        if (p & 1) r *= x;
-        x *= x;
-        p >>= 1;
-    }
-    return r;
-}
-
-static inline float cumo_half_pow_int(float x, int p)
-{
-    if (p < 0) return 1.0f / cumo_half_pow_positive_int(x, -(unsigned int)p);
-    return cumo_half_pow_positive_int(x, (unsigned int)p);
-}
-
-static inline float cumo_half_ldexp(float x, float y)
-{
-    if (!(y == y)) return y;
-    if (y > 64.0f) y = 64.0f;
-    if (y < -64.0f) y = -64.0f;
-    return ldexpf(x, (int)y);
-}
-
-static inline float cumo_half_sinc(float x)
-{
-    return (x==0) ? 1.0f : (sinf(x)/x);
-}
-
-static inline float cumo_half_sign(float x)
-{
-    return (x==0) ? 0.0f : ((x>0) ? 1.0f : ((x<0) ? -1.0f : x));
-}
-
-static inline cumo_half cumo_na_ruby_num_to_half(VALUE x)
-{
-    return cumo_double2half(NIL_P(x) ? nan("") : NUM2DBL(x));
-}
-
-static inline cumo_half f_seq(cumo_half x, cumo_half y, double c)
-{
-    return cumo_float2half(cumo_half2float(x) + cumo_half2float(y) * (float)c);
-}
-
-#include "real_accum.h"
+#include "f16_macro.h"

data/ext/cumo/include/cumo/types/half_macro_kernel.h

@@ -2,197 +2,18 @@
 #define CUMO_HALF_MACRO_KERNEL_H
 
 #include "float_def_kernel.h"
+#include "bf16_def_kernel.h"
 #include "half_def_kernel.h"
-#include <string.h>
 
 #define CUMO_HALF_EPSILON 9.765625e-04f
 
-#define m_zero cumo_float2half(0.0f)
-#define m_one  cumo_float2half(1.0f)
+#define CUMO_F16_TO_F(x) cumo_half2float(x)
+#define CUMO_F_TO_F16(x) cumo_float2half(x)
+#define CUMO_D_TO_F16(x) cumo_double2half(x)
 
-#define m_from_double(x) cumo_double2half((double)(x))
-#define m_from_real(x)   cumo_double2half((double)(x))
-#define m_from_sint(x)   cumo_double2half((double)(x))
-#define m_from_int32(x)  cumo_double2half((double)(x))
-#define m_from_int64(x)  cumo_double2half((double)(x))
-#define m_from_uint32(x) cumo_double2half((double)(x))
-#define m_from_uint64(x) cumo_double2half((double)(x))
 #define m_from_half(x)   (x)
+#define m_from_bfloat(x) cumo_float2half(cumo_bfloat2float(x))
 
-// Overloaded rather than substituted, so that a scan or a reduction can apply
-// the same rule to its own wider accumulator.
-__host__ __device__ static inline float cumo_half_add(float x, float y) { return x + y; }
-__host__ __device__ static inline float cumo_half_sub(float x, float y) { return x - y; }
-__host__ __device__ static inline float cumo_half_mul(float x, float y) { return x * y; }
-__host__ __device__ static inline float cumo_half_div(float x, float y) { return x / y; }
-__host__ __device__ static inline bool cumo_half_not_nan(float x) { return x == x; }
-__host__ __device__ static inline cumo_half cumo_half_add(cumo_half x, cumo_half y) { return cumo_float2half(cumo_half2float(x) + cumo_half2float(y)); }
-__host__ __device__ static inline cumo_half cumo_half_sub(cumo_half x, cumo_half y) { return cumo_float2half(cumo_half2float(x) - cumo_half2float(y)); }
-__host__ __device__ static inline cumo_half cumo_half_mul(cumo_half x, cumo_half y) { return cumo_float2half(cumo_half2float(x) * cumo_half2float(y)); }
-__host__ __device__ static inline cumo_half cumo_half_div(cumo_half x, cumo_half y) { return cumo_float2half(cumo_half2float(x) / cumo_half2float(y)); }
-__host__ __device__ static inline bool cumo_half_not_nan(cumo_half x) { return cumo_half2float(x) == cumo_half2float(x); }
-
-#define m_add(x,y) cumo_half_add(x,y)
-#define m_sub(x,y) cumo_half_sub(x,y)
-#define m_mul(x,y) cumo_half_mul(x,y)
-#define m_div(x,y) cumo_half_div(x,y)
-#define m_div_check(x,y) (cumo_half2float(y)==0)
-
-__host__ __device__ static inline float cumo_half_floored_mod(float x, float y)
-{
-    float r = fmodf(x,y);
-    if (r != 0 && ((r < 0) != (y < 0))) {
-        r += y;
-    }
-    return r;
-}
-
-#define m_mod(x,y) cumo_float2half(cumo_half_floored_mod(cumo_half2float(x),cumo_half2float(y)))
-#define m_divmod(x,y,a,b)                                               \
-    {float cumo_h_x = cumo_half2float(x), cumo_h_y = cumo_half2float(y); \
-     float cumo_h_b = fmodf(cumo_h_x,cumo_h_y);                         \
-     float cumo_h_a = roundf((cumo_h_x-cumo_h_b)/cumo_h_y);             \
-     if (cumo_h_b != 0 && ((cumo_h_b < 0) != (cumo_h_y < 0))) {         \
-         cumo_h_b += cumo_h_y; cumo_h_a -= 1;                           \
-     }                                                                  \
-     a = cumo_float2half(cumo_h_a); b = cumo_float2half(cumo_h_b);}
-#define m_pow(x,y) cumo_float2half(powf(cumo_half2float(x),cumo_half2float(y)))
-#define m_pow_int(x,y) cumo_float2half(cumo_half_pow_int(cumo_half2float(x),(int)(y)))
-
-#define m_abs(x)     cumo_float2half(fabsf(cumo_half2float(x)))
-#define m_minus(x)   cumo_float2half(-cumo_half2float(x))
-#define m_reciprocal(x) cumo_float2half(1.0f/cumo_half2float(x))
-#define m_square(x)  cumo_float2half(cumo_half2float(x)*cumo_half2float(x))
-#define m_floor(x)   cumo_float2half(floorf(cumo_half2float(x)))
-#define m_round(x)   cumo_float2half(roundf(cumo_half2float(x)))
-#define m_ceil(x)    cumo_float2half(ceilf(cumo_half2float(x)))
-#define m_trunc(x)   cumo_float2half(truncf(cumo_half2float(x)))
-#define m_rint(x)    cumo_float2half(rintf(cumo_half2float(x)))
-#define m_sign(x)    cumo_float2half(cumo_half_sign(cumo_half2float(x)))
-#define m_copysign(x,y) cumo_float2half(copysignf(cumo_half2float(x),cumo_half2float(y)))
-#define m_signbit(x) signbit(cumo_half2float(x))
-#define m_modf(x,y,z) {float cumo_h_i; float cumo_h_f = modff(cumo_half2float(x),&cumo_h_i); y = cumo_float2half(cumo_h_f); z = cumo_float2half(cumo_h_i);}
-
-#define m_eq(x,y) (cumo_half2float(x)==cumo_half2float(y))
-#define m_ne(x,y) (cumo_half2float(x)!=cumo_half2float(y))
-#define m_gt(x,y) (cumo_half2float(x)>cumo_half2float(y))
-#define m_ge(x,y) (cumo_half2float(x)>=cumo_half2float(y))
-#define m_lt(x,y) (cumo_half2float(x)<cumo_half2float(y))
-#define m_le(x,y) (cumo_half2float(x)<=cumo_half2float(y))
-
-#define m_isnan(x) isnan(cumo_half2float(x))
-#define m_isinf(x) isinf(cumo_half2float(x))
-#define m_isposinf(x) (isinf(cumo_half2float(x)) && signbit(cumo_half2float(x))==0)
-#define m_isneginf(x) (isinf(cumo_half2float(x)) && signbit(cumo_half2float(x)))
-#define m_isfinite(x) isfinite(cumo_half2float(x))
-
-#define not_nan(x) cumo_half_not_nan(x)
-
-#define m_mulsum_init m_zero
-
-#define m_sprintf(s,x) sprintf(s,"%g",cumo_half2float(x))
-
-#define m_sqrt(x)    cumo_float2half(sqrtf(cumo_half2float(x)))
-#define m_cbrt(x)    cumo_float2half(cbrtf(cumo_half2float(x)))
-#define m_log(x)     cumo_float2half(logf(cumo_half2float(x)))
-#define m_log2(x)    cumo_float2half(log2f(cumo_half2float(x)))
-#define m_log10(x)   cumo_float2half(log10f(cumo_half2float(x)))
-#define m_exp(x)     cumo_float2half(expf(cumo_half2float(x)))
-#define m_exp2(x)    cumo_float2half(exp2f(cumo_half2float(x)))
-#ifdef HAVE_EXP10
-#define m_exp10(x)   cumo_float2half(exp10f(cumo_half2float(x)))
-#else
-#define m_exp10(x)   cumo_float2half(powf(10.0f,cumo_half2float(x)))
-#endif
-#define m_expm1(x)   cumo_float2half(expm1f(cumo_half2float(x)))
-#define m_log1p(x)   cumo_float2half(log1pf(cumo_half2float(x)))
-
-#define m_sin(x)     cumo_float2half(sinf(cumo_half2float(x)))
-#define m_cos(x)     cumo_float2half(cosf(cumo_half2float(x)))
-#define m_tan(x)     cumo_float2half(tanf(cumo_half2float(x)))
-#define m_asin(x)    cumo_float2half(asinf(cumo_half2float(x)))
-#define m_acos(x)    cumo_float2half(acosf(cumo_half2float(x)))
-#define m_atan(x)    cumo_float2half(atanf(cumo_half2float(x)))
-#define m_sinh(x)    cumo_float2half(sinhf(cumo_half2float(x)))
-#define m_cosh(x)    cumo_float2half(coshf(cumo_half2float(x)))
-#define m_tanh(x)    cumo_float2half(tanhf(cumo_half2float(x)))
-#define m_asinh(x)   cumo_float2half(asinhf(cumo_half2float(x)))
-#define m_acosh(x)   cumo_float2half(acoshf(cumo_half2float(x)))
-#define m_atanh(x)   cumo_float2half(atanhf(cumo_half2float(x)))
-#define m_atan2(x,y) cumo_float2half(atan2f(cumo_half2float(x),cumo_half2float(y)))
-#define m_hypot(x,y) cumo_float2half(hypotf(cumo_half2float(x),cumo_half2float(y)))
-#define m_sinc(x)    cumo_float2half(cumo_half_sinc(cumo_half2float(x)))
-
-#define m_erf(x)     cumo_float2half(erff(cumo_half2float(x)))
-#define m_erfc(x)    cumo_float2half(erfcf(cumo_half2float(x)))
-#define m_ldexp(x,y) cumo_float2half(cumo_half_ldexp(cumo_half2float(x),cumo_half2float(y)))
-#define m_frexp(x,exp) cumo_float2half(frexpf(cumo_half2float(x),exp))
-
-
-__host__ __device__ static inline float cumo_half_pow_positive_int(float x, unsigned int p)
-{
-    float r = 1.0f;
-    switch (p) {
-    case 0: return 1.0f;
-    case 1: return x;
-    case 2: return x*x;
-    case 3: return x*x*x;
-    case 4: x = x*x; return x*x;
-    }
-    if (p > 64) return powf(x, (float)p);
-    while (p) {
-        if (p & 1) r *= x;
-        x *= x;
-        p >>= 1;
-    }
-    return r;
-}
-
-__host__ __device__ static inline float cumo_half_pow_int(float x, int p)
-{
-    if (p < 0) return 1.0f / cumo_half_pow_positive_int(x, -(unsigned int)p);
-    return cumo_half_pow_positive_int(x, (unsigned int)p);
-}
-
-// The bit-pattern intrinsics are device-only before CUDA 12, so the bits are
-// taken the way a host caller can take them too.
-__host__ __device__ static inline cumo_half cumo_half_step_down(cumo_half h)
-{
-    unsigned short b;
-    cumo_half r;
-    memcpy(&b, &h, sizeof(b));
-    if (b == 0x0000u) {
-        b = 0x8001u;
-    } else {
-        b = (unsigned short)((b & 0x8000u) ? (b + 1) : (b - 1));
-    }
-    memcpy(&r, &b, sizeof(b));
-    return r;
-}
-
-__host__ __device__ static inline float cumo_half_ldexp(float x, float y)
-{
-    if (!(y == y)) return y;
-    if (y > 64.0f) y = 64.0f;
-    if (y < -64.0f) y = -64.0f;
-    return ldexpf(x, (int)y);
-}
-
-__host__ __device__ static inline float cumo_half_sinc(float x)
-{
-    return (x==0) ? 1.0f : (sinf(x)/x);
-}
-
-__host__ __device__ static inline float cumo_half_sign(float x)
-{
-    return (x==0) ? 0.0f : ((x>0) ? 1.0f : ((x<0) ? -1.0f : x));
-}
-
-__host__ __device__ static inline cumo_half f_seq(cumo_half x, cumo_half y, double c)
-{
-    return cumo_float2half(cumo_half2float(x) + cumo_half2float(y) * (float)c);
-}
-
-#include "real_accum_kernel.h"
+#include "f16_macro_kernel.h"
 
 #endif // CUMO_HALF_MACRO_KERNEL_H

data/ext/cumo/include/cumo/types/robj_macro.h

@@ -1,4 +1,5 @@
 #include "half_def.h"
+#include "bf16_def.h"
 
 #define m_zero INT2FIX(0)
 #define m_one  INT2FIX(1)
@@ -9,6 +10,7 @@
 #define m_from_double(x) rb_float_new(x)
 #define m_from_real(x)   rb_float_new(x)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x)   INT2FIX(x)
 #define m_from_int32(x)  INT322NUM(x)
 #define m_from_int64(x)  INT642NUM(x)

data/ext/cumo/include/cumo/types/xint_macro.h

@@ -1,4 +1,5 @@
 #include "half_def.h"
+#include "bf16_def.h"
 
 #define m_zero 0
 #define m_one  1
@@ -9,6 +10,7 @@
 #define m_from_double(x) ((x) < 0 ? (dtype)((long long)(x)) : (dtype)(x))
 #define m_from_real(x) ((x) < 0 ? (dtype)((long long)(x)) : (dtype)(x))
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x) (x)
 #define m_from_int32(x) (x)
 #define m_from_int64(x) (x)
@@ -49,12 +51,6 @@
 #define m_cumsum(x,y) {x += y;}
 #define m_cumprod(x,y) {x *= y;}
 
-#define cmp(a,b)                                        \
-    ((qsort_cast(a)==qsort_cast(b)) ? 0 :               \
-     (qsort_cast(a) > qsort_cast(b)) ? 1 : -1)
-#define cmpgt(a,b)                              \
-    (qsort_cast(a) > qsort_cast(b))
-
 static inline dtype f_min(size_t n, char *p, ssize_t stride)
 {
     dtype x,y;

data/ext/cumo/include/cumo/types/xint_macro_kernel.h

@@ -1,6 +1,7 @@
 #ifndef CUMO_XINT_MACRO_KERNEL_H
 #define CUMO_XINT_MACRO_KERNEL_H
 #include "half_def_kernel.h"
+#include "bf16_def_kernel.h"
 
 #define m_zero 0
 #define m_one  1
@@ -8,6 +9,7 @@
 #define m_from_double(x) (x)
 #define m_from_real(x) (x)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x) (x)
 #define m_from_int32(x) (x)
 #define m_from_int64(x) (x)

data/ext/cumo/include/cumo.h

@@ -10,8 +10,8 @@ extern "C" {
 #endif
 #endif
 
-#define CUMO_VERSION "0.7.0"
-#define CUMO_VERSION_CODE 700
+#define CUMO_VERSION "0.9.0"
+#define CUMO_VERSION_CODE 900
 
 bool cumo_compatible_mode_enabled_p();
 bool cumo_show_warning_enabled_p();

data/ext/cumo/narray/data.c

@@ -55,6 +55,7 @@ static ID cumo_id_swap_byte;
 
 void cumo_iter_copy_bytes_kernel_launch(char *p1, char *p2, ssize_t s1, ssize_t s2, size_t *idx1, size_t *idx2, size_t n, int elmsz);
 void cumo_iter_copy_bytes_indexer_kernel_launch(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_indexer_t* indexer, ssize_t elmsz);
+void cumo_iter_copy_bytes_stridx_kernel_launch(cumo_na_iarray_stridx_t* a1, cumo_na_iarray_stridx_t* a2, cumo_na_indexer_t* indexer, ssize_t elmsz);
 void cumo_iter_swap_byte_indexer_kernel_launch(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_indexer_t* indexer, ssize_t elmsz);
 #define m_memcpy(src,dst) memcpy(dst,src,e)
 
@@ -86,11 +87,19 @@ iter_copy_bytes(cumo_na_loop_t *const lp)
 static void
 iter_copy_bytes_indexer(cumo_na_loop_t *const lp)
 {
-    cumo_na_iarray_t a1 = cumo_na_make_iarray(&lp->args[0]);
-    cumo_na_iarray_t a2 = cumo_na_make_iarray(&lp->args[1]);
     cumo_na_indexer_t indexer = cumo_na_make_indexer(&lp->args[0]);
 
-    cumo_iter_copy_bytes_indexer_kernel_launch(&a1, &a2, &indexer, (ssize_t)lp->args[0].elmsz);
+    if (cumo_na_loop_has_index(lp)) {
+        cumo_na_iarray_stridx_t b1 = cumo_na_make_iarray_stridx(&lp->args[0]);
+        cumo_na_iarray_stridx_t b2 = cumo_na_make_iarray_stridx(&lp->args[1]);
+
+        cumo_iter_copy_bytes_stridx_kernel_launch(&b1, &b2, &indexer, (ssize_t)lp->args[0].elmsz);
+    } else {
+        cumo_na_iarray_t a1 = cumo_na_make_iarray(&lp->args[0]);
+        cumo_na_iarray_t a2 = cumo_na_make_iarray(&lp->args[1]);
+
+        cumo_iter_copy_bytes_indexer_kernel_launch(&a1, &a2, &indexer, (ssize_t)lp->args[0].elmsz);
+    }
 }
 
 VALUE
@@ -99,7 +108,7 @@ cumo_na_copy(VALUE self)
     VALUE v;
     cumo_ndfunc_arg_in_t ain[1] = {{Qnil,0}};
     cumo_ndfunc_arg_out_t aout[1] = {{INT2FIX(0),0}};
-    cumo_ndfunc_t ndf = { iter_copy_bytes, CUMO_FULL_LOOP, 1, 1, ain, aout };
+    cumo_ndfunc_t ndf = { iter_copy_bytes, CUMO_FULL_LOOP_NIP, 1, 1, ain, aout };
 
     // The loops below move whole bytes, which is one element of every class but
     // Cumo::Bit, whose element is one bit. Its own copy is the one that can
@@ -114,7 +123,7 @@ cumo_na_copy(VALUE self)
     // synchronization per step when an operand carries an index array.
     if (!rb_obj_is_kind_of(self, cumo_cRObject)) {
         ndf.func = iter_copy_bytes_indexer;
-        ndf.flag = CUMO_STRIDE_LOOP|CUMO_NDF_INDEXER_LOOP;
+        ndf.flag = CUMO_FULL_LOOP_NIP|CUMO_NDF_INDEXER_LOOP;
     }
 
     v = cumo_na_ndloop(&ndf, 1, self);
@@ -502,7 +511,7 @@ cumo_na_reshape_bang(int argc, VALUE *argv, VALUE self)
 
   @overload  reshape(size0,size1,...)
   @param sizeN [Integer] new shape
-  @return [Cumo::NArray] return self.
+  @return [Cumo::NArray] a copy carrying the new shape.
   @example
 */
 static VALUE

data/ext/cumo/narray/data_kernel.cu

@@ -1,5 +1,6 @@
 #include "cumo/narray_kernel.h"
 #include "cumo/indexer.h"
+#include "cumo/template_kernel.h"
 
 #if defined(__cplusplus)
 extern "C" {
@@ -239,8 +240,117 @@ __global__ void cumo_iter_copy_bytes_indexer_kernel_dim(cumo_na_iarray_t a1, cum
     }
 }
 
+#define CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(NDIM) \
+__global__ void cumo_iter_copy_bytes_stridx_kernel_dim##NDIM( \
+        cumo_na_iarray_stridx_t a1, cumo_na_iarray_stridx_t a2, cumo_na_indexer_t indexer, ssize_t elmsz) { \
+    for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < indexer.total_size; i += blockDim.x * gridDim.x) { \
+        cumo_na_indexer_set_dim##NDIM(&indexer, i); \
+        char* p1 = cumo_na_iarray_stridx_at_dim##NDIM(&a1, &indexer); \
+        char* p2 = cumo_na_iarray_stridx_at_dim##NDIM(&a2, &indexer); \
+        memcpy(p2, p1, elmsz); \
+    } \
+}
+
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(0)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(1)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(2)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(3)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(4)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(5)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(6)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(7)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL(8)
+CUMO_ITER_COPY_BYTES_STRIDX_KERNEL()
+
+#undef CUMO_ITER_COPY_BYTES_STRIDX_KERNEL
+
+void cumo_iter_copy_bytes_stridx_kernel_launch(cumo_na_iarray_stridx_t* a1, cumo_na_iarray_stridx_t* a2, cumo_na_indexer_t* indexer, ssize_t elmsz)
+{
+    size_t grid_dim = cumo_get_grid_dim(indexer->total_size);
+    size_t block_dim = cumo_get_block_dim(indexer->total_size);
+    switch (indexer->ndim) {
+#define CUMO_ITER_COPY_BYTES_STRIDX_CASE(NDIM) \
+    case NDIM: \
+        cumo_iter_copy_bytes_stridx_kernel_dim##NDIM<<<grid_dim, block_dim>>>(*a1, *a2, *indexer, elmsz); \
+        break;
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(0)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(1)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(2)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(3)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(4)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(5)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(6)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(7)
+    CUMO_ITER_COPY_BYTES_STRIDX_CASE(8)
+#undef CUMO_ITER_COPY_BYTES_STRIDX_CASE
+    default:
+        cumo_iter_copy_bytes_stridx_kernel_dim<<<grid_dim, block_dim>>>(*a1, *a2, *indexer, elmsz);
+        break;
+    }
+    cumo_cuda_runtime_check_kernel_launch();
+}
+
+// A copy of a transposed view has one side strided whichever way it is walked,
+// so it stages a square tile in shared memory the way the typed store does. The
+// element size is only known here at run time, so there is one kernel per size
+// a tile can carry.
+struct cumo_copy_bytes_elm16 { uint64_t lo, hi; };
+
+#define CUMO_ITER_COPY_BYTES_TRANSPOSE_KERNEL(SZ, TYPE) \
+__global__ void cumo_iter_copy_bytes_transpose_kernel_##SZ( \
+        char *dst, char *src, uint64_t rows, uint64_t cols) { \
+    CUMO_TRANSPOSE_TILE_DECL(TYPE, tile); \
+    CUMO_TRANSPOSE_TILE_LOOP(tile, rows, cols, \
+        *(TYPE*)(src + cumo_tile_src * sizeof(TYPE)), \
+        *(TYPE*)(dst + cumo_tile_dst * sizeof(TYPE)) = cumo_tile_val; \
+    ); \
+}
+
+CUMO_ITER_COPY_BYTES_TRANSPOSE_KERNEL(1, uint8_t)
+CUMO_ITER_COPY_BYTES_TRANSPOSE_KERNEL(2, uint16_t)
+CUMO_ITER_COPY_BYTES_TRANSPOSE_KERNEL(4, uint32_t)
+CUMO_ITER_COPY_BYTES_TRANSPOSE_KERNEL(8, uint64_t)
+CUMO_ITER_COPY_BYTES_TRANSPOSE_KERNEL(16, cumo_copy_bytes_elm16)
+
+// True when the source runs along its columns and the destination along its
+// rows. a1 is the source here and a2 the destination, the other way round from
+// the typed store.
+static int
+cumo_iter_copy_bytes_is_transpose(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_indexer_t* indexer, ssize_t elmsz)
+{
+    return (elmsz == 1 || elmsz == 2 || elmsz == 4 || elmsz == 8 || elmsz == 16) &&
+        CUMO_TRANSPOSE_TILE_FITS(indexer) &&
+        a2->step[1] == elmsz &&
+        a2->step[0] == elmsz * (ssize_t)indexer->shape[1] &&
+        a1->step[0] == elmsz &&
+        a1->step[1] == elmsz * (ssize_t)indexer->shape[0];
+}
+
 void cumo_iter_copy_bytes_indexer_kernel_launch(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_indexer_t* indexer, ssize_t elmsz)
 {
+    if (cumo_iter_copy_bytes_is_transpose(a1, a2, indexer, elmsz)) {
+        uint64_t rows = indexer->shape[0];
+        uint64_t cols = indexer->shape[1];
+        switch (elmsz) {
+        case 1:
+            CUMO_TRANSPOSE_LAUNCH(cumo_iter_copy_bytes_transpose_kernel_1, rows, cols, a2->ptr, a1->ptr);
+            break;
+        case 2:
+            CUMO_TRANSPOSE_LAUNCH(cumo_iter_copy_bytes_transpose_kernel_2, rows, cols, a2->ptr, a1->ptr);
+            break;
+        case 4:
+            CUMO_TRANSPOSE_LAUNCH(cumo_iter_copy_bytes_transpose_kernel_4, rows, cols, a2->ptr, a1->ptr);
+            break;
+        case 8:
+            CUMO_TRANSPOSE_LAUNCH(cumo_iter_copy_bytes_transpose_kernel_8, rows, cols, a2->ptr, a1->ptr);
+            break;
+        default:
+            CUMO_TRANSPOSE_LAUNCH(cumo_iter_copy_bytes_transpose_kernel_16, rows, cols, a2->ptr, a1->ptr);
+            break;
+        }
+        cumo_cuda_runtime_check_kernel_launch();
+        return;
+    }
     size_t grid_dim = cumo_get_grid_dim(indexer->total_size);
     size_t block_dim = cumo_get_block_dim(indexer->total_size);
     switch (indexer->ndim) {