cumo 0.7.0 → 0.8.0

data/ext/cumo/include/cumo/types/bf16_macro.h

@@ -0,0 +1,186 @@
+#include "float_def.h"
+#include "half_def.h"
+#include "bf16_def.h"
+
+#define CUMO_BFLOAT_EPSILON 7.8125e-03f
+
+#define m_zero cumo_float2bfloat(0.0f)
+#define m_one  cumo_float2bfloat(1.0f)
+
+#define m_num_to_data(x) cumo_na_ruby_num_to_bfloat(x)
+#define m_data_to_num(x) rb_float_new(cumo_bfloat2float(x))
+
+#define m_from_double(x) cumo_double2bfloat((double)(x))
+#define m_from_real(x)   cumo_double2bfloat((double)(x))
+#define m_from_sint(x)   cumo_double2bfloat((double)(x))
+#define m_from_int32(x)  cumo_double2bfloat((double)(x))
+#define m_from_int64(x)  cumo_double2bfloat((double)(x))
+#define m_from_uint32(x) cumo_double2bfloat((double)(x))
+#define m_from_uint64(x) cumo_double2bfloat((double)(x))
+#define m_from_bfloat(x)   (x)
+#define m_from_half(x) cumo_float2bfloat(cumo_half2float(x))
+
+#define m_add(x,y) cumo_float2bfloat(cumo_bfloat2float(x)+cumo_bfloat2float(y))
+#define m_sub(x,y) cumo_float2bfloat(cumo_bfloat2float(x)-cumo_bfloat2float(y))
+#define m_mul(x,y) cumo_float2bfloat(cumo_bfloat2float(x)*cumo_bfloat2float(y))
+#define m_div(x,y) cumo_float2bfloat(cumo_bfloat2float(x)/cumo_bfloat2float(y))
+#define m_div_check(x,y) (cumo_bfloat2float(y)==0)
+
+static inline float cumo_bfloat_floored_mod(float x, float y)
+{
+    float r = fmodf(x,y);
+    if (r != 0 && ((r < 0) != (y < 0))) {
+        r += y;
+    }
+    return r;
+}
+
+#define m_mod(x,y) cumo_float2bfloat(cumo_bfloat_floored_mod(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_divmod(x,y,a,b)                                               \
+    {float cumo_b_x = cumo_bfloat2float(x), cumo_b_y = cumo_bfloat2float(y); \
+     float cumo_b_b = fmodf(cumo_b_x,cumo_b_y);                         \
+     float cumo_b_a = roundf((cumo_b_x-cumo_b_b)/cumo_b_y);             \
+     if (cumo_b_b != 0 && ((cumo_b_b < 0) != (cumo_b_y < 0))) {         \
+         cumo_b_b += cumo_b_y; cumo_b_a -= 1;                           \
+     }                                                                  \
+     a = cumo_float2bfloat(cumo_b_a); b = cumo_float2bfloat(cumo_b_b);}
+#define m_pow(x,y) cumo_float2bfloat(powf(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_pow_int(x,y) cumo_float2bfloat(cumo_bfloat_pow_int(cumo_bfloat2float(x),(int)(y)))
+
+#define m_abs(x)     cumo_float2bfloat(fabsf(cumo_bfloat2float(x)))
+#define m_minus(x)   cumo_float2bfloat(-cumo_bfloat2float(x))
+#define m_reciprocal(x) cumo_float2bfloat(1.0f/cumo_bfloat2float(x))
+#define m_square(x)  cumo_float2bfloat(cumo_bfloat2float(x)*cumo_bfloat2float(x))
+#define m_floor(x)   cumo_float2bfloat(floorf(cumo_bfloat2float(x)))
+#define m_round(x)   cumo_float2bfloat(roundf(cumo_bfloat2float(x)))
+#define m_ceil(x)    cumo_float2bfloat(ceilf(cumo_bfloat2float(x)))
+#define m_trunc(x)   cumo_float2bfloat(truncf(cumo_bfloat2float(x)))
+#define m_rint(x)    cumo_float2bfloat(rintf(cumo_bfloat2float(x)))
+#define m_sign(x)    cumo_float2bfloat(cumo_bfloat_sign(cumo_bfloat2float(x)))
+#define m_copysign(x,y) cumo_float2bfloat(copysignf(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_signbit(x) signbit(cumo_bfloat2float(x))
+#define m_modf(x,y,z) {float cumo_b_i; float cumo_b_f = modff(cumo_bfloat2float(x),&cumo_b_i); y = cumo_float2bfloat(cumo_b_f); z = cumo_float2bfloat(cumo_b_i);}
+
+#define m_eq(x,y) (cumo_bfloat2float(x)==cumo_bfloat2float(y))
+#define m_ne(x,y) (cumo_bfloat2float(x)!=cumo_bfloat2float(y))
+#define m_gt(x,y) (cumo_bfloat2float(x)>cumo_bfloat2float(y))
+#define m_ge(x,y) (cumo_bfloat2float(x)>=cumo_bfloat2float(y))
+#define m_lt(x,y) (cumo_bfloat2float(x)<cumo_bfloat2float(y))
+#define m_le(x,y) (cumo_bfloat2float(x)<=cumo_bfloat2float(y))
+
+#define m_isnan(x) isnan(cumo_bfloat2float(x))
+#define m_isinf(x) isinf(cumo_bfloat2float(x))
+#define m_isposinf(x) (isinf(cumo_bfloat2float(x)) && signbit(cumo_bfloat2float(x))==0)
+#define m_isneginf(x) (isinf(cumo_bfloat2float(x)) && signbit(cumo_bfloat2float(x)))
+#define m_isfinite(x) isfinite(cumo_bfloat2float(x))
+
+#define not_nan(x) (cumo_bfloat2float(x)==cumo_bfloat2float(x))
+
+#define m_mulsum_init INT2FIX(0)
+
+#define m_sprintf(s,x) sprintf(s,"%g",cumo_bfloat2float(x))
+
+#define m_sqrt(x)    cumo_float2bfloat(sqrtf(cumo_bfloat2float(x)))
+#define m_cbrt(x)    cumo_float2bfloat(cbrtf(cumo_bfloat2float(x)))
+#define m_log(x)     cumo_float2bfloat(logf(cumo_bfloat2float(x)))
+#define m_log2(x)    cumo_float2bfloat(log2f(cumo_bfloat2float(x)))
+#define m_log10(x)   cumo_float2bfloat(log10f(cumo_bfloat2float(x)))
+#define m_exp(x)     cumo_float2bfloat(expf(cumo_bfloat2float(x)))
+#define m_exp2(x)    cumo_float2bfloat(exp2f(cumo_bfloat2float(x)))
+#ifdef HAVE_EXP10
+#define m_exp10(x)   cumo_float2bfloat(exp10f(cumo_bfloat2float(x)))
+#else
+#define m_exp10(x)   cumo_float2bfloat(powf(10.0f,cumo_bfloat2float(x)))
+#endif
+#define m_expm1(x)   cumo_float2bfloat(expm1f(cumo_bfloat2float(x)))
+#define m_log1p(x)   cumo_float2bfloat(log1pf(cumo_bfloat2float(x)))
+
+#define m_sin(x)     cumo_float2bfloat(sinf(cumo_bfloat2float(x)))
+#define m_cos(x)     cumo_float2bfloat(cosf(cumo_bfloat2float(x)))
+#define m_tan(x)     cumo_float2bfloat(tanf(cumo_bfloat2float(x)))
+#define m_asin(x)    cumo_float2bfloat(asinf(cumo_bfloat2float(x)))
+#define m_acos(x)    cumo_float2bfloat(acosf(cumo_bfloat2float(x)))
+#define m_atan(x)    cumo_float2bfloat(atanf(cumo_bfloat2float(x)))
+#define m_sinh(x)    cumo_float2bfloat(sinhf(cumo_bfloat2float(x)))
+#define m_cosh(x)    cumo_float2bfloat(coshf(cumo_bfloat2float(x)))
+#define m_tanh(x)    cumo_float2bfloat(tanhf(cumo_bfloat2float(x)))
+#define m_asinh(x)   cumo_float2bfloat(asinhf(cumo_bfloat2float(x)))
+#define m_acosh(x)   cumo_float2bfloat(acoshf(cumo_bfloat2float(x)))
+#define m_atanh(x)   cumo_float2bfloat(atanhf(cumo_bfloat2float(x)))
+#define m_atan2(x,y) cumo_float2bfloat(atan2f(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_hypot(x,y) cumo_float2bfloat(hypotf(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_sinc(x)    cumo_float2bfloat(cumo_bfloat_sinc(cumo_bfloat2float(x)))
+
+#define m_erf(x)     cumo_float2bfloat(erff(cumo_bfloat2float(x)))
+#define m_erfc(x)    cumo_float2bfloat(erfcf(cumo_bfloat2float(x)))
+#define m_gelu(x)      cumo_float2bfloat(cumo_bfloat_gelu(cumo_bfloat2float(x)))
+#define m_gelu_tanh(x) cumo_float2bfloat(cumo_bfloat_gelu_tanh(cumo_bfloat2float(x)))
+#define m_ldexp(x,y) cumo_float2bfloat(cumo_bfloat_ldexp(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_frexp(x,exp) cumo_float2bfloat(frexpf(cumo_bfloat2float(x),exp))
+
+
+static inline float cumo_bfloat_pow_positive_int(float x, unsigned int p)
+{
+    float r = 1.0f;
+    switch (p) {
+    case 0: return 1.0f;
+    case 1: return x;
+    case 2: return x*x;
+    case 3: return x*x*x;
+    case 4: x = x*x; return x*x;
+    }
+    if (p > 64) return powf(x, (float)p);
+    while (p) {
+        if (p & 1) r *= x;
+        x *= x;
+        p >>= 1;
+    }
+    return r;
+}
+
+static inline float cumo_bfloat_pow_int(float x, int p)
+{
+    if (p < 0) return 1.0f / cumo_bfloat_pow_positive_int(x, -(unsigned int)p);
+    return cumo_bfloat_pow_positive_int(x, (unsigned int)p);
+}
+
+static inline float cumo_bfloat_ldexp(float x, float y)
+{
+    if (!(y == y)) return y;
+    if (y > 64.0f) y = 64.0f;
+    if (y < -64.0f) y = -64.0f;
+    return ldexpf(x, (int)y);
+}
+
+static inline float cumo_bfloat_sinc(float x)
+{
+    return (x==0) ? 1.0f : (sinf(x)/x);
+}
+
+static inline float cumo_bfloat_sign(float x)
+{
+    return (x==0) ? 0.0f : ((x>0) ? 1.0f : ((x<0) ? -1.0f : x));
+}
+
+static inline cumo_bfloat cumo_na_ruby_num_to_bfloat(VALUE x)
+{
+    return cumo_double2bfloat(NIL_P(x) ? nan("") : NUM2DBL(x));
+}
+
+static inline cumo_bfloat f_seq(cumo_bfloat x, cumo_bfloat y, double c)
+{
+    return cumo_float2bfloat(cumo_bfloat2float(x) + cumo_bfloat2float(y) * (float)c);
+}
+
+#include "real_accum.h"
+
+static inline float cumo_bfloat_gelu(float x)
+{
+    return 0.5f * x * (1.0f + erff(x * (float)CUMO_M_SQRT1_2));
+}
+
+// The approximation GPT-2 and the transformers after it were trained with.
+static inline float cumo_bfloat_gelu_tanh(float x)
+{
+    return 0.5f * x * (1.0f + tanhf((float)CUMO_M_SQRT_2_OVER_PI * (x + (float)CUMO_GELU_TANH_CUBIC * x * x * x)));
+}

data/ext/cumo/include/cumo/types/bf16_macro_kernel.h

@@ -0,0 +1,213 @@
+#ifndef CUMO_BFLOAT_MACRO_KERNEL_H
+#define CUMO_BFLOAT_MACRO_KERNEL_H
+
+#include "float_def_kernel.h"
+#include "half_def_kernel.h"
+#include "bf16_def_kernel.h"
+#include <string.h>
+
+#define CUMO_BFLOAT_EPSILON 7.8125e-03f
+
+#define m_zero cumo_float2bfloat(0.0f)
+#define m_one  cumo_float2bfloat(1.0f)
+
+#define m_from_double(x) cumo_double2bfloat((double)(x))
+#define m_from_real(x)   cumo_double2bfloat((double)(x))
+#define m_from_sint(x)   cumo_double2bfloat((double)(x))
+#define m_from_int32(x)  cumo_double2bfloat((double)(x))
+#define m_from_int64(x)  cumo_double2bfloat((double)(x))
+#define m_from_uint32(x) cumo_double2bfloat((double)(x))
+#define m_from_uint64(x) cumo_double2bfloat((double)(x))
+#define m_from_bfloat(x)   (x)
+#define m_from_half(x) cumo_float2bfloat(cumo_half2float(x))
+
+// Overloaded rather than substituted, so that a scan or a reduction can apply
+// the same rule to its own wider accumulator.
+__host__ __device__ static inline float cumo_bfloat_add(float x, float y) { return x + y; }
+__host__ __device__ static inline float cumo_bfloat_sub(float x, float y) { return x - y; }
+__host__ __device__ static inline float cumo_bfloat_mul(float x, float y) { return x * y; }
+__host__ __device__ static inline float cumo_bfloat_div(float x, float y) { return x / y; }
+__host__ __device__ static inline bool cumo_bfloat_not_nan(float x) { return x == x; }
+__host__ __device__ static inline cumo_bfloat cumo_bfloat_add(cumo_bfloat x, cumo_bfloat y) { return cumo_float2bfloat(cumo_bfloat2float(x) + cumo_bfloat2float(y)); }
+__host__ __device__ static inline cumo_bfloat cumo_bfloat_sub(cumo_bfloat x, cumo_bfloat y) { return cumo_float2bfloat(cumo_bfloat2float(x) - cumo_bfloat2float(y)); }
+__host__ __device__ static inline cumo_bfloat cumo_bfloat_mul(cumo_bfloat x, cumo_bfloat y) { return cumo_float2bfloat(cumo_bfloat2float(x) * cumo_bfloat2float(y)); }
+__host__ __device__ static inline cumo_bfloat cumo_bfloat_div(cumo_bfloat x, cumo_bfloat y) { return cumo_float2bfloat(cumo_bfloat2float(x) / cumo_bfloat2float(y)); }
+__host__ __device__ static inline bool cumo_bfloat_not_nan(cumo_bfloat x) { return cumo_bfloat2float(x) == cumo_bfloat2float(x); }
+
+#define m_add(x,y) cumo_bfloat_add(x,y)
+#define m_sub(x,y) cumo_bfloat_sub(x,y)
+#define m_mul(x,y) cumo_bfloat_mul(x,y)
+#define m_div(x,y) cumo_bfloat_div(x,y)
+#define m_div_check(x,y) (cumo_bfloat2float(y)==0)
+
+__host__ __device__ static inline float cumo_bfloat_floored_mod(float x, float y)
+{
+    float r = fmodf(x,y);
+    if (r != 0 && ((r < 0) != (y < 0))) {
+        r += y;
+    }
+    return r;
+}
+
+#define m_mod(x,y) cumo_float2bfloat(cumo_bfloat_floored_mod(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_divmod(x,y,a,b)                                               \
+    {float cumo_b_x = cumo_bfloat2float(x), cumo_b_y = cumo_bfloat2float(y); \
+     float cumo_b_b = fmodf(cumo_b_x,cumo_b_y);                         \
+     float cumo_b_a = roundf((cumo_b_x-cumo_b_b)/cumo_b_y);             \
+     if (cumo_b_b != 0 && ((cumo_b_b < 0) != (cumo_b_y < 0))) {         \
+         cumo_b_b += cumo_b_y; cumo_b_a -= 1;                           \
+     }                                                                  \
+     a = cumo_float2bfloat(cumo_b_a); b = cumo_float2bfloat(cumo_b_b);}
+#define m_pow(x,y) cumo_float2bfloat(powf(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_pow_int(x,y) cumo_float2bfloat(cumo_bfloat_pow_int(cumo_bfloat2float(x),(int)(y)))
+
+#define m_abs(x)     cumo_float2bfloat(fabsf(cumo_bfloat2float(x)))
+#define m_minus(x)   cumo_float2bfloat(-cumo_bfloat2float(x))
+#define m_reciprocal(x) cumo_float2bfloat(1.0f/cumo_bfloat2float(x))
+#define m_square(x)  cumo_float2bfloat(cumo_bfloat2float(x)*cumo_bfloat2float(x))
+#define m_floor(x)   cumo_float2bfloat(floorf(cumo_bfloat2float(x)))
+#define m_round(x)   cumo_float2bfloat(roundf(cumo_bfloat2float(x)))
+#define m_ceil(x)    cumo_float2bfloat(ceilf(cumo_bfloat2float(x)))
+#define m_trunc(x)   cumo_float2bfloat(truncf(cumo_bfloat2float(x)))
+#define m_rint(x)    cumo_float2bfloat(rintf(cumo_bfloat2float(x)))
+#define m_sign(x)    cumo_float2bfloat(cumo_bfloat_sign(cumo_bfloat2float(x)))
+#define m_copysign(x,y) cumo_float2bfloat(copysignf(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_signbit(x) signbit(cumo_bfloat2float(x))
+#define m_modf(x,y,z) {float cumo_b_i; float cumo_b_f = modff(cumo_bfloat2float(x),&cumo_b_i); y = cumo_float2bfloat(cumo_b_f); z = cumo_float2bfloat(cumo_b_i);}
+
+#define m_eq(x,y) (cumo_bfloat2float(x)==cumo_bfloat2float(y))
+#define m_ne(x,y) (cumo_bfloat2float(x)!=cumo_bfloat2float(y))
+#define m_gt(x,y) (cumo_bfloat2float(x)>cumo_bfloat2float(y))
+#define m_ge(x,y) (cumo_bfloat2float(x)>=cumo_bfloat2float(y))
+#define m_lt(x,y) (cumo_bfloat2float(x)<cumo_bfloat2float(y))
+#define m_le(x,y) (cumo_bfloat2float(x)<=cumo_bfloat2float(y))
+
+#define m_isnan(x) isnan(cumo_bfloat2float(x))
+#define m_isinf(x) isinf(cumo_bfloat2float(x))
+#define m_isposinf(x) (isinf(cumo_bfloat2float(x)) && signbit(cumo_bfloat2float(x))==0)
+#define m_isneginf(x) (isinf(cumo_bfloat2float(x)) && signbit(cumo_bfloat2float(x)))
+#define m_isfinite(x) isfinite(cumo_bfloat2float(x))
+
+#define not_nan(x) cumo_bfloat_not_nan(x)
+
+#define m_mulsum_init m_zero
+
+#define m_sprintf(s,x) sprintf(s,"%g",cumo_bfloat2float(x))
+
+#define m_sqrt(x)    cumo_float2bfloat(sqrtf(cumo_bfloat2float(x)))
+#define m_cbrt(x)    cumo_float2bfloat(cbrtf(cumo_bfloat2float(x)))
+#define m_log(x)     cumo_float2bfloat(logf(cumo_bfloat2float(x)))
+#define m_log2(x)    cumo_float2bfloat(log2f(cumo_bfloat2float(x)))
+#define m_log10(x)   cumo_float2bfloat(log10f(cumo_bfloat2float(x)))
+#define m_exp(x)     cumo_float2bfloat(expf(cumo_bfloat2float(x)))
+#define m_exp2(x)    cumo_float2bfloat(exp2f(cumo_bfloat2float(x)))
+#ifdef HAVE_EXP10
+#define m_exp10(x)   cumo_float2bfloat(exp10f(cumo_bfloat2float(x)))
+#else
+#define m_exp10(x)   cumo_float2bfloat(powf(10.0f,cumo_bfloat2float(x)))
+#endif
+#define m_expm1(x)   cumo_float2bfloat(expm1f(cumo_bfloat2float(x)))
+#define m_log1p(x)   cumo_float2bfloat(log1pf(cumo_bfloat2float(x)))
+
+#define m_sin(x)     cumo_float2bfloat(sinf(cumo_bfloat2float(x)))
+#define m_cos(x)     cumo_float2bfloat(cosf(cumo_bfloat2float(x)))
+#define m_tan(x)     cumo_float2bfloat(tanf(cumo_bfloat2float(x)))
+#define m_asin(x)    cumo_float2bfloat(asinf(cumo_bfloat2float(x)))
+#define m_acos(x)    cumo_float2bfloat(acosf(cumo_bfloat2float(x)))
+#define m_atan(x)    cumo_float2bfloat(atanf(cumo_bfloat2float(x)))
+#define m_sinh(x)    cumo_float2bfloat(sinhf(cumo_bfloat2float(x)))
+#define m_cosh(x)    cumo_float2bfloat(coshf(cumo_bfloat2float(x)))
+#define m_tanh(x)    cumo_float2bfloat(tanhf(cumo_bfloat2float(x)))
+#define m_asinh(x)   cumo_float2bfloat(asinhf(cumo_bfloat2float(x)))
+#define m_acosh(x)   cumo_float2bfloat(acoshf(cumo_bfloat2float(x)))
+#define m_atanh(x)   cumo_float2bfloat(atanhf(cumo_bfloat2float(x)))
+#define m_atan2(x,y) cumo_float2bfloat(atan2f(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_hypot(x,y) cumo_float2bfloat(hypotf(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_sinc(x)    cumo_float2bfloat(cumo_bfloat_sinc(cumo_bfloat2float(x)))
+
+#define m_erf(x)     cumo_float2bfloat(erff(cumo_bfloat2float(x)))
+#define m_erfc(x)    cumo_float2bfloat(erfcf(cumo_bfloat2float(x)))
+#define m_gelu(x)      cumo_float2bfloat(cumo_bfloat_gelu(cumo_bfloat2float(x)))
+#define m_gelu_tanh(x) cumo_float2bfloat(cumo_bfloat_gelu_tanh(cumo_bfloat2float(x)))
+#define m_ldexp(x,y) cumo_float2bfloat(cumo_bfloat_ldexp(cumo_bfloat2float(x),cumo_bfloat2float(y)))
+#define m_frexp(x,exp) cumo_float2bfloat(frexpf(cumo_bfloat2float(x),exp))
+
+
+__host__ __device__ static inline float cumo_bfloat_pow_positive_int(float x, unsigned int p)
+{
+    float r = 1.0f;
+    switch (p) {
+    case 0: return 1.0f;
+    case 1: return x;
+    case 2: return x*x;
+    case 3: return x*x*x;
+    case 4: x = x*x; return x*x;
+    }
+    if (p > 64) return powf(x, (float)p);
+    while (p) {
+        if (p & 1) r *= x;
+        x *= x;
+        p >>= 1;
+    }
+    return r;
+}
+
+__host__ __device__ static inline float cumo_bfloat_pow_int(float x, int p)
+{
+    if (p < 0) return 1.0f / cumo_bfloat_pow_positive_int(x, -(unsigned int)p);
+    return cumo_bfloat_pow_positive_int(x, (unsigned int)p);
+}
+
+// The bit-pattern intrinsics are device-only before CUDA 12, so the bits are
+// taken the way a host caller can take them too.
+__host__ __device__ static inline cumo_bfloat cumo_bfloat_step_down(cumo_bfloat h)
+{
+    unsigned short b;
+    cumo_bfloat r;
+    memcpy(&b, &h, sizeof(b));
+    if (b == 0x0000u) {
+        b = 0x8001u;
+    } else {
+        b = (unsigned short)((b & 0x8000u) ? (b + 1) : (b - 1));
+    }
+    memcpy(&r, &b, sizeof(b));
+    return r;
+}
+
+__host__ __device__ static inline float cumo_bfloat_ldexp(float x, float y)
+{
+    if (!(y == y)) return y;
+    if (y > 64.0f) y = 64.0f;
+    if (y < -64.0f) y = -64.0f;
+    return ldexpf(x, (int)y);
+}
+
+__host__ __device__ static inline float cumo_bfloat_sinc(float x)
+{
+    return (x==0) ? 1.0f : (sinf(x)/x);
+}
+
+__host__ __device__ static inline float cumo_bfloat_sign(float x)
+{
+    return (x==0) ? 0.0f : ((x>0) ? 1.0f : ((x<0) ? -1.0f : x));
+}
+
+__host__ __device__ static inline cumo_bfloat f_seq(cumo_bfloat x, cumo_bfloat y, double c)
+{
+    return cumo_float2bfloat(cumo_bfloat2float(x) + cumo_bfloat2float(y) * (float)c);
+}
+
+#include "real_accum_kernel.h"
+
+__host__ __device__ static inline float cumo_bfloat_gelu(float x)
+{
+    return 0.5f * x * (1.0f + erff(x * (float)CUMO_M_SQRT1_2));
+}
+
+// The approximation GPT-2 and the transformers after it were trained with.
+__host__ __device__ static inline float cumo_bfloat_gelu_tanh(float x)
+{
+    return 0.5f * x * (1.0f + tanhf((float)CUMO_M_SQRT_2_OVER_PI * (x + (float)CUMO_GELU_TANH_CUBIC * x * x * x)));
+}
+
+#endif // CUMO_BFLOAT_MACRO_KERNEL_H

data/ext/cumo/include/cumo/types/bfloat.h

@@ -0,0 +1,22 @@
+#include "bf16_def.h"
+
+typedef cumo_bfloat dtype;
+typedef cumo_bfloat rtype;
+#define cT  cumo_cBFloat
+#define cRT cumo_cBFloat
+#define mTM cumo_mBFloatMath
+
+#include "bf16_macro.h"
+#include "cublas_v2.h"
+#include "cumo/cuda/cublas.h"
+#include "cumo/cuda/cudnn.h"
+
+#define m_extract(x) rb_float_new(cumo_bfloat2float(*(cumo_bfloat*)x))
+#define m_nearly_eq(x,y) (fabsf(cumo_bfloat2float(x)-cumo_bfloat2float(y))<=(fabsf(cumo_bfloat2float(x))+fabsf(cumo_bfloat2float(y)))*CUMO_BFLOAT_EPSILON*2)
+
+#define M_EPSILON rb_float_new(7.8125e-03)
+#define M_MIN     rb_float_new(1.17549435082228750797e-38)
+#define M_MAX     rb_float_new(3.38953138925153547590e+38)
+
+#define DATA_MIN cumo_float2bfloat(-3.38953138925153547590e+38f)
+#define DATA_MAX cumo_float2bfloat(3.38953138925153547590e+38f)

data/ext/cumo/include/cumo/types/bfloat_kernel.h

@@ -0,0 +1,16 @@
+#ifndef CUMO_BFLOAT_KERNEL_H
+#define CUMO_BFLOAT_KERNEL_H
+
+#include "bf16_def_kernel.h"
+
+typedef cumo_bfloat dtype;
+typedef cumo_bfloat rtype;
+
+#include "bf16_macro_kernel.h"
+
+#define m_nearly_eq(x,y) (fabsf(cumo_bfloat2float(x)-cumo_bfloat2float(y))<=(fabsf(cumo_bfloat2float(x))+fabsf(cumo_bfloat2float(y)))*CUMO_BFLOAT_EPSILON*2)
+
+#define DATA_MIN cumo_float2bfloat(-3.38953138925153547590e+38f)
+#define DATA_MAX cumo_float2bfloat(3.38953138925153547590e+38f)
+
+#endif // CUMO_BFLOAT_KERNEL_H

data/ext/cumo/include/cumo/types/bit.h

@@ -1,4 +1,5 @@
 #include "half_def.h"
+#include "bf16_def.h"
 
 typedef CUMO_BIT_DIGIT dtype;
 typedef CUMO_BIT_DIGIT rtype;
@@ -14,6 +15,7 @@ typedef CUMO_BIT_DIGIT rtype;
 #define m_from_double(x) (((x)==0) ? 0 : 1)
 #define m_from_real(x) (((x)==0) ? 0 : 1)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x) (((x)==0) ? 0 : 1)
 #define m_from_int32(x) (((x)==0) ? 0 : 1)
 #define m_from_int64(x) (((x)==0) ? 0 : 1)

data/ext/cumo/include/cumo/types/bit_kernel.h

@@ -2,6 +2,7 @@
 #define CUMO_BIT_KERNEL_H
 
 #include "half_def_kernel.h"
+#include "bf16_def_kernel.h"
 
 typedef CUMO_BIT_DIGIT dtype;
 typedef CUMO_BIT_DIGIT rtype;
@@ -15,6 +16,7 @@ typedef CUMO_BIT_DIGIT rtype;
 #define m_from_double(x) (((x)==0) ? 0 : 1)
 #define m_from_real(x) (((x)==0) ? 0 : 1)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x) (((x)==0) ? 0 : 1)
 #define m_from_int32(x) (((x)==0) ? 0 : 1)
 #define m_from_int64(x) (((x)==0) ? 0 : 1)

data/ext/cumo/include/cumo/types/complex_macro.h

@@ -1,5 +1,6 @@
 #include "float_def.h"
 #include "half_def.h"
+#include "bf16_def.h"
 
 extern double round(double);
 extern double log2(double);
@@ -45,6 +46,7 @@ static inline dtype c_from_dcomplex(cumo_dcomplex x) {
 #define m_from_double(x) c_new(x,0)
 #define m_from_real(x)   c_new(x,0)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x)   c_new(x,0)
 #define m_from_int32(x)  c_new(x,0)
 #define m_from_int64(x)  c_new(x,0)

data/ext/cumo/include/cumo/types/complex_macro_kernel.h

@@ -3,6 +3,7 @@
 
 #include "float_def_kernel.h"
 #include "half_def_kernel.h"
+#include "bf16_def_kernel.h"
 
 extern double round(double);
 extern double log2(double);
@@ -48,6 +49,7 @@ __host__ __device__ static inline dtype c_from_dcomplex(cumo_dcomplex x) {
 #define m_from_double(x) c_new(x,0)
 #define m_from_real(x)   c_new(x,0)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x)   c_new(x,0)
 #define m_from_int32(x)  c_new(x,0)
 #define m_from_int64(x)  c_new(x,0)

data/ext/cumo/include/cumo/types/float_def.h

@@ -32,3 +32,11 @@
 #ifndef M_LN10
 #define M_LN10         2.30258509299404568402  /* log_e 10 */
 #endif
+
+/* The constants the two gelu forms are written with, spelled once on each side
+   so the macro headers cannot drift a digit apart. */
+#ifndef CUMO_M_SQRT1_2
+#define CUMO_M_SQRT1_2        0.70710678118654752440  /* 1/sqrt(2) */
+#define CUMO_M_SQRT_2_OVER_PI 0.79788456080286535588  /* sqrt(2/pi) */
+#define CUMO_GELU_TANH_CUBIC  0.044715                /* fitted, not a closed form */
+#endif

data/ext/cumo/include/cumo/types/float_def_kernel.h

@@ -36,4 +36,12 @@
 #define M_LN10         2.30258509299404568402  /* log_e 10 */
 #endif
 
+/* The constants the two gelu forms are written with, spelled once on each side
+   so the macro headers cannot drift a digit apart. */
+#ifndef CUMO_M_SQRT1_2
+#define CUMO_M_SQRT1_2        0.70710678118654752440  /* 1/sqrt(2) */
+#define CUMO_M_SQRT_2_OVER_PI 0.79788456080286535588  /* sqrt(2/pi) */
+#define CUMO_GELU_TANH_CUBIC  0.044715                /* fitted, not a closed form */
+#endif
+
 #endif // CUMO_FLOAT_DEF_KERNEL_H

data/ext/cumo/include/cumo/types/float_macro.h

@@ -1,5 +1,6 @@
 #include "float_def.h"
 #include "half_def.h"
+#include "bf16_def.h"
 
 extern double round(double);
 extern double log2(double);
@@ -19,6 +20,7 @@ extern double pow(double, double);
 #define m_from_double(x) (x)
 #define m_from_real(x) (x)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x) (x)
 #define m_from_int32(x) (x)
 #define m_from_int64(x) (x)
@@ -79,23 +81,6 @@ static inline dtype m_floored_mod(dtype x, dtype y) {
 
 #define m_sprintf(s,x) sprintf(s,"%g",x)
 
-#define cmp_prnan(a,b)                         \
-    ((qsort_cast(a)==qsort_cast(b)) ? 0 :      \
-     (qsort_cast(a) > qsort_cast(b)) ? 1 : -1)
-
-#define cmp_ignan(a,b)                                                  \
-    (m_isnan(qsort_cast(a)) ? (m_isnan(qsort_cast(b)) ? 0 : 1) :        \
-     (m_isnan(qsort_cast(b)) ? -1 :                                     \
-      ((qsort_cast(a)==qsort_cast(b)) ? 0 :                             \
-       (qsort_cast(a) > qsort_cast(b)) ? 1 : -1)))
-
-#define cmpgt_prnan(a,b)                        \
-    (qsort_cast(a) > qsort_cast(b))
-
-#define cmpgt_ignan(a,b)                                      \
-    ((m_isnan(qsort_cast(a)) && !m_isnan(qsort_cast(b))) ||   \
-     (qsort_cast(a) > qsort_cast(b)))
-
 #define m_sqrt(x)    sqrt(x)
 #define m_cbrt(x)    cbrt(x)
 #define m_log(x)     log(x)
@@ -129,6 +114,8 @@ static inline dtype m_floored_mod(dtype x, dtype y) {
 
 #define m_erf(x)     erf(x)
 #define m_erfc(x)    erfc(x)
+#define m_gelu(x)      cumo_gelu(x)
+#define m_gelu_tanh(x) cumo_gelu_tanh(x)
 #define m_ldexp(x,y) ldexp(x,y)
 #define m_frexp(x,exp) frexp(x,exp)
 
@@ -209,3 +196,18 @@ static inline dtype f_kahan_sum_nan(size_t n, char *p, ssize_t stride)
 }
 
 #include "real_accum.h"
+
+// Typed by the dtype so that a float array reaches erff and tanhf. A helper
+// taking double instead widens every element and runs the fp64 routines, which
+// this card answers an order of magnitude slower than the fp32 ones.
+static inline dtype cumo_gelu(dtype x)
+{
+    return (dtype)0.5 * x * ((dtype)1 + erf(x * (dtype)CUMO_M_SQRT1_2));
+}
+
+// The approximation GPT-2 and the transformers after it were trained with.
+static inline dtype cumo_gelu_tanh(dtype x)
+{
+    return (dtype)0.5 * x * ((dtype)1 + tanh((dtype)CUMO_M_SQRT_2_OVER_PI *
+                                             (x + (dtype)CUMO_GELU_TANH_CUBIC * x * x * x)));
+}

data/ext/cumo/include/cumo/types/float_macro_kernel.h

@@ -3,6 +3,7 @@
 
 #include "float_def_kernel.h"
 #include "half_def_kernel.h"
+#include "bf16_def_kernel.h"
 
 extern double round(double);
 extern double log2(double);
@@ -22,6 +23,7 @@ extern double pow(double, double);
 #define m_from_double(x) (x)
 #define m_from_real(x) (x)
 #define m_from_half(x) m_from_real(cumo_half2float(x))
+#define m_from_bfloat(x) m_from_real(cumo_bfloat2float(x))
 #define m_from_sint(x) (x)
 #define m_from_int32(x) (x)
 #define m_from_int64(x) (x)
@@ -82,25 +84,6 @@ __host__ __device__ static inline dtype m_floored_mod(dtype x, dtype y) {
 
 #define m_sprintf(s,x) sprintf(s,"%g",x)
 
-/*
-#define cmp_prnan(a,b)                         \
-    ((qsort_cast(a)==qsort_cast(b)) ? 0 :      \
-     (qsort_cast(a) > qsort_cast(b)) ? 1 : -1)
-
-#define cmp_ignan(a,b)                                                  \
-    (m_isnan(qsort_cast(a)) ? (m_isnan(qsort_cast(b)) ? 0 : 1) :        \
-     (m_isnan(qsort_cast(b)) ? -1 :                                     \
-      ((qsort_cast(a)==qsort_cast(b)) ? 0 :                             \
-       (qsort_cast(a) > qsort_cast(b)) ? 1 : -1)))
-
-#define cmpgt_prnan(a,b)                        \
-    (qsort_cast(a) > qsort_cast(b))
-
-#define cmpgt_ignan(a,b)                                      \
-    ((m_isnan(qsort_cast(a)) && !m_isnan(qsort_cast(b))) ||   \
-     (qsort_cast(a) > qsort_cast(b)))
-*/
-
 #define m_sqrt(x)    sqrt(x)
 #define m_cbrt(x)    cbrt(x)
 #define m_log(x)     log(x)
@@ -134,6 +117,8 @@ __host__ __device__ static inline dtype m_floored_mod(dtype x, dtype y) {
 
 #define m_erf(x)     erf(x)
 #define m_erfc(x)    erfc(x)
+#define m_gelu(x)      cumo_gelu(x)
+#define m_gelu_tanh(x) cumo_gelu_tanh(x)
 #define m_ldexp(x,y) ldexp(x,y)
 #define m_frexp(x,exp) frexp(x,exp)
 
@@ -172,4 +157,19 @@ __host__ __device__ static inline dtype f_seq(dtype x, dtype y, double c)
 
 #include "real_accum_kernel.h"
 
+// Typed by the dtype so that a float array reaches erff and tanhf. A helper
+// taking double instead widens every element and runs the fp64 routines, which
+// this card answers an order of magnitude slower than the fp32 ones.
+__host__ __device__ static inline dtype cumo_gelu(dtype x)
+{
+    return (dtype)0.5 * x * ((dtype)1 + erf(x * (dtype)CUMO_M_SQRT1_2));
+}
+
+// The approximation GPT-2 and the transformers after it were trained with.
+__host__ __device__ static inline dtype cumo_gelu_tanh(dtype x)
+{
+    return (dtype)0.5 * x * ((dtype)1 + tanh((dtype)CUMO_M_SQRT_2_OVER_PI *
+                                             (x + (dtype)CUMO_GELU_TANH_CUBIC * x * x * x)));
+}
+
 #endif // CUMO_FLOAT_MACRO_KERNEL_H