cumo 0.1.0

data/ext/cumo/include/cumo/narray_kernel.h

@@ -0,0 +1,149 @@
+#ifndef CUMO_NARRAY_KERNEL_H
+#define CUMO_NARRAY_KERNEL_H
+
+#if defined(__cplusplus)
+extern "C" {
+#if 0
+} /* satisfy cc-mode */
+#endif
+#endif
+
+#include <math.h>
+//#include "cumo/compat.h"
+#include "cumo/template_kernel.h"
+//#include "cumo/extconf.h"
+
+#ifdef HAVE_STDBOOL_H
+# include <stdbool.h>
+#endif
+
+#ifdef HAVE_STDINT_H
+# include <stdint.h>
+#endif
+
+#ifdef HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+
+#ifndef HAVE_U_INT8_T
+# ifdef HAVE_UINT8_T
+    typedef uint8_t u_int8_t;
+# endif
+#endif
+
+#ifndef HAVE_U_INT16_T
+# ifdef HAVE_UINT16_T
+    typedef uint16_t u_int16_t;
+# endif
+#endif
+
+#ifndef HAVE_U_INT32_T
+# ifdef HAVE_UINT32_T
+    typedef uint32_t u_int32_t;
+# endif
+#endif
+
+#ifndef HAVE_U_INT64_T
+# ifdef HAVE_UINT64_T
+    typedef uint64_t u_int64_t;
+# endif
+#endif
+
+#define SZF PRI_SIZE_PREFIX // defined in ruby.h
+
+#if   SIZEOF_LONG==8
+# define NUM2INT64(x) NUM2LONG(x)
+# define INT642NUM(x) LONG2NUM(x)
+# define NUM2UINT64(x) NUM2ULONG(x)
+# define UINT642NUM(x) ULONG2NUM(x)
+# ifndef PRId64
+#  define PRId64 "ld"
+# endif
+# ifndef PRIu64
+#  define PRIu64 "lu"
+# endif
+#elif SIZEOF_LONG_LONG==8
+# define NUM2INT64(x) NUM2LL(x)
+# define INT642NUM(x) LL2NUM(x)
+# define NUM2UINT64(x) NUM2ULL(x)
+# define UINT642NUM(x) ULL2NUM(x)
+# ifndef PRId64
+#  define PRId64 "lld"
+# endif
+# ifndef PRIu64
+#  define PRIu64 "llu"
+# endif
+#endif
+
+#if   SIZEOF_LONG==4
+# define NUM2INT32(x) NUM2LONG(x)
+# define INT322NUM(x) LONG2NUM(x)
+# define NUM2UINT32(x) NUM2ULONG(x)
+# define UINT322NUM(x) ULONG2NUM(x)
+# ifndef PRId32
+#  define PRId32 "ld"
+# endif
+# ifndef PRIu32
+#  define PRIu32 "lu"
+# endif
+#elif SIZEOF_INT==4
+# define NUM2INT32(x) NUM2INT(x)
+# define INT322NUM(x) INT2NUM(x)
+# define NUM2UINT32(x) NUM2UINT(x)
+# define UINT322NUM(x) UINT2NUM(x)
+# ifndef PRId32
+#  define PRId32 "d"
+# endif
+# ifndef PRIu32
+#  define PRIu32 "u"
+# endif
+#endif
+
+#ifndef HAVE_TYPE_BOOL
+  typedef int bool;
+#endif
+#ifndef FALSE                   /* in case these macros already exist */
+# define FALSE   0              /* values of bool */
+#endif
+#ifndef TRUE
+# define TRUE    1
+#endif
+
+typedef struct { float dat[2]; }  scomplex;
+typedef struct { double dat[2]; } dcomplex;
+typedef int fortran_integer;
+
+#define REAL(x) ((x).dat[0])
+#define IMAG(x) ((x).dat[1])
+
+extern int na_debug_flag;
+
+#define NARRAY_DATA_T     0x1
+#define NARRAY_VIEW_T     0x2
+#define NARRAY_FILEMAP_T  0x3
+
+//#define NA_MAX_DIMENSION (int)(sizeof(VALUE)*8-2)
+#define NA_MAX_DIMENSION 12
+#define NA_MAX_ELMSZ     65535
+
+typedef unsigned int BIT_DIGIT;
+#define BYTE_BIT_DIGIT sizeof(BIT_DIGIT)
+#define NB     (sizeof(BIT_DIGIT)*8)
+#define BALL   (~(BIT_DIGIT)0)
+#define SLB(n) (((n)==NB)?~(BIT_DIGIT)0:(~(~(BIT_DIGIT)0<<(n))))
+
+#define ELEMENT_BIT_SIZE  "ELEMENT_BIT_SIZE"
+#define ELEMENT_BYTE_SIZE "ELEMENT_BYTE_SIZE"
+#define CONTIGUOUS_STRIDE "CONTIGUOUS_STRIDE"
+
+#include "cumo/indexer.h"
+#include "cumo/intern_kernel.h"
+
+#if defined(__cplusplus)
+#if 0
+{ /* satisfy cc-mode */
+#endif
+}  /* extern "C" { */
+#endif
+
+#endif /* ifndef CUMO_NARRAY_KERNEL_H */

data/ext/cumo/include/cumo/ndloop.h

@@ -0,0 +1,95 @@
+#ifndef CUMO_NDLOOP_H
+#define CUMO_NDLOOP_H
+
+typedef struct NA_LOOP_ITER {
+    ssize_t    pos; // - required for each dimension.
+    ssize_t    step;
+    size_t    *idx;
+} na_loop_iter_t;
+
+typedef struct NA_LOOP_ARGS {
+    VALUE    value;
+    ssize_t  elmsz;
+    char    *ptr;
+    //char    *buf_ptr;  //
+    int      ndim;       // required for each argument.
+    // ssize_t pos; - not required here.
+    size_t  *shape;
+    na_loop_iter_t *iter;  // moved from na_loop_t
+} na_loop_args_t;
+
+// pass this structure to user iterator
+typedef struct NA_LOOP {
+    int  narg;
+    int  ndim;             // n of user dimention used at user function.
+    size_t *n;             // n of elements for each dim (=shape)
+    na_loop_args_t *args;  // for each arg
+    VALUE  option;
+    void  *opt_ptr;
+    VALUE  err_type;
+    int    reduce_dim;     // number of dimensions to reduce in reduction kernel, e.g., for an array of shape: [2,3,4],
+                           // 3 for sum(), 1 for sum(axis: 1), 2 for sum(axis: [1,2])
+    VALUE  reduce;         // dimension indicies to reduce in reduction kernel (in bits), e.g., for an array of shape:
+                           // [2,3,4], 111b for sum(), 010b for sum(axis: 1), 110b for sum(axis: [1,2])
+} na_loop_t;
+
+
+// ------------------ ndfunc -------------------------------------------
+
+#define NDF_HAS_LOOP            (1<<0) // x[i]
+#define NDF_STRIDE_LOOP         (1<<1) // *(x+stride*i)
+#define NDF_INDEX_LOOP          (1<<2) // *(x+idx[i])
+#define NDF_KEEP_DIM            (1<<3)
+#define NDF_INPLACE             (1<<4)
+#define NDF_ACCEPT_BYTESWAP     (1<<5)
+
+#define NDF_FLAT_REDUCE         (1<<6)
+#define NDF_EXTRACT             (1<<7)
+#define NDF_CUM                 (1<<8)
+
+#define NDF_INDEXER_LOOP        (1<<9) // Cumo custom. Use cumo own indexer.
+
+#define FULL_LOOP       (NDF_HAS_LOOP|NDF_STRIDE_LOOP|NDF_INDEX_LOOP|NDF_INPLACE)
+#define FULL_LOOP_NIP   (NDF_HAS_LOOP|NDF_STRIDE_LOOP|NDF_INDEX_LOOP)
+#define STRIDE_LOOP     (NDF_HAS_LOOP|NDF_STRIDE_LOOP|NDF_INPLACE)
+#define STRIDE_LOOP_NIP (NDF_HAS_LOOP|NDF_STRIDE_LOOP)
+#define NO_LOOP         0
+
+#define OVERWRITE Qtrue // used for CASTABLE(t)
+
+#define NDF_TEST(nf,fl)  ((nf)->flag & (fl))
+#define NDF_SET(nf,fl)  {(nf)->flag |= (fl);}
+
+#define NDF_ARG_READ_ONLY   1
+#define NDF_ARG_WRITE_ONLY  2
+#define NDF_ARG_READ_WRITE  3
+
+// type of user function
+typedef void (*na_iter_func_t) _((na_loop_t *const));
+typedef VALUE (*na_text_func_t) _((char *ptr, size_t pos, VALUE opt));
+//typedef void (*) void (*loop_func)(ndfunc_t*, na_md_loop_t*))
+
+
+typedef struct NDF_ARG_IN {
+    VALUE   type;    // argument types
+    int     dim;     // # of dimension of argument handled by user function
+                     // if dim==-1, reduce dimension
+} ndfunc_arg_in_t;
+
+typedef struct NDF_ARG_OUT {
+    VALUE   type;    // argument types
+    int     dim;     // # of dimension of argument handled by user function
+    size_t *shape;
+} ndfunc_arg_out_t;
+
+// spec of user function
+typedef struct NDFUNCTION {
+    na_iter_func_t func;    // user function
+    unsigned int flag;      // what kind of loop user function supports
+    int nin;                // # of arguments
+    int nout;               // # of results
+    ndfunc_arg_in_t *ain;   // spec of input arguments
+    ndfunc_arg_out_t *aout; // spec of output result
+} ndfunc_t;
+
+#endif /* CUMO_NDLOOP_H */

data/ext/cumo/include/cumo/reduce_kernel.h

@@ -0,0 +1,126 @@
+#ifndef CUMO_REDUCE_KERNEL_H
+#define CUMO_REDUCE_KERNEL_H
+
+#include <algorithm>
+#include <cstdint>
+#include <type_traits>
+
+#include "cumo/indexer.h"
+
+static inline int64_t round_up_to_power_of_2(int64_t x) {
+    --x;
+    x |= x >> 1;
+    x |= x >> 2;
+    x |= x >> 4;
+    x |= x >> 8;
+    x |= x >> 16;
+    x |= x >> 32;
+    return x + 1;
+}
+
+#define _REDUCE(offset) \
+    if (tid < offset) { \
+        impl.Reduce(sdata[(tid + offset)], sdata[tid]); \
+    }
+
+// reference: cupy reduction kernel
+
+template <typename TypeIn, typename TypeOut, typename ReductionImpl>
+__global__ static void reduction_kernel(na_reduction_arg_t arg, ReductionImpl impl) {
+    na_iarray_t& in_iarray = arg.in;
+    na_iarray_t& out_iarray = arg.out;
+    na_indexer_t& in_indexer = arg.in_indexer;
+    na_indexer_t& out_indexer = arg.out_indexer;
+    na_indexer_t& reduce_indexer = arg.reduce_indexer;
+
+    using TypeReduce = decltype(impl.Identity());
+
+    extern __shared__ __align__(8) char sdata_raw[];
+    TypeReduce* sdata = (TypeReduce*)sdata_raw;
+    unsigned int tid = threadIdx.x;
+    unsigned int block_size = blockDim.x;  // number of threads
+
+    for (uint64_t i_out = blockIdx.x; i_out < out_indexer.total_size; i_out += gridDim.x) {
+        cumo_na_indexer_set_dim(&out_indexer, i_out);
+        TypeReduce accum = impl.Identity();
+
+        for (int8_t i_out_dim = 0; i_out_dim < out_indexer.ndim; ++i_out_dim) {
+            in_indexer.index[i_out_dim] = out_indexer.index[i_out_dim];
+        }
+        for (auto i_reduce = tid; i_reduce < reduce_indexer.total_size; i_reduce += block_size) {
+            cumo_na_indexer_set_dim(&reduce_indexer, i_reduce);
+            for (int8_t i_reduce_dim = 0; i_reduce_dim < reduce_indexer.ndim; ++i_reduce_dim) {
+                in_indexer.index[out_indexer.ndim + i_reduce_dim] = reduce_indexer.index[i_reduce_dim];
+            }
+            TypeIn* in_ptr = reinterpret_cast<TypeIn*>(cumo_na_iarray_at_dim(&in_iarray, &in_indexer));
+            uint64_t i_in = in_ptr - reinterpret_cast<TypeIn*>(in_iarray.ptr);
+            impl.Reduce(impl.MapIn(*in_ptr, i_in), accum);
+        }
+
+        if (block_size >= 2) {
+            sdata[tid] = accum;
+            __syncthreads();
+
+            if (block_size > 2) {
+                if (block_size > 4) {
+                    if (block_size > 8) {
+                        if (block_size > 16) {
+                            if (block_size > 32) {
+                                if (block_size > 64) {
+                                    if (block_size > 128) {
+                                        if (block_size > 256) {
+                                            _REDUCE(256);
+                                            __syncthreads();
+                                        }
+                                        _REDUCE(128);
+                                        __syncthreads();
+                                    }
+                                    _REDUCE(64);
+                                    __syncthreads();
+                                }
+                                _REDUCE(32);
+                                __syncthreads();
+                            }
+                            _REDUCE(16);
+                            __syncthreads();
+                        }
+                        _REDUCE(8);
+                        __syncthreads();
+                    }
+                    _REDUCE(4);
+                    __syncthreads();
+                }
+                _REDUCE(2);
+                __syncthreads();
+            }
+            _REDUCE(1);
+            accum = sdata[0];
+        }
+        if (tid == 0) {
+            TypeOut* out_ptr = reinterpret_cast<TypeOut*>(cumo_na_iarray_at_dim(&out_iarray, &out_indexer));
+            *out_ptr = impl.MapOut(accum);
+            //printf("threadId.x:%d blockIdx.x:%d blockDim.x:%d gridDim.x:%d block_size:%d accum:%d out:%p(%d)\n", threadIdx.x, blockIdx.x, blockDim.x, gridDim.x, block_size, accum, out_ptr, *out_ptr);
+        }
+    }
+}
+
+#undef _REDUCE
+
+static constexpr size_t max_block_size = 512;
+
+template <typename TypeIn, typename TypeOut, typename ReductionImpl>
+void cumo_reduce(na_reduction_arg_t arg, ReductionImpl&& impl) {
+    na_indexer_t& out_indexer = arg.out_indexer;
+    na_indexer_t& reduce_indexer = arg.reduce_indexer;
+
+    using TypeReduce = decltype(impl.Identity());
+
+    size_t block_size = round_up_to_power_of_2(std::max(int64_t{1}, static_cast<int64_t>(reduce_indexer.total_size)));
+    block_size = std::min(max_block_size, block_size);
+    size_t grid_size = out_indexer.total_size;
+    size_t shared_mem_size = sizeof(TypeReduce) * block_size;
+
+    reduction_kernel<TypeIn,TypeOut,ReductionImpl><<<grid_size, block_size, shared_mem_size>>>(arg, impl);
+}
+
+#endif // CUMO_REDUCE_KERNEL_H

data/ext/cumo/include/cumo/template.h

@@ -0,0 +1,158 @@
+#ifndef CUMO_TEMPLATE_H
+#define CUMO_TEMPLATE_H
+
+#define INIT_COUNTER( lp, c )                   \
+    {   c = (lp)->n[0]; }
+
+#define NDL_CNT(lp) ((lp)->n[0])
+#define NDL_PTR(lp,i) ((lp)->args[i].ptr + (lp)->args[i].iter[0].pos)
+#define NDL_STEP(lp,i) ((lp)->args[i].iter[0].step)
+#define NDL_IDX(lp,i) ((lp)->args[i].iter[0].idx)
+#define NDL_ESZ(lp,i) ((lp)->args[i].elmsz)
+#define NDL_SHAPE(lp,i) ((lp)->args[i].shape)
+
+#define INIT_PTR( lp, i, pt, st )                               \
+    {                                                           \
+        pt = ((lp)->args[i]).ptr + ((lp)->args[i].iter[0]).pos;         \
+        st = ((lp)->args[i].iter[0]).step;                              \
+    }
+
+#define INIT_PTR_IDX( lp, i, pt, st, id )                       \
+    {                                                           \
+        pt = ((lp)->args[i]).ptr + ((lp)->args[i].iter[0]).pos;         \
+        st = ((lp)->args[i].iter[0]).step;                              \
+        id = ((lp)->args[i].iter[0]).idx;                               \
+    }
+
+#define INIT_ELMSIZE( lp, i, es )                               \
+    {                                                           \
+        es = ((lp)->args[i]).elmsz;                             \
+    }
+
+#define INIT_PTR_BIT( lp, i, ad, ps, st )               \
+    {                                                   \
+        ps = ((lp)->args[i].iter[0]).pos;                       \
+        ad = (BIT_DIGIT*)(((lp)->args[i]).ptr) + ps/NB; \
+        ps %= NB;                                       \
+        st = ((lp)->args[i].iter[0]).step;                      \
+    }
+
+#define INIT_PTR_BIT_IDX( lp, i, ad, ps, st, id )       \
+    {                                                   \
+        ps = ((lp)->args[i].iter[0]).pos;                       \
+        ad = (BIT_DIGIT*)(((lp)->args[i]).ptr) + ps/NB; \
+        ps %= NB;                                       \
+        st = ((lp)->args[i].iter[0]).step;                      \
+        id = ((lp)->args[i].iter[0]).idx;                       \
+    }
+
+#define GET_DATA( ptr, type, val )                 \
+    {                                              \
+        val = *(type*)(ptr);                       \
+    }
+
+#define SET_DATA( ptr, type, val )                 \
+    {                                              \
+        *(type*)(ptr) = val;                       \
+    }
+
+#define GET_DATA_STRIDE( ptr, step, type, val )    \
+    {                                              \
+        val = *(type*)(ptr);                       \
+        ptr += step;                               \
+    }
+
+#define GET_DATA_INDEX( ptr, idx, type, val )     \
+    {                                           \
+        val = *(type*)(ptr + *idx);             \
+        idx++;                                  \
+    }
+
+#define SET_DATA_STRIDE( ptr, step, type, val ) \
+    {                                           \
+        *(type*)(ptr) = val;                    \
+        ptr += step;                            \
+    }
+
+#define SET_DATA_INDEX( ptr, idx, type, val )   \
+    {                                           \
+        *(type*)(ptr + *idx) = val;             \
+        idx++;                                  \
+    }
+
+#define LOAD_BIT( adr, pos, val )                       \
+    {                                                   \
+        size_t dig = (pos) / NB;                        \
+        int    bit = (pos) % NB;                        \
+        val = (((BIT_DIGIT*)(adr))[dig]>>(bit)) & 1u;   \
+    }
+
+#define LOAD_BIT_STEP( adr, pos, step, idx, val )       \
+    {                                                   \
+        size_t dig; int bit;                            \
+        if (idx) {                                      \
+            dig = ((pos) + *(idx)) / NB;                \
+            bit = ((pos) + *(idx)) % NB;                \
+            idx++;                                      \
+        } else {                                        \
+            dig = (pos) / NB;                           \
+            bit = (pos) % NB;                           \
+            pos += step;                                \
+        }                                               \
+        val = (((BIT_DIGIT*)(adr))[dig]>>bit) & 1u;     \
+    }
+
+#define STORE_BIT(adr,pos,val)                  \
+    {                                           \
+        size_t dig = (pos) / NB;                \
+        int    bit = (pos) % NB;                \
+        ((BIT_DIGIT*)(adr))[dig] =              \
+            (((BIT_DIGIT*)(adr))[dig] & ~(1u<<(bit))) | ((val)<<(bit)); \
+    }
+// val -> val&1 ??
+
+#define STORE_BIT_STEP( adr, pos, step, idx, val )\
+    {                                           \
+        size_t dig; int bit;                    \
+        if (idx) {                              \
+            dig = ((pos) + *(idx)) / NB;        \
+            bit = ((pos) + *(idx)) % NB;        \
+            idx++;                              \
+        } else {                                \
+            dig = (pos) / NB;                   \
+            bit = (pos) % NB;                   \
+            pos += step;                        \
+        }                                       \
+        ((BIT_DIGIT*)(adr))[dig] =              \
+            (((BIT_DIGIT*)(adr))[dig] & ~(1u<<(bit))) | ((val)<<(bit)); \
+    }
+// val -> val&1 ??
+
+static inline int
+is_aligned(const void *ptr, const size_t alignment)
+{
+    return ((size_t)(ptr) & ((alignment)-1)) == 0;
+}
+
+static inline int
+is_aligned_step(const ssize_t step, const size_t alignment)
+{
+    return ((step) & ((alignment)-1)) == 0;
+}
+
+#define SHOW_WARNING_ONCE( c_str ) \
+    { \
+        static bool show_warning = true; \
+        if (show_warning) { \
+            fprintf(stderr, (c_str)); \
+            show_warning = false; \
+        } \
+    }
+
+#define SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE( func_name, type_name ) \
+    SHOW_WARNING_ONCE("Warning: FIXME: Method \"" func_name "\" for dtype \"" type_name "\" synchronizes with CPU.\n")
+
+#define SHOW_SYNCHRONIZE_WARNING_ONCE( func_name, type_name ) \
+    SHOW_WARNING_ONCE("Warning: Method \"" func_name "\" for dtype \"" type_name "\" synchronizes with CPU.\n")
+
+#endif /* ifndef CUMO_TEMPLATE_H */