cumo 0.1.0

data/ext/cumo/narray/gen/tmpl/sort_index.c

@@ -0,0 +1,111 @@
+<% (is_float ? ["_ignan","_prnan"] : [""]).each do |j|
+   [64,32].each do |i| %>
+#define idx_t int<%=i%>_t
+static void
+<%=type_name%>_index<%=i%>_qsort<%=j%>(na_loop_t *const lp)
+{
+    size_t   i, n, idx;
+    char    *d_ptr, *i_ptr, *o_ptr;
+    ssize_t  d_step, i_step, o_step;
+    char   **ptr;
+
+    INIT_COUNTER(lp, n);
+    INIT_PTR(lp, 0, d_ptr, d_step);
+    INIT_PTR(lp, 1, i_ptr, i_step);
+    INIT_PTR(lp, 2, o_ptr, o_step);
+
+    ptr = (char**)(lp->opt_ptr);
+
+    //printf("(ptr=%lx, d_ptr=%lx,d_step=%ld, i_ptr=%lx,i_step=%ld, o_ptr=%lx,o_step=%ld)\n",(size_t)ptr,(size_t)d_ptr,(ssize_t)d_step,(size_t)i_ptr,(ssize_t)i_step,(size_t)o_ptr,(ssize_t)o_step);
+
+    if (n==1) {
+        *(idx_t*)o_ptr = *(idx_t*)(i_ptr);
+        return;
+    }
+
+    for (i=0; i<n; i++) {
+        ptr[i] = d_ptr + d_step * i;
+        //printf("(%ld,%.3f)",i,*(double*)ptr[i]);
+    }
+
+    <%=type_name%>_index_qsort<%=j%>(ptr, n, sizeof(dtype*));
+
+    //d_ptr = lp->args[0].ptr;
+    //printf("(d_ptr=%lx)\n",(size_t)d_ptr);
+
+    for (i=0; i<n; i++) {
+        idx = (ptr[i] - d_ptr) / d_step;
+        *(idx_t*)o_ptr = *(idx_t*)(i_ptr + i_step * idx);
+        //printf("(idx[%ld]=%ld,%d)",i,idx,*(idx_t*)o_ptr);
+        o_ptr += o_step;
+    }
+    //printf("\n");
+}
+#undef idx_t
+<% end;end %>
+
+/*
+  <%=name%>. Returns an index array of sort result.
+<% if is_float %>
+  @overload <%=name%>(axis:nil, nan:false)
+  @param [TrueClass] nan  If true, propagete NaN. If false, ignore NaN.
+<% else %>
+  @overload <%=name%>(axis:nil)
+<% end %>
+  @param [Numeric,Array,Range] axis  Affected dimensions.
+  @return [Integer,Cumo::Int] returns result index of <%=name%>.
+  @example
+      Cumo::NArray[3,4,1,2].sort_index => Cumo::Int32[2,3,0,1]
+*/
+static VALUE
+<%=c_func(-1)%>(int argc, VALUE *argv, VALUE self)
+{
+    size_t size;
+    narray_t *na;
+    VALUE idx, tmp, reduce, res;
+    char *buf;
+    ndfunc_arg_in_t ain[3] = {{cT,0},{0,0},{sym_reduce,0}};
+    ndfunc_arg_out_t aout[1] = {{0,0,0}};
+    ndfunc_t ndf = {0, STRIDE_LOOP_NIP|NDF_FLAT_REDUCE|NDF_CUM, 3,1, ain,aout};
+
+    GetNArray(self,na);
+    if (na->ndim==0) {
+        return INT2FIX(0);
+    }
+    if (na->size > (~(u_int32_t)0)) {
+        ain[1].type =
+        aout[0].type = cumo_cInt64;
+        idx = nary_new(cumo_cInt64, na->ndim, na->shape);
+       <% if is_float %>
+         ndf.func = <%=type_name%>_index64_qsort_ignan;
+         reduce = na_reduce_dimension(argc, argv, 1, &self, &ndf,
+                                      <%=type_name%>_index64_qsort_prnan);
+       <% else %>
+         ndf.func = <%=type_name%>_index64_qsort;
+         reduce = na_reduce_dimension(argc, argv, 1, &self, &ndf, 0);
+       <% end %>
+    } else {
+        ain[1].type =
+        aout[0].type = cumo_cInt32;
+        idx = nary_new(cumo_cInt32, na->ndim, na->shape);
+       <% if is_float %>
+         ndf.func = <%=type_name%>_index32_qsort_ignan;
+         reduce = na_reduce_dimension(argc, argv, 1, &self, &ndf,
+                                      <%=type_name%>_index32_qsort_prnan);
+       <% else %>
+         ndf.func = <%=type_name%>_index32_qsort;
+         reduce = na_reduce_dimension(argc, argv, 1, &self, &ndf, 0);
+       <% end %>
+    }
+    rb_funcall(idx, rb_intern("seq"), 0);
+
+    size = na->size*sizeof(void*); // max capa
+    buf = rb_alloc_tmp_buffer(&tmp, size);
+
+    SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
+    cudaDeviceSynchronize();
+
+    res = na_ndloop3(&ndf, buf, 3, self, idx, reduce);
+    rb_free_tmp_buffer(&tmp);
+    return res;
+}

data/ext/cumo/narray/gen/tmpl/store.c

@@ -0,0 +1,41 @@
+<% children.each do |c|%>
+<%= c.result %>
+
+<% end %>
+/*
+  Store elements to Cumo::<%=class_name%> from other.
+  @overload store(other)
+  @param [Object] other
+  @return [Cumo::<%=class_name%>] self
+*/
+static VALUE
+<%=c_func(1)%>(VALUE self, VALUE obj)
+{
+    VALUE r, klass;
+
+    klass = CLASS_OF(obj);
+
+    <% definitions.each do |x| %>
+    if (<%=x.condition("klass")%>) {
+        <%=x.c_func%>(self,obj);
+        return self;
+    }
+    <% end %>
+
+    if (IsNArray(obj)) {
+        r = rb_funcall(obj, rb_intern("coerce_cast"), 1, cT);
+        if (CLASS_OF(r)==cT) {
+            <%=c_func%>(self,r);
+            return self;
+        }
+    }
+
+    <% if is_object %>
+    robject_store_numeric(self,obj);
+    <% else %>
+    rb_raise(nary_eCastError, "unknown conversion from %s to %s",
+             rb_class2name(CLASS_OF(obj)),
+             rb_class2name(CLASS_OF(self)));
+    <% end %>
+    return self;
+}

data/ext/cumo/narray/gen/tmpl/store_array.c

@@ -0,0 +1,187 @@
+//<% unless c_iter.include? 'robject' %>
+void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, dtype* z, uint64_t n);
+void <%="cumo_#{c_iter}_stride_kernel_launch"%>(char *p1, ssize_t s1, dtype* z, uint64_t n);
+void <%="cumo_#{c_iter}_index_scalar_kernel_launch"%>(char *p1, size_t *idx1, dtype z, uint64_t n);
+void <%="cumo_#{c_iter}_stride_scalar_kernel_launch"%>(char *p1, ssize_t s1, dtype z, uint64_t n);
+
+static void CUDART_CB
+<%=c_iter%>_callback(cudaStream_t stream, cudaError_t status, void *data)
+{
+    xfree(data);
+}
+//<% end %>
+
+static void
+<%=c_iter%>(na_loop_t *const lp)
+{
+    size_t i, n;
+    size_t i1, n1;
+    VALUE  v1, *ptr;
+    char   *p1;
+    size_t s1, *idx1;
+    VALUE  x;
+    double y;
+    dtype  z;
+    size_t len, c;
+    double beg, step;
+
+    INIT_COUNTER(lp, n);
+    INIT_PTR_IDX(lp, 0, p1, s1, idx1);
+    v1 = lp->args[1].value;
+    i = 0;
+
+    if (lp->args[1].ptr) {
+        if (v1 == Qtrue) {
+            iter_<%=type_name%>_store_<%=type_name%>(lp);
+            i = lp->args[1].shape[0];
+            if (idx1) {
+                idx1 += i;
+            } else {
+                p1 += s1 * i;
+            }
+        }
+        goto loop_end;
+    }
+
+    ptr = &v1;
+
+    switch(TYPE(v1)) {
+    case T_ARRAY:
+        n1 = RARRAY_LEN(v1);
+        ptr = RARRAY_PTR(v1);
+        break;
+    case T_NIL:
+        n1 = 0;
+        break;
+    default:
+        n1 = 1;
+    }
+
+    //<% if c_iter.include? 'robject' %>
+    {
+        SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("store_<%=name%>", "<%=type_name%>");
+        cumo_cuda_runtime_check_status(cudaDeviceSynchronize());
+
+        if (idx1) {
+            for (i=i1=0; i1<n1 && i<n; i++,i1++) {
+                x = ptr[i1];
+                if (rb_obj_is_kind_of(x, rb_cRange) || rb_obj_is_kind_of(x, na_cStep)) {
+                    nary_step_sequence(x,&len,&beg,&step);
+                    for (c=0; c<len && i<n; c++,i++) {
+                        y = beg + step * c;
+                        z = m_from_double(y);
+                        SET_DATA_INDEX(p1, idx1, dtype, z);
+                    }
+                }
+                else if (TYPE(x) != T_ARRAY) {
+                    z = m_num_to_data(x);
+                    SET_DATA_INDEX(p1, idx1, dtype, z);
+                }
+            }
+        } else {
+            for (i=i1=0; i1<n1 && i<n; i++,i1++) {
+                x = ptr[i1];
+                if (rb_obj_is_kind_of(x, rb_cRange) || rb_obj_is_kind_of(x, na_cStep)) {
+                    nary_step_sequence(x,&len,&beg,&step);
+                    for (c=0; c<len && i<n; c++,i++) {
+                        y = beg + step * c;
+                        z = m_from_double(y);
+                        SET_DATA_STRIDE(p1, s1, dtype, z);
+                    }
+                }
+                else if (TYPE(x) != T_ARRAY) {
+                    z = m_num_to_data(x);
+                    SET_DATA_STRIDE(p1, s1, dtype, z);
+                }
+            }
+        }
+    }
+    //<% else %>
+    {
+        // To copy ruby non-contiguous array values into cuda memory asynchronously, we do
+        // 1. copy to contiguous heap memory
+        // 2. copy to contiguous device memory
+        // 3. launch kernel to copy the contiguous device memory into strided (or indexed) narray cuda memory
+        // 4. free the contiguous device memory
+        // 5. run callback to free the heap memory after kernel finishes
+        //
+        // FYI: We may have to care of cuda stream callback serializes stream execution when we support stream.
+        // https://devtalk.nvidia.com/default/topic/822942/why-does-cudastreamaddcallback-serialize-kernel-execution-and-break-concurrency-/
+        dtype* host_z = ALLOC_N(dtype, n);
+        for (i=i1=0; i1<n1 && i<n; i1++) {
+            x = ptr[i1];
+            if (rb_obj_is_kind_of(x, rb_cRange) || rb_obj_is_kind_of(x, na_cStep)) {
+                nary_step_sequence(x,&len,&beg,&step);
+                for (c=0; c<len && i<n; c++,i++) {
+                    y = beg + step * c;
+                    host_z[i] = m_from_double(y);
+                }
+            }
+            else if (TYPE(x) != T_ARRAY) {
+                host_z[i] = m_num_to_data(x);
+                i++;
+            }
+        }
+
+        if (!idx1 && s1 == sizeof(dtype)) {
+            // optimization: Since p1 is contiguous, we skip creating another contiguous device memory
+            cudaError_t status = cudaMemcpyAsync(p1,host_z,sizeof(dtype)*i,cudaMemcpyHostToDevice,0);
+            if (status == 0) {
+                cumo_cuda_runtime_check_status(cudaStreamAddCallback(0,<%=c_iter%>_callback,host_z,0));
+            } else {
+                xfree(host_z);
+            }
+            cumo_cuda_runtime_check_status(status);
+        } else {
+            dtype* device_z = (dtype*)cumo_cuda_runtime_malloc(sizeof(dtype) * n);
+            cudaError_t status = cudaMemcpyAsync(device_z,host_z,sizeof(dtype)*i,cudaMemcpyHostToDevice,0);
+            if (status == 0) {
+                if (idx1) {
+                    <%="cumo_#{c_iter}_index_kernel_launch"%>(p1,idx1,device_z,i);
+                } else {
+                    <%="cumo_#{c_iter}_stride_kernel_launch"%>(p1,s1,device_z,i);
+                }
+                cumo_cuda_runtime_check_status(cudaStreamAddCallback(0,<%=c_iter%>_callback,host_z,0));
+            } else {
+                xfree(host_z);
+            }
+            cumo_cuda_runtime_free((void*)device_z);
+            cumo_cuda_runtime_check_status(status);
+        }
+    }
+    //<% end %>
+
+ loop_end:
+    z = m_zero;
+    //<% if c_iter.include? 'robject' %>
+    {
+        if (idx1) {
+            for (; i<n; i++) {
+                SET_DATA_INDEX(p1, idx1, dtype, z);
+            }
+        } else {
+            for (; i<n; i++) {
+                SET_DATA_STRIDE(p1, s1, dtype, z);
+            }
+        }
+    }
+    //<% else %>
+    {
+        if (idx1) {
+            <%="cumo_#{c_iter}_index_scalar_kernel_launch"%>(p1,idx1+i,z,n-i);
+        } else {
+            <%="cumo_#{c_iter}_stride_scalar_kernel_launch"%>(p1+s1*i,s1,z,n-i);
+        }
+    }
+    //<% end %>
+}
+
+static VALUE
+<%=c_func%>(VALUE self, VALUE rary)
+{
+    ndfunc_arg_in_t ain[2] = {{OVERWRITE,0},{rb_cArray,0}};
+    ndfunc_t ndf = {<%=c_iter%>, FULL_LOOP, 2, 0, ain, 0};
+
+    na_ndloop_store_rarray(&ndf, self, rary);
+    return self;
+}

data/ext/cumo/narray/gen/tmpl/store_array_kernel.cu

@@ -0,0 +1,58 @@
+<% unless c_iter.include? 'robject' %>
+__global__ void <%="cumo_#{c_iter}_index_kernel"%>(char *p1, size_t *idx1, dtype* z, uint64_t n)
+{
+    for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) {
+        *(dtype*)(p1 + idx1[i]) = z[i];
+    }
+}
+
+__global__ void <%="cumo_#{c_iter}_stride_kernel"%>(char *p1, ssize_t s1, dtype* z, uint64_t n)
+{
+    for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) {
+        *(dtype*)(p1 + (i * s1)) = z[i];
+    }
+}
+
+__global__ void <%="cumo_#{c_iter}_index_scalar_kernel"%>(char *p1, size_t *idx1, dtype z, uint64_t n)
+{
+    for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) {
+        *(dtype*)(p1 + idx1[i]) = z;
+    }
+}
+
+__global__ void <%="cumo_#{c_iter}_stride_scalar_kernel"%>(char *p1, ssize_t s1, dtype z, uint64_t n)
+{
+    for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) {
+        *(dtype*)(p1 + (i * s1)) = z;
+    }
+}
+
+void <%="cumo_#{c_iter}_index_kernel_launch"%>(char *p1, size_t *idx1, dtype* z, uint64_t n)
+{
+    size_t gridDim = get_gridDim(n);
+    size_t blockDim = get_blockDim(n);
+    <%="cumo_#{c_iter}_index_kernel"%><<<gridDim, blockDim>>>(p1,idx1,z,n);
+}
+
+void <%="cumo_#{c_iter}_stride_kernel_launch"%>(char *p1, ssize_t s1, dtype* z, uint64_t n)
+{
+    size_t gridDim = get_gridDim(n);
+    size_t blockDim = get_blockDim(n);
+    <%="cumo_#{c_iter}_stride_kernel"%><<<gridDim, blockDim>>>(p1,s1,z,n);
+}
+
+void <%="cumo_#{c_iter}_index_scalar_kernel_launch"%>(char *p1, size_t *idx1, dtype z, uint64_t n)
+{
+    size_t gridDim = get_gridDim(n);
+    size_t blockDim = get_blockDim(n);
+    <%="cumo_#{c_iter}_index_scalar_kernel"%><<<gridDim, blockDim>>>(p1,idx1,z,n);
+}
+
+void <%="cumo_#{c_iter}_stride_scalar_kernel_launch"%>(char *p1, ssize_t s1, dtype z, uint64_t n)
+{
+    size_t gridDim = get_gridDim(n);
+    size_t blockDim = get_blockDim(n);
+    <%="cumo_#{c_iter}_stride_scalar_kernel"%><<<gridDim, blockDim>>>(p1,s1,z,n);
+}
+
+<% end %>

data/ext/cumo/narray/gen/tmpl/store_bit.c

@@ -0,0 +1,86 @@
+//<% unless c_iter.include? 'robject' %>
+void <%="cumo_#{c_iter}_index_index_kernel_launch"%>(char *p1, size_t p2, BIT_DIGIT *a2, size_t *idx1, size_t *idx2, uint64_t n);
+void <%="cumo_#{c_iter}_stride_index_kernel_launch"%>(char *p1, size_t p2, BIT_DIGIT *a2, ssize_t s1, size_t *idx2, uint64_t n);
+void <%="cumo_#{c_iter}_index_stride_kernel_launch"%>(char *p1, size_t p2, BIT_DIGIT *a2, size_t *idx1, ssize_t s2, uint64_t n);
+void <%="cumo_#{c_iter}_stride_stride_kernel_launch"%>(char *p1, size_t p2, BIT_DIGIT *a2, ssize_t s1, ssize_t s2, uint64_t n);
+//<% end %>
+
+static void
+<%=c_iter%>(na_loop_t *const lp)
+{
+    size_t     i;
+    char      *p1;
+    size_t     p2;
+    ssize_t    s1, s2;
+    size_t    *idx1, *idx2;
+    BIT_DIGIT *a2;
+
+    INIT_COUNTER(lp, i);
+    INIT_PTR_IDX(lp, 0, p1, s1, idx1);
+    INIT_PTR_BIT_IDX(lp, 1, a2, p2, s2, idx2);
+
+    //<% if c_iter.include? 'robject' %>
+    {
+        BIT_DIGIT x;
+        dtype y;
+        SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>");
+        SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
+        if (idx2) {
+            if (idx1) {
+                for (; i--;) {
+                    LOAD_BIT(a2, p2+*idx2, x); idx2++;
+                    y = m_from_sint(x);
+                    SET_DATA_INDEX(p1,idx1,dtype,y);
+                }
+            } else {
+                for (; i--;) {
+                    LOAD_BIT(a2, p2+*idx2, x); idx2++;
+                    y = m_from_sint(x);
+                    SET_DATA_STRIDE(p1,s1,dtype,y);
+                }
+            }
+        } else {
+            if (idx1) {
+                for (; i--;) {
+                    LOAD_BIT(a2, p2, x); p2 += s2;
+                    y = m_from_sint(x);
+                    SET_DATA_INDEX(p1,idx1,dtype,y);
+                }
+            } else {
+                for (; i--;) {
+                    LOAD_BIT(a2, p2, x); p2 += s2;
+                    y = m_from_sint(x);
+                    SET_DATA_STRIDE(p1,s1,dtype,y);
+                }
+            }
+        }
+    }
+    //<% else %>
+    {
+        if (idx2) {
+            if (idx1) {
+                <%="cumo_#{c_iter}_index_index_kernel_launch"%>(p1,p2,a2,idx1,idx2,i);
+            } else {
+                <%="cumo_#{c_iter}_stride_index_kernel_launch"%>(p1,p2,a2,s1,idx2,i);
+            }
+        } else {
+            if (idx1) {
+                <%="cumo_#{c_iter}_index_stride_kernel_launch"%>(p1,p2,a2,idx1,s2,i);
+            } else {
+                <%="cumo_#{c_iter}_stride_stride_kernel_launch"%>(p1,p2,a2,s1,s2,i);
+            }
+        }
+    }
+    //<% end %>
+}
+
+
+static VALUE
+<%=c_func(:nodef)%>(VALUE self, VALUE obj)
+{
+    ndfunc_arg_in_t ain[2] = {{OVERWRITE,0},{Qnil,0}};
+    ndfunc_t ndf = {<%=c_iter%>, FULL_LOOP, 2,0, ain,0};
+
+    na_ndloop(&ndf, 2, self, obj);
+    return self;
+}