cumo 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -1
- data/README.md +24 -0
- data/Rakefile +32 -1
- data/ext/cumo/cuda/cub.cu +26 -0
- data/ext/cumo/extconf.rb +1 -1
- data/ext/cumo/include/cumo/cuda/runtime.h +3 -2
- data/ext/cumo/include/cumo/ndloop.h +7 -0
- data/ext/cumo/{narray/sort_kernel.cu → include/cumo/sort_kernel.h} +15 -15
- data/ext/cumo/include/cumo/template_kernel.h +2 -0
- data/ext/cumo/narray/data.c +3 -0
- data/ext/cumo/narray/gen/cogen_kernel.rb +19 -10
- data/ext/cumo/narray/gen/kernel_parts.rb +5 -3
- data/ext/cumo/narray/gen/spec.rb +4 -0
- data/ext/cumo/narray/gen/tmpl/accum_arg_kernel.cu +2 -2
- data/ext/cumo/narray/gen/tmpl/accum_binary_kernel.cu +1 -1
- data/ext/cumo/narray/gen/tmpl/accum_index_kernel.cu +2 -2
- data/ext/cumo/narray/gen/tmpl/binary.c +50 -0
- data/ext/cumo/narray/gen/tmpl/binary_kernel.cu +24 -0
- data/ext/cumo/narray/gen/tmpl/complex_accum_kernel.cu +17 -3
- data/ext/cumo/narray/gen/tmpl/cum_kernel.cu +2 -0
- data/ext/cumo/narray/gen/tmpl/float_accum_kernel.cu +15 -3
- data/ext/cumo/narray/gen/tmpl/map_with_index.c +18 -0
- data/ext/cumo/narray/gen/tmpl/real_accum_kernel.cu +19 -1
- data/ext/cumo/narray/gen/tmpl/sort_kernel.cu +19 -0
- data/ext/cumo/narray/gen/tmpl/unary.c +8 -0
- data/ext/cumo/narray/index.c +98 -82
- data/ext/cumo/narray/narray.c +10 -0
- data/ext/cumo/narray/ndloop.c +141 -16
- data/ext/cumo/narray/step.c +0 -2
- data/lib/cumo/version.rb +1 -1
- metadata +5 -3
|
@@ -162,6 +162,7 @@ struct <%="cumo_thrust_#{name}#{j}"%>
|
|
|
162
162
|
template<typename Iterator1, typename Iterator2>
|
|
163
163
|
static cudaError_t <%="cumo_#{type_name}_#{name}#{j}_scan_by_key"%>(Iterator1 first, Iterator1 last, Iterator2 result, uint64_t row_len)
|
|
164
164
|
{
|
|
165
|
+
cumo_cuda_cub_prime_ptx_version();
|
|
165
166
|
cumo_thrust_pool_allocator alloc;
|
|
166
167
|
<%="cumo_thrust_#{name}_row"%> row = {row_len};
|
|
167
168
|
auto keys = thrust::make_transform_iterator(thrust::make_counting_iterator<uint64_t>(0), row);
|
|
@@ -190,6 +191,7 @@ static cudaError_t <%="cumo_#{type_name}_#{name}#{j}_scan_by_key"%>(Iterator1 fi
|
|
|
190
191
|
template<typename Iterator1, typename Iterator2>
|
|
191
192
|
static cudaError_t <%="cumo_#{type_name}_#{name}#{j}_scan"%>(Iterator1 first, Iterator1 last, Iterator2 result)
|
|
192
193
|
{
|
|
194
|
+
cumo_cuda_cub_prime_ptx_version();
|
|
193
195
|
cumo_thrust_pool_allocator alloc;
|
|
194
196
|
try {
|
|
195
197
|
<% if widen %>
|
|
@@ -181,23 +181,29 @@ extern "C" {
|
|
|
181
181
|
#endif
|
|
182
182
|
#endif
|
|
183
183
|
|
|
184
|
-
<% if kernel_part? %>
|
|
184
|
+
<% if kernel_part?("mean") %>
|
|
185
185
|
void cumo_<%=type_name%>_mean_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
186
186
|
{
|
|
187
187
|
<%=acc%> n = (<%=acc%>)(arg->in_indexer.total_size / arg->out_indexer.total_size);
|
|
188
188
|
cumo_reduce_split<dtype, dtype, cumo_<%=type_name%>_mean_impl>(*arg, cumo_<%=type_name%>_mean_impl{n});
|
|
189
189
|
}
|
|
190
|
+
<% end %>
|
|
190
191
|
|
|
192
|
+
<% if kernel_part?("var") %>
|
|
191
193
|
void cumo_<%=type_name%>_var_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
192
194
|
{
|
|
193
195
|
cumo_reduce_split<dtype, rtype, cumo_<%=type_name%>_var_impl>(*arg, cumo_<%=type_name%>_var_impl{});
|
|
194
196
|
}
|
|
197
|
+
<% end %>
|
|
195
198
|
|
|
199
|
+
<% if kernel_part?("stddev") %>
|
|
196
200
|
void cumo_<%=type_name%>_stddev_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
197
201
|
{
|
|
198
202
|
cumo_reduce_split<dtype, rtype, cumo_<%=type_name%>_stddev_impl>(*arg, cumo_<%=type_name%>_stddev_impl{});
|
|
199
203
|
}
|
|
204
|
+
<% end %>
|
|
200
205
|
|
|
206
|
+
<% if kernel_part?("rms") %>
|
|
201
207
|
void cumo_<%=type_name%>_rms_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
202
208
|
{
|
|
203
209
|
<%=acc%> n = (<%=acc%>)(arg->in_indexer.total_size / arg->out_indexer.total_size);
|
|
@@ -226,25 +232,31 @@ void cumo_<%=type_name%>_rms_nan_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
|
226
232
|
cumo_reduce_split<dtype, rtype, cumo_<%=type_name%>_rms_nan_impl>(*arg, cumo_<%=type_name%>_rms_nan_impl{});
|
|
227
233
|
}
|
|
228
234
|
<% end %>
|
|
229
|
-
<% if %w[sfloat dfloat].include?(type_name)
|
|
235
|
+
<% if %w[sfloat dfloat].include?(type_name) %>
|
|
236
|
+
<% if kernel_part?("abs_sum") %>
|
|
230
237
|
void cumo_<%=type_name%>_abs_sum_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
231
238
|
{
|
|
232
239
|
cumo_reduce_split<dtype, dtype, cumo_<%=type_name%>_abs_sum_impl>(*arg, cumo_<%=type_name%>_abs_sum_impl{});
|
|
233
240
|
}
|
|
241
|
+
<% end %>
|
|
234
242
|
|
|
243
|
+
<% if kernel_part?("abs_max") %>
|
|
235
244
|
void cumo_<%=type_name%>_abs_max_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
236
245
|
{
|
|
237
246
|
using impl = cumo_<%=type_name%>_abs_extremum_of<cumo_<%=type_name%>_extremum_rules::Max>;
|
|
238
247
|
cumo_reduce_split<dtype, dtype, impl>(*arg, impl{});
|
|
239
248
|
}
|
|
249
|
+
<% end %>
|
|
240
250
|
|
|
251
|
+
<% if kernel_part?("abs_min") %>
|
|
241
252
|
void cumo_<%=type_name%>_abs_min_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
242
253
|
{
|
|
243
254
|
using impl = cumo_<%=type_name%>_abs_extremum_of<cumo_<%=type_name%>_extremum_rules::Min>;
|
|
244
255
|
cumo_reduce_split<dtype, dtype, impl>(*arg, impl{});
|
|
245
256
|
}
|
|
246
257
|
<% end %>
|
|
247
|
-
<%
|
|
258
|
+
<% end %>
|
|
259
|
+
<% if is_double_precision && kernel_part?("kahan_sum") %>
|
|
248
260
|
|
|
249
261
|
void cumo_<%=type_name%>_kahan_sum_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
250
262
|
{
|
|
@@ -16,6 +16,12 @@ static void
|
|
|
16
16
|
size_t *c;
|
|
17
17
|
int nd, md;
|
|
18
18
|
|
|
19
|
+
// In place, a view that visits an element twice reads what it wrote the
|
|
20
|
+
// first time, so the copy the elements are read from takes the write too,
|
|
21
|
+
// after the element read next was copied again.
|
|
22
|
+
int own = lp->args[0].value == lp->args[1].value;
|
|
23
|
+
int counts = cumo_na_ndloop_writes_existing(lp);
|
|
24
|
+
|
|
19
25
|
c = (size_t*)(lp->opt_ptr);
|
|
20
26
|
cumo_na_with_index_dims(lp->ndim, &nd, &md);
|
|
21
27
|
a = ALLOCA_N(VALUE,md);
|
|
@@ -30,40 +36,52 @@ static void
|
|
|
30
36
|
if (idx1) {
|
|
31
37
|
if (idx2) {
|
|
32
38
|
for (; i--;) {
|
|
39
|
+
char *q = p1 + *idx1;
|
|
33
40
|
CUMO_GET_DATA_INDEX(p1,idx1,dtype,x);
|
|
34
41
|
x = yield_map_with_index(x,c,a,nd,md);
|
|
35
42
|
if (i) cumo_na_ndloop_refresh_next(lp, p1 + *idx1, sizeof(dtype));
|
|
36
43
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
44
|
+
if (own) *(dtype*)q = x;
|
|
37
45
|
CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
|
|
46
|
+
if (counts) cumo_na_ndloop_wrote(lp);
|
|
38
47
|
c[nd]++;
|
|
39
48
|
}
|
|
40
49
|
} else {
|
|
41
50
|
for (; i--;) {
|
|
51
|
+
char *q = p1 + *idx1;
|
|
42
52
|
CUMO_GET_DATA_INDEX(p1,idx1,dtype,x);
|
|
43
53
|
x = yield_map_with_index(x,c,a,nd,md);
|
|
44
54
|
if (i) cumo_na_ndloop_refresh_next(lp, p1 + *idx1, sizeof(dtype));
|
|
45
55
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
56
|
+
if (own) *(dtype*)q = x;
|
|
46
57
|
CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
|
|
58
|
+
if (counts) cumo_na_ndloop_wrote(lp);
|
|
47
59
|
c[nd]++;
|
|
48
60
|
}
|
|
49
61
|
}
|
|
50
62
|
} else {
|
|
51
63
|
if (idx2) {
|
|
52
64
|
for (; i--;) {
|
|
65
|
+
char *q = p1;
|
|
53
66
|
CUMO_GET_DATA_STRIDE(p1,s1,dtype,x);
|
|
54
67
|
x = yield_map_with_index(x,c,a,nd,md);
|
|
55
68
|
if (i) cumo_na_ndloop_refresh_next(lp, p1, sizeof(dtype));
|
|
56
69
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
70
|
+
if (own) *(dtype*)q = x;
|
|
57
71
|
CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
|
|
72
|
+
if (counts) cumo_na_ndloop_wrote(lp);
|
|
58
73
|
c[nd]++;
|
|
59
74
|
}
|
|
60
75
|
} else {
|
|
61
76
|
for (; i--;) {
|
|
77
|
+
char *q = p1;
|
|
62
78
|
CUMO_GET_DATA_STRIDE(p1,s1,dtype,x);
|
|
63
79
|
x = yield_map_with_index(x,c,a,nd,md);
|
|
64
80
|
if (i) cumo_na_ndloop_refresh_next(lp, p1, sizeof(dtype));
|
|
65
81
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
82
|
+
if (own) *(dtype*)q = x;
|
|
66
83
|
CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
|
|
84
|
+
if (counts) cumo_na_ndloop_wrote(lp);
|
|
67
85
|
c[nd]++;
|
|
68
86
|
}
|
|
69
87
|
}
|
|
@@ -198,54 +198,72 @@ extern "C" {
|
|
|
198
198
|
#endif
|
|
199
199
|
#endif
|
|
200
200
|
|
|
201
|
-
<% if kernel_part? %>
|
|
201
|
+
<% if kernel_part?("sum") %>
|
|
202
202
|
void cumo_<%=type_name%>_sum_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
203
203
|
{
|
|
204
204
|
cumo_reduce_split<dtype, <%=dtype%>, cumo_<%=type_name%>_sum_impl>(*arg, cumo_<%=type_name%>_sum_impl{});
|
|
205
205
|
}
|
|
206
|
+
<% end %>
|
|
206
207
|
|
|
208
|
+
<% if kernel_part?("prod") %>
|
|
207
209
|
void cumo_<%=type_name%>_prod_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
208
210
|
{
|
|
209
211
|
cumo_reduce_split<dtype, <%=dtype%>, cumo_<%=type_name%>_prod_impl>(*arg, cumo_<%=type_name%>_prod_impl{});
|
|
210
212
|
}
|
|
213
|
+
<% end %>
|
|
211
214
|
|
|
215
|
+
<% if kernel_part?("min") %>
|
|
212
216
|
void cumo_<%=type_name%>_min_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
213
217
|
{
|
|
214
218
|
cumo_reduce_split<dtype, dtype, cumo_<%=type_name%>_min_impl>(*arg, cumo_<%=type_name%>_min_impl{});
|
|
215
219
|
}
|
|
220
|
+
<% end %>
|
|
216
221
|
|
|
222
|
+
<% if kernel_part?("max") %>
|
|
217
223
|
void cumo_<%=type_name%>_max_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
218
224
|
{
|
|
219
225
|
cumo_reduce_split<dtype, dtype, cumo_<%=type_name%>_max_impl>(*arg, cumo_<%=type_name%>_max_impl{});
|
|
220
226
|
}
|
|
227
|
+
<% end %>
|
|
221
228
|
|
|
229
|
+
<% if kernel_part?("ptp") %>
|
|
222
230
|
void cumo_<%=type_name%>_ptp_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
223
231
|
{
|
|
224
232
|
cumo_reduce_split<dtype, dtype, cumo_<%=type_name%>_ptp_impl>(*arg, cumo_<%=type_name%>_ptp_impl{});
|
|
225
233
|
}
|
|
234
|
+
<% end %>
|
|
226
235
|
|
|
236
|
+
<% if kernel_part?("minmax") %>
|
|
227
237
|
void cumo_<%=type_name%>_minmax_kernel_launch(cumo_na_reduction_arg_t* arg, cumo_na_iarray_t* out2)
|
|
228
238
|
{
|
|
229
239
|
cumo_reduce_pair_split<dtype, dtype, cumo_<%=type_name%>_minmax_impl>(*arg, *out2, cumo_<%=type_name%>_minmax_impl{});
|
|
230
240
|
}
|
|
241
|
+
<% end %>
|
|
231
242
|
<% unless is_float %>
|
|
232
243
|
|
|
244
|
+
<% if kernel_part?("mean") %>
|
|
233
245
|
void cumo_<%=type_name%>_mean_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
234
246
|
{
|
|
235
247
|
double n = (double)(arg->in_indexer.total_size / arg->out_indexer.total_size);
|
|
236
248
|
cumo_reduce_split<dtype, double, cumo_<%=type_name%>_mean_impl>(*arg, cumo_<%=type_name%>_mean_impl{n});
|
|
237
249
|
}
|
|
250
|
+
<% end %>
|
|
238
251
|
|
|
252
|
+
<% if kernel_part?("var") %>
|
|
239
253
|
void cumo_<%=type_name%>_var_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
240
254
|
{
|
|
241
255
|
cumo_reduce_split<dtype, double, cumo_<%=type_name%>_var_impl>(*arg, cumo_<%=type_name%>_var_impl{});
|
|
242
256
|
}
|
|
257
|
+
<% end %>
|
|
243
258
|
|
|
259
|
+
<% if kernel_part?("stddev") %>
|
|
244
260
|
void cumo_<%=type_name%>_stddev_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
245
261
|
{
|
|
246
262
|
cumo_reduce_split<dtype, double, cumo_<%=type_name%>_stddev_impl>(*arg, cumo_<%=type_name%>_stddev_impl{});
|
|
247
263
|
}
|
|
264
|
+
<% end %>
|
|
248
265
|
|
|
266
|
+
<% if kernel_part?("rms") %>
|
|
249
267
|
void cumo_<%=type_name%>_rms_kernel_launch(cumo_na_reduction_arg_t* arg)
|
|
250
268
|
{
|
|
251
269
|
double n = (double)(arg->in_indexer.total_size / arg->out_indexer.total_size);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#if defined(__cplusplus)
|
|
2
|
+
#if 0
|
|
3
|
+
{ /* satisfy cc-mode */
|
|
4
|
+
#endif
|
|
5
|
+
} /* extern "C" { */
|
|
6
|
+
#endif
|
|
7
|
+
|
|
8
|
+
#include "cumo/sort_kernel.h"
|
|
9
|
+
|
|
10
|
+
// sort_index and median sort with the same kernels, so their entry points are
|
|
11
|
+
// made here too.
|
|
12
|
+
CUMO_DEF_SORT(<%=type_name%>, dtype, <%=is_float%>)
|
|
13
|
+
|
|
14
|
+
#if defined(__cplusplus)
|
|
15
|
+
extern "C" {
|
|
16
|
+
#if 0
|
|
17
|
+
} /* satisfy cc-mode */
|
|
18
|
+
#endif
|
|
19
|
+
#endif
|
|
@@ -19,6 +19,8 @@ static void
|
|
|
19
19
|
size_t i;
|
|
20
20
|
dtype x;
|
|
21
21
|
<% if name == 'map' %>
|
|
22
|
+
// RObject data is host memory, which no loop stages.
|
|
23
|
+
int counts = <%= type_name == 'robject' ? '0' : 'cumo_na_ndloop_writes_existing(lp)' %>;
|
|
22
24
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
23
25
|
<% else %>
|
|
24
26
|
CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
|
|
@@ -30,6 +32,7 @@ static void
|
|
|
30
32
|
x = m_<%=name%>(x);
|
|
31
33
|
<% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
|
|
32
34
|
CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
|
|
35
|
+
<% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
|
|
33
36
|
}
|
|
34
37
|
} else {
|
|
35
38
|
for (i=0; i<n; i++) {
|
|
@@ -37,6 +40,7 @@ static void
|
|
|
37
40
|
x = m_<%=name%>(x);
|
|
38
41
|
<% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
|
|
39
42
|
CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
|
|
43
|
+
<% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
|
|
40
44
|
}
|
|
41
45
|
}
|
|
42
46
|
} else {
|
|
@@ -46,6 +50,7 @@ static void
|
|
|
46
50
|
x = m_<%=name%>(x);
|
|
47
51
|
<% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
|
|
48
52
|
CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
|
|
53
|
+
<% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
|
|
49
54
|
}
|
|
50
55
|
} else {
|
|
51
56
|
//<% if need_align %>
|
|
@@ -58,6 +63,7 @@ static void
|
|
|
58
63
|
x = m_<%=name%>(((dtype*)p1)[i]);
|
|
59
64
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
60
65
|
((dtype*)p2)[i] = x;
|
|
66
|
+
if (counts) cumo_na_ndloop_wrote(lp);
|
|
61
67
|
<% else %>
|
|
62
68
|
((dtype*)p2)[i] = m_<%=name%>(((dtype*)p1)[i]);
|
|
63
69
|
<% end %>
|
|
@@ -72,6 +78,7 @@ static void
|
|
|
72
78
|
x = m_<%=name%>(*(dtype*)p1);
|
|
73
79
|
if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
|
|
74
80
|
*(dtype*)p2 = x;
|
|
81
|
+
if (counts) cumo_na_ndloop_wrote(lp);
|
|
75
82
|
<% else %>
|
|
76
83
|
*(dtype*)p2 = m_<%=name%>(*(dtype*)p1);
|
|
77
84
|
<% end %>
|
|
@@ -87,6 +94,7 @@ static void
|
|
|
87
94
|
x = m_<%=name%>(x);
|
|
88
95
|
<% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
|
|
89
96
|
CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
|
|
97
|
+
<% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
|
|
90
98
|
}
|
|
91
99
|
//<% end %>
|
|
92
100
|
}
|
data/ext/cumo/narray/index.c
CHANGED
|
@@ -68,7 +68,7 @@ static void
|
|
|
68
68
|
cumo_na_index_set_step(cumo_na_index_arg_t *q, int i, size_t n, size_t beg, ssize_t step)
|
|
69
69
|
{
|
|
70
70
|
q->n = n;
|
|
71
|
-
q->beg = beg;
|
|
71
|
+
q->beg = n == 0 ? 0 : beg;
|
|
72
72
|
q->step = step;
|
|
73
73
|
q->idx = NULL;
|
|
74
74
|
q->reduce = 0;
|
|
@@ -178,6 +178,11 @@ cumo_na_parse_array(VALUE ary, int orig_dim, ssize_t size, cumo_na_index_arg_t *
|
|
|
178
178
|
cudaError_t status;
|
|
179
179
|
int n = RARRAY_LEN(ary);
|
|
180
180
|
|
|
181
|
+
if (n == 0) {
|
|
182
|
+
cumo_na_index_set_step(q, orig_dim, 0, 0, 1);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
181
186
|
// A step names the one element a one-element subscript does, without a
|
|
182
187
|
// device allocation, a kernel to fill it, or the synchronize a loop pays
|
|
183
188
|
// to read that one entry back. at() is left out: it gathers through an
|
|
@@ -250,10 +255,16 @@ cumo_na_parse_narray_index(VALUE a, int orig_dim, ssize_t size, cumo_na_index_ar
|
|
|
250
255
|
if (CUMO_NA_SIZE(na) != (size_t)size) {
|
|
251
256
|
rb_raise(rb_eIndexError, "Bit-NArray size mismatch");
|
|
252
257
|
}
|
|
253
|
-
|
|
254
|
-
|
|
258
|
+
if (size > 0) {
|
|
259
|
+
a = rb_funcall(a, cumo_id_where, 0);
|
|
260
|
+
CumoGetNArray(a,na);
|
|
261
|
+
}
|
|
255
262
|
}
|
|
256
263
|
n = CUMO_NA_SIZE(na);
|
|
264
|
+
if (n == 0) {
|
|
265
|
+
cumo_na_index_set_step(q, orig_dim, 0, 0, 1);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
257
268
|
idx = cumo_na_new(cIndex,1,&n);
|
|
258
269
|
cumo_na_store(idx,a);
|
|
259
270
|
|
|
@@ -301,6 +312,28 @@ cumo_na_parse_narray_index(VALUE a, int orig_dim, ssize_t size, cumo_na_index_ar
|
|
|
301
312
|
q->orig_dim = orig_dim;
|
|
302
313
|
}
|
|
303
314
|
|
|
315
|
+
static ssize_t
|
|
316
|
+
cumo_na_range_count(ssize_t beg, ssize_t end, int excl, ssize_t step, ssize_t size)
|
|
317
|
+
{
|
|
318
|
+
ssize_t last;
|
|
319
|
+
|
|
320
|
+
if (step > 0) {
|
|
321
|
+
if (excl ? end <= beg : end < beg) {
|
|
322
|
+
return (beg < 0 || beg > size || end < -1 || end > size) ? -1 : 0;
|
|
323
|
+
}
|
|
324
|
+
last = excl ? end - 1 : end;
|
|
325
|
+
} else {
|
|
326
|
+
if (excl ? end >= beg : end > beg) {
|
|
327
|
+
return (beg < -1 || beg >= size || end < -1 || end > size) ? -1 : 0;
|
|
328
|
+
}
|
|
329
|
+
last = excl ? end + 1 : end;
|
|
330
|
+
}
|
|
331
|
+
if (beg < 0 || beg >= size || last < 0 || last >= size) {
|
|
332
|
+
return -1;
|
|
333
|
+
}
|
|
334
|
+
return (last - beg) / step + 1;
|
|
335
|
+
}
|
|
336
|
+
|
|
304
337
|
static void
|
|
305
338
|
cumo_na_parse_range(VALUE range, ssize_t step, int orig_dim, ssize_t size, cumo_na_index_arg_t *q)
|
|
306
339
|
{
|
|
@@ -309,8 +342,8 @@ cumo_na_parse_range(VALUE range, ssize_t step, int orig_dim, ssize_t size, cumo_
|
|
|
309
342
|
// caught that: a truncated count came out negative up to 2**32 and was
|
|
310
343
|
// rounded to an empty view, and past that it came out positive and wrong.
|
|
311
344
|
ssize_t n;
|
|
312
|
-
ssize_t beg, end
|
|
313
|
-
|
|
345
|
+
ssize_t beg, end;
|
|
346
|
+
int excl;
|
|
314
347
|
|
|
315
348
|
#ifdef HAVE_RB_ARITHMETIC_SEQUENCE_EXTRACT
|
|
316
349
|
rb_arithmetic_sequence_components_t x;
|
|
@@ -323,61 +356,34 @@ cumo_na_parse_range(VALUE range, ssize_t step, int orig_dim, ssize_t size, cumo_
|
|
|
323
356
|
rb_raise(rb_eArgError, "step must not be zero");
|
|
324
357
|
}
|
|
325
358
|
|
|
326
|
-
beg =
|
|
327
|
-
if (beg < 0) {
|
|
328
|
-
beg += size;
|
|
329
|
-
}
|
|
359
|
+
beg = NUM2SSIZET(x.begin);
|
|
330
360
|
if (T_NIL == TYPE(x.end)) { // endless range
|
|
331
|
-
end = size - 1;
|
|
332
|
-
|
|
333
|
-
dot = edot;
|
|
334
|
-
}
|
|
335
|
-
if (beg < 0 || beg >= size) {
|
|
336
|
-
rb_raise(rb_eRangeError,
|
|
337
|
-
"%"SZF"d%s is out of range for size=%"SZF"d",
|
|
338
|
-
beg_orig, dot, size);
|
|
339
|
-
}
|
|
361
|
+
end = step > 0 ? size - 1 : 0;
|
|
362
|
+
excl = 0;
|
|
340
363
|
} else {
|
|
341
|
-
end =
|
|
364
|
+
end = NUM2SSIZET(x.end);
|
|
342
365
|
if (end < 0) {
|
|
343
366
|
end += size;
|
|
344
367
|
}
|
|
345
|
-
|
|
346
|
-
end--;
|
|
347
|
-
dot = edot;
|
|
348
|
-
}
|
|
349
|
-
if (beg < 0 || beg >= size || end < 0 || end >= size) {
|
|
350
|
-
rb_raise(rb_eRangeError,
|
|
351
|
-
"%"SZF"d%s%"SZF"d is out of range for size=%"SZF"d",
|
|
352
|
-
beg_orig, dot, end_orig, size);
|
|
353
|
-
}
|
|
368
|
+
excl = RTEST(x.exclude_end);
|
|
354
369
|
}
|
|
355
370
|
#else
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
beg = beg_orig = NUM2SSIZET(rb_funcall(range,cumo_id_beg,0));
|
|
359
|
-
if (beg < 0) {
|
|
360
|
-
beg += size;
|
|
361
|
-
}
|
|
362
|
-
end = end_orig = NUM2SSIZET(rb_funcall(range,cumo_id_end,0));
|
|
371
|
+
beg = NUM2SSIZET(rb_funcall(range,cumo_id_beg,0));
|
|
372
|
+
end = NUM2SSIZET(rb_funcall(range,cumo_id_end,0));
|
|
363
373
|
if (end < 0) {
|
|
364
374
|
end += size;
|
|
365
375
|
}
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
376
|
+
excl = RTEST(rb_funcall(range,cumo_id_exclude_end,0));
|
|
377
|
+
#endif
|
|
378
|
+
if (beg < 0) {
|
|
379
|
+
beg += size;
|
|
370
380
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
381
|
+
n = cumo_na_range_count(beg, end, excl, step, size);
|
|
382
|
+
if (n < 0) {
|
|
383
|
+
rb_raise(rb_eRangeError, "%"PRIsVALUE" is out of range for size=%"SZF"d",
|
|
384
|
+
rb_inspect(range), size);
|
|
375
385
|
}
|
|
376
|
-
#endif
|
|
377
|
-
n = (end-beg)/step+1;
|
|
378
|
-
if (n<0) n=0;
|
|
379
386
|
cumo_na_index_set_step(q,orig_dim,n,beg,step);
|
|
380
|
-
|
|
381
387
|
}
|
|
382
388
|
|
|
383
389
|
void
|
|
@@ -622,6 +628,9 @@ cumo_na_index_aref_nadata(cumo_narray_data_t *na1, cumo_narray_view_t *na2,
|
|
|
622
628
|
j++;
|
|
623
629
|
total *= size;
|
|
624
630
|
}
|
|
631
|
+
if (total == 0) {
|
|
632
|
+
na2->offset = 0;
|
|
633
|
+
}
|
|
625
634
|
cumo_na_set_newaxis_strides(na2, newaxis, n_newaxis, j, elmsz);
|
|
626
635
|
na2->base.size = total;
|
|
627
636
|
}
|
|
@@ -696,10 +705,12 @@ cumo_na_index_aref_naview(cumo_narray_view_t *na1, cumo_narray_view_t *na2,
|
|
|
696
705
|
size_t last;
|
|
697
706
|
stride1 = -stride1;
|
|
698
707
|
last = na1->base.shape[q[i].orig_dim] - 1;
|
|
699
|
-
if (
|
|
700
|
-
|
|
708
|
+
if (na1->base.size > 0) {
|
|
709
|
+
if (na2->offset < last * stride1) {
|
|
710
|
+
rb_raise(rb_eStandardError,"bug: negative offset");
|
|
711
|
+
}
|
|
712
|
+
na2->offset -= last * stride1;
|
|
701
713
|
}
|
|
702
|
-
na2->offset -= last * stride1;
|
|
703
714
|
cumo_na_index_aref_naview_index_stride_last_kernel_launch(index, stride1, last, size);
|
|
704
715
|
} else {
|
|
705
716
|
cumo_na_index_aref_naview_index_stride_kernel_launch(index, stride1, size);
|
|
@@ -727,10 +738,27 @@ cumo_na_index_aref_naview(cumo_narray_view_t *na1, cumo_narray_view_t *na2,
|
|
|
727
738
|
j++;
|
|
728
739
|
total *= size;
|
|
729
740
|
}
|
|
741
|
+
if (total == 0) {
|
|
742
|
+
na2->offset = na1->offset;
|
|
743
|
+
}
|
|
730
744
|
cumo_na_set_newaxis_strides(na2, newaxis, n_newaxis, j, elmsz);
|
|
731
745
|
na2->base.size = total;
|
|
732
746
|
}
|
|
733
747
|
|
|
748
|
+
static size_t
|
|
749
|
+
cumo_na_index_at_size(const cumo_na_index_arg_t *q, int ndim)
|
|
750
|
+
{
|
|
751
|
+
int i;
|
|
752
|
+
size_t size = q[ndim-1].n;
|
|
753
|
+
|
|
754
|
+
for (i=0; i<ndim-1; i++) {
|
|
755
|
+
if (q[i].n != size) {
|
|
756
|
+
rb_raise(cumo_na_eShapeError, "index array sizes mismatch");
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
return size;
|
|
760
|
+
}
|
|
761
|
+
|
|
734
762
|
void cumo_na_index_at_nadata_index_stride_add_kernel_launch(size_t *idx, size_t *idx1, ssize_t s1, uint64_t n);
|
|
735
763
|
void cumo_na_index_at_nadata_index_beg_step_stride_kernel_launch(size_t *idx, size_t beg, ssize_t step, ssize_t s1, uint64_t n);
|
|
736
764
|
void cumo_na_index_at_nadata_index_beg_step_stride_add_kernel_launch(size_t *idx, size_t beg, ssize_t step, ssize_t s1, uint64_t n);
|
|
@@ -740,7 +768,7 @@ cumo_na_index_at_nadata(cumo_narray_data_t *na1, cumo_narray_view_t *na2,
|
|
|
740
768
|
cumo_na_index_arg_t *q, ssize_t elmsz, int ndim, int keep_dim)
|
|
741
769
|
{
|
|
742
770
|
int i;
|
|
743
|
-
size_t size =
|
|
771
|
+
size_t size = na2->base.size;
|
|
744
772
|
ssize_t stride1;
|
|
745
773
|
ssize_t *strides_na1;
|
|
746
774
|
size_t *index;
|
|
@@ -759,15 +787,6 @@ cumo_na_index_at_nadata(cumo_narray_data_t *na1, cumo_narray_view_t *na2,
|
|
|
759
787
|
|
|
760
788
|
for (i=ndim-1; i>=0; i--) {
|
|
761
789
|
stride1 = strides_na1[q[i].orig_dim];
|
|
762
|
-
if (i==ndim-1) {
|
|
763
|
-
if (size == 0) {
|
|
764
|
-
rb_raise(cumo_na_eShapeError, "cannot get element of empty array");
|
|
765
|
-
}
|
|
766
|
-
} else {
|
|
767
|
-
if (size != q[i].n) {
|
|
768
|
-
rb_raise(cumo_na_eShapeError, "index array sizes mismatch");
|
|
769
|
-
}
|
|
770
|
-
}
|
|
771
790
|
|
|
772
791
|
if (q[i].idx != NULL) {
|
|
773
792
|
if (i==ndim-1) {
|
|
@@ -787,8 +806,6 @@ cumo_na_index_at_nadata(cumo_narray_data_t *na1, cumo_narray_view_t *na2,
|
|
|
787
806
|
}
|
|
788
807
|
|
|
789
808
|
}
|
|
790
|
-
na2->base.size = size;
|
|
791
|
-
na2->base.shape[0] = size;
|
|
792
809
|
}
|
|
793
810
|
|
|
794
811
|
void cumo_na_index_at_naview_index_index_index_add_kernel_launch(size_t *idx, size_t *idx1, size_t *idx2, uint64_t n);
|
|
@@ -801,7 +818,7 @@ cumo_na_index_at_naview(cumo_narray_view_t *na1, cumo_narray_view_t *na2,
|
|
|
801
818
|
{
|
|
802
819
|
int i;
|
|
803
820
|
size_t *index;
|
|
804
|
-
size_t size =
|
|
821
|
+
size_t size = na2->base.size;
|
|
805
822
|
|
|
806
823
|
if (q[ndim-1].idx != NULL) {
|
|
807
824
|
index = q[ndim-1].idx;
|
|
@@ -813,15 +830,6 @@ cumo_na_index_at_naview(cumo_narray_view_t *na1, cumo_narray_view_t *na2,
|
|
|
813
830
|
|
|
814
831
|
for (i=ndim-1; i>=0; i--) {
|
|
815
832
|
cumo_stridx_t sdx1 = na1->stridx[q[i].orig_dim];
|
|
816
|
-
if (i==ndim-1) {
|
|
817
|
-
if (size == 0) {
|
|
818
|
-
rb_raise(cumo_na_eShapeError, "cannot get element of empty array");
|
|
819
|
-
}
|
|
820
|
-
} else {
|
|
821
|
-
if (size != q[i].n) {
|
|
822
|
-
rb_raise(cumo_na_eShapeError, "index array sizes mismatch");
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
833
|
|
|
826
834
|
if (q[i].idx != NULL && CUMO_SDX_IS_INDEX(sdx1)) {
|
|
827
835
|
// index <- index
|
|
@@ -899,8 +907,6 @@ cumo_na_index_at_naview(cumo_narray_view_t *na1, cumo_narray_view_t *na2,
|
|
|
899
907
|
}
|
|
900
908
|
}
|
|
901
909
|
}
|
|
902
|
-
na2->base.size = size;
|
|
903
|
-
na2->base.shape[0] = size;
|
|
904
910
|
}
|
|
905
911
|
|
|
906
912
|
static int
|
|
@@ -981,6 +987,13 @@ VALUE cumo_na_aref_md_protected(VALUE data_value)
|
|
|
981
987
|
|
|
982
988
|
elmsz = cumo_na_element_stride(self);
|
|
983
989
|
|
|
990
|
+
if (at_mode && ndim > 0) {
|
|
991
|
+
size_t size = cumo_na_index_at_size(q, ndim);
|
|
992
|
+
na2->base.size = size;
|
|
993
|
+
na2->base.shape[0] = size;
|
|
994
|
+
CUMO_SDX_SET_STRIDE(na2->stridx[0], elmsz);
|
|
995
|
+
}
|
|
996
|
+
|
|
984
997
|
switch(na1->type) {
|
|
985
998
|
case CUMO_NARRAY_DATA_T:
|
|
986
999
|
case CUMO_NARRAY_FILEMAP_T:
|
|
@@ -989,7 +1002,9 @@ VALUE cumo_na_aref_md_protected(VALUE data_value)
|
|
|
989
1002
|
na2->base.size = 1;
|
|
990
1003
|
} else {
|
|
991
1004
|
if (at_mode) {
|
|
992
|
-
|
|
1005
|
+
if (na2->base.size > 0) {
|
|
1006
|
+
cumo_na_index_at_nadata((cumo_narray_data_t *)na1,na2,q,elmsz,ndim,keep_dim);
|
|
1007
|
+
}
|
|
993
1008
|
} else {
|
|
994
1009
|
cumo_na_index_aref_nadata((cumo_narray_data_t *)na1,na2,q,elmsz,ndim,keep_dim);
|
|
995
1010
|
}
|
|
@@ -1005,7 +1020,9 @@ VALUE cumo_na_aref_md_protected(VALUE data_value)
|
|
|
1005
1020
|
na2->offset = ((cumo_narray_view_t *)na1)->offset;
|
|
1006
1021
|
na2->data = ((cumo_narray_view_t *)na1)->data;
|
|
1007
1022
|
if (at_mode) {
|
|
1008
|
-
|
|
1023
|
+
if (na2->base.size > 0) {
|
|
1024
|
+
cumo_na_index_at_naview((cumo_narray_view_t *)na1,na2,q,elmsz,ndim,keep_dim);
|
|
1025
|
+
}
|
|
1009
1026
|
} else {
|
|
1010
1027
|
cumo_na_index_aref_naview((cumo_narray_view_t *)na1,na2,q,elmsz,ndim,keep_dim);
|
|
1011
1028
|
}
|
|
@@ -1201,9 +1218,6 @@ cumo_na_get_result_dimension(VALUE self, int argc, VALUE *argv, ssize_t stride,
|
|
|
1201
1218
|
VALUE a;
|
|
1202
1219
|
|
|
1203
1220
|
CumoGetNArray(self,na);
|
|
1204
|
-
if (na->size == 0) {
|
|
1205
|
-
rb_raise(cumo_na_eShapeError, "cannot get element of empty array");
|
|
1206
|
-
}
|
|
1207
1221
|
idx = ALLOCA_N(ssize_t, argc);
|
|
1208
1222
|
for (i=j=0; i<argc; i++) {
|
|
1209
1223
|
a = argv[i];
|
|
@@ -1241,13 +1255,15 @@ cumo_na_get_result_dimension(VALUE self, int argc, VALUE *argv, ssize_t stride,
|
|
|
1241
1255
|
pos = nv->offset;
|
|
1242
1256
|
if (j == na->ndim) {
|
|
1243
1257
|
for (i=j-1; i>=0; i--) {
|
|
1244
|
-
|
|
1258
|
+
idx[i] = cumo_na_range_check(idx[i], na->shape[i], i);
|
|
1259
|
+
}
|
|
1260
|
+
for (i=j-1; i>=0; i--) {
|
|
1245
1261
|
sdx = nv->stridx[i];
|
|
1246
1262
|
if (CUMO_SDX_IS_INDEX(sdx)) {
|
|
1247
1263
|
cumo_na_index_wait_fill(nv);
|
|
1248
|
-
pos += CUMO_SDX_GET_INDEX(sdx)[
|
|
1264
|
+
pos += CUMO_SDX_GET_INDEX(sdx)[idx[i]];
|
|
1249
1265
|
} else {
|
|
1250
|
-
pos += CUMO_SDX_GET_STRIDE(sdx)*
|
|
1266
|
+
pos += CUMO_SDX_GET_STRIDE(sdx)*idx[i];
|
|
1251
1267
|
}
|
|
1252
1268
|
}
|
|
1253
1269
|
*pos_idx = pos;
|
data/ext/cumo/narray/narray.c
CHANGED
|
@@ -1820,6 +1820,9 @@ cumo_na_s_from_binary(int argc, VALUE *argv, VALUE type)
|
|
|
1820
1820
|
VALUE vstr, vshape, vna;
|
|
1821
1821
|
VALUE velmsz;
|
|
1822
1822
|
|
|
1823
|
+
if (RTEST(rb_class_inherited_p(type, cumo_cRObject))) {
|
|
1824
|
+
rb_raise(rb_eTypeError, "cannot restore %s from binary data", rb_class2name(type));
|
|
1825
|
+
}
|
|
1823
1826
|
narg = rb_scan_args(argc,argv,"11",&vstr,&vshape);
|
|
1824
1827
|
Check_Type(vstr,T_STRING);
|
|
1825
1828
|
str_len = RSTRING_LEN(vstr);
|
|
@@ -1931,6 +1934,9 @@ cumo_na_store_binary(int argc, VALUE *argv, VALUE self)
|
|
|
1931
1934
|
VALUE velmsz;
|
|
1932
1935
|
cumo_narray_t *na;
|
|
1933
1936
|
|
|
1937
|
+
if (RTEST(rb_obj_is_kind_of(self, cumo_cRObject))) {
|
|
1938
|
+
rb_raise(rb_eTypeError, "cannot store binary data to %s", rb_obj_classname(self));
|
|
1939
|
+
}
|
|
1934
1940
|
narg = rb_scan_args(argc,argv,"11",&vstr,&voffset);
|
|
1935
1941
|
Check_Type(vstr,T_STRING);
|
|
1936
1942
|
if (narg==2) {
|
|
@@ -1997,6 +2003,7 @@ cumo_na_store_binary(int argc, VALUE *argv, VALUE self)
|
|
|
1997
2003
|
CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("cumo_na_store_binary", "any");
|
|
1998
2004
|
cumo_cuda_runtime_device_synchronize();
|
|
1999
2005
|
memcpy(ptr+cumo_na_get_offset(self), RSTRING_PTR(vstr)+offset, byte_size);
|
|
2006
|
+
cumo_cuda_runtime_note_device_write();
|
|
2000
2007
|
}
|
|
2001
2008
|
|
|
2002
2009
|
return SIZET2NUM(byte_size);
|
|
@@ -2015,6 +2022,9 @@ cumo_na_to_binary(VALUE self)
|
|
|
2015
2022
|
VALUE str;
|
|
2016
2023
|
cumo_narray_t *na;
|
|
2017
2024
|
|
|
2025
|
+
if (RTEST(rb_obj_is_kind_of(self, cumo_cRObject))) {
|
|
2026
|
+
rb_raise(rb_eTypeError, "cannot convert %s into binary data", rb_obj_classname(self));
|
|
2027
|
+
}
|
|
2018
2028
|
CumoGetNArray(self,na);
|
|
2019
2029
|
if (na->type == CUMO_NARRAY_VIEW_T &&
|
|
2020
2030
|
!(cumo_na_view_offset_reaches_bytes(self) && cumo_na_check_contiguous(self)==Qtrue)) {
|