carray 3.0.4 → 3.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +205 -0
- data/carray.gemspec +1 -1
- data/ext/ca_array_pool.c +25 -1
- data/ext/ca_axis_dispatch.c +13 -20
- data/ext/ca_categorical_iterator.c +27 -20
- data/ext/ca_compare.c +4 -4
- data/ext/ca_composite_dispatch.c +9 -9
- data/ext/ca_kernel_iterator.c +6 -6
- data/ext/ca_kernel_iterator.h +112 -83
- data/ext/ca_obj_array.c +69 -19
- data/ext/ca_obj_bincmp.c +1 -0
- data/ext/ca_obj_binop.c +1 -0
- data/ext/ca_obj_bitarray.c +5 -1
- data/ext/ca_obj_bitfield.c +58 -43
- data/ext/ca_obj_byte_swap.c +2 -1
- data/ext/ca_obj_const_string.c +15 -3
- data/ext/ca_obj_face.c +42 -1
- data/ext/ca_obj_field.c +13 -3
- data/ext/ca_obj_grid.c +52 -5
- data/ext/ca_obj_monop.c +46 -10
- data/ext/ca_obj_object.c +64 -12
- data/ext/ca_obj_record.c +3 -0
- data/ext/ca_obj_reduce.c +4 -0
- data/ext/ca_obj_refer.c +18 -9
- data/ext/ca_obj_roll.c +26 -6
- data/ext/ca_obj_select.c +34 -5
- data/ext/ca_obj_shift.c +31 -7
- data/ext/ca_obj_stack.c +1 -0
- data/ext/ca_obj_stride.c +28 -19
- data/ext/ca_obj_string.c +11 -2
- data/ext/ca_obj_tile.c +27 -7
- data/ext/ca_obj_transpose.c +8 -4
- data/ext/ca_obj_triop.c +3 -0
- data/ext/ca_obj_window.c +79 -50
- data/ext/ca_op_ipower.c +4 -4
- data/ext/ca_sort_kernels.h +6 -3
- data/ext/carray.h +33 -0
- data/ext/carray_access.c +16 -3
- data/ext/carray_attribute.c +24 -0
- data/ext/carray_cast.c +243 -82
- data/ext/carray_cast_func.rb +56 -26
- data/ext/carray_conversion.c +2 -2
- data/ext/carray_core.c +3 -0
- data/ext/carray_count.c +6 -0
- data/ext/carray_element.c +9 -4
- data/ext/carray_factorize.c +20 -20
- data/ext/carray_generate.c +8 -2
- data/ext/carray_histogram.c +1 -1
- data/ext/carray_index_classifier.c +4 -4
- data/ext/carray_internal.h +12 -0
- data/ext/carray_kernels_bincmp.c +168 -168
- data/ext/carray_kernels_binop.c +900 -900
- data/ext/carray_kernels_init.c +4 -4
- data/ext/carray_kernels_moncmp.c +96 -96
- data/ext/carray_kernels_monop.c +1984 -1984
- data/ext/carray_kernels_reduce_aggregate.c +1733 -1733
- data/ext/carray_kernels_reduce_cumulative.c +642 -642
- data/ext/carray_kernels_reduce_extreme.c +1150 -1150
- data/ext/carray_kernels_reduce_variance.c +258 -258
- data/ext/carray_kernels_scan.c +128 -128
- data/ext/carray_kernels_search.c +340 -340
- data/ext/carray_kernels_sort.c +295 -304
- data/ext/carray_kernels_triop.c +114 -114
- data/ext/carray_key_windows.c +4 -0
- data/ext/carray_mask.c +45 -22
- data/ext/carray_memory_view.c +2 -2
- data/ext/carray_operator.c +5 -5
- data/ext/carray_order.c +227 -8
- data/ext/carray_random.c +18 -18
- data/ext/carray_scatter.c +24 -7
- data/ext/carray_slab.c +32 -9
- data/ext/carray_sort.c +224 -16
- data/ext/carray_sort_kernel.c +5 -7
- data/ext/carray_struct.c +35 -43
- data/ext/carray_time_text.c +4 -2
- data/ext/carray_utils.c +63 -21
- data/ext/mkkernel.rb +210 -115
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +17 -4
- data/lib/carray/basics.rb +27 -7
- data/lib/carray/categorical.rb +30 -16
- data/lib/carray/categorical_iterator.rb +6 -6
- data/lib/carray/const_string.rb +10 -0
- data/lib/carray/data_type_extension.rb +8 -1
- data/lib/carray/fixlen_string.rb +41 -2
- data/lib/carray/frame/column_verbs.rb +4 -3
- data/lib/carray/frame/concat.rb +17 -3
- data/lib/carray/frame/frame.rb +6 -7
- data/lib/carray/frame/group.rb +100 -13
- data/lib/carray/frame/io.rb +3 -1
- data/lib/carray/fuse_source.rb +31 -2
- data/lib/carray/lazy.rb +35 -0
- data/lib/carray/methods/is_in.rb +8 -1
- data/lib/carray/segment_iterator.rb +12 -8
- data/lib/carray/serialize.rb +19 -2
- data/lib/carray/stack.rb +15 -12
- data/lib/carray/string.rb +8 -0
- data/lib/carray/string_operation_extension.rb +13 -5
- data/lib/carray/struct_builder.rb +5 -3
- data/lib/carray/time.rb +107 -68
- data/yard-stubs/ca_obj_transpose.rb +5 -4
- data/yard-stubs/carray_attribute.rb +3 -1
- data/yard-stubs/carray_cast.rb +2 -0
- data/yard-stubs/carray_conversion.rb +5 -1
- data/yard-stubs/carray_element.rb +7 -6
- data/yard-stubs/carray_mask.rb +8 -3
- data/yard-stubs/carray_math.rb +14 -4
- data/yard-stubs/carray_order.rb +100 -4
- data/yard-stubs/carray_scatter.rb +7 -5
- metadata +1 -1
|
@@ -1167,18 +1167,18 @@ accumulate_ki_stream_i16 (VALUE arg)
|
|
|
1167
1167
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
1168
1168
|
ca_size_t __counts[CA_RANK_MAX];
|
|
1169
1169
|
ca_size_t __strides[CA_RANK_MAX];
|
|
1170
|
-
ca_size_t __s = sizeof(
|
|
1170
|
+
ca_size_t __s = sizeof(ca_u_int16_t);
|
|
1171
1171
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
1172
1172
|
__strides[__k] = __s;
|
|
1173
1173
|
__s *= ca->dim[__k];
|
|
1174
1174
|
}
|
|
1175
1175
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
1176
1176
|
ca_size_t __outer_off = 0;
|
|
1177
|
-
|
|
1177
|
+
ca_u_int16_t acc;
|
|
1178
1178
|
acc = (0);
|
|
1179
1179
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
1180
1180
|
(void) masked_cnt;
|
|
1181
|
-
|
|
1181
|
+
ca_u_int16_t *__chunk = (ca_u_int16_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_int16_t));
|
|
1182
1182
|
__ctx->chunk = __chunk;
|
|
1183
1183
|
while ( __outer_off < __outer ) {
|
|
1184
1184
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -1188,12 +1188,12 @@ accumulate_ki_stream_i16 (VALUE arg)
|
|
|
1188
1188
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
1189
1189
|
ca_size_t __i;
|
|
1190
1190
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
1191
|
-
|
|
1191
|
+
ca_u_int16_t v = __chunk[__i];
|
|
1192
1192
|
acc += v;
|
|
1193
1193
|
}
|
|
1194
1194
|
__outer_off += __r;
|
|
1195
1195
|
}
|
|
1196
|
-
return LL2NUM((
|
|
1196
|
+
return LL2NUM((ca_u_int16_t) (acc));
|
|
1197
1197
|
}
|
|
1198
1198
|
|
|
1199
1199
|
static VALUE
|
|
@@ -1237,36 +1237,36 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1237
1237
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1238
1238
|
CArray *co;
|
|
1239
1239
|
GetCArray(vout, co);
|
|
1240
|
-
|
|
1240
|
+
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1241
1241
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1242
|
-
const
|
|
1242
|
+
const ca_u_int16_t *p_root = (const ca_u_int16_t *) ca->ptr;
|
|
1243
1243
|
|
|
1244
1244
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
1245
1245
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
1246
|
-
|
|
1246
|
+
ca_u_int16_t __li_buf_acc[512];
|
|
1247
1247
|
|
|
1248
1248
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1249
1249
|
const ca_size_t __li_TILE = 512;
|
|
1250
1250
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1251
|
-
const
|
|
1251
|
+
const ca_u_int16_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1252
1252
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1253
1253
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1254
1254
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1255
1255
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1256
|
-
__li_buf_acc[__j] = (
|
|
1256
|
+
__li_buf_acc[__j] = (ca_u_int16_t) (0);
|
|
1257
1257
|
}
|
|
1258
1258
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1259
|
-
const
|
|
1259
|
+
const ca_u_int16_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1260
1260
|
_Pragma("omp simd")
|
|
1261
1261
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1262
|
-
|
|
1262
|
+
ca_u_int16_t v = __li_row[__j];
|
|
1263
1263
|
__li_buf_acc[__j] += v;
|
|
1264
1264
|
(void) v;
|
|
1265
1265
|
}
|
|
1266
1266
|
}
|
|
1267
1267
|
|
|
1268
1268
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1269
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
1269
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int16_t) (__li_buf_acc[__j]);
|
|
1270
1270
|
}
|
|
1271
1271
|
}
|
|
1272
1272
|
}
|
|
@@ -1317,37 +1317,37 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1317
1317
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1318
1318
|
CArray *co;
|
|
1319
1319
|
GetCArray(vout, co);
|
|
1320
|
-
|
|
1320
|
+
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1321
1321
|
|
|
1322
|
-
|
|
1322
|
+
ca_u_int16_t __li_buf_acc[512];
|
|
1323
1323
|
|
|
1324
1324
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1325
1325
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
1326
1326
|
const ca_size_t __li_TILE = 512;
|
|
1327
1327
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1328
1328
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
1329
|
-
const
|
|
1330
|
-
|
|
1329
|
+
const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
|
|
1330
|
+
ca_u_int16_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
1331
1331
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1332
|
-
const
|
|
1332
|
+
const ca_u_int16_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1333
1333
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1334
1334
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1335
1335
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1336
1336
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1337
|
-
__li_buf_acc[__j] = (
|
|
1337
|
+
__li_buf_acc[__j] = (ca_u_int16_t) (0);
|
|
1338
1338
|
}
|
|
1339
1339
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1340
|
-
const
|
|
1340
|
+
const ca_u_int16_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1341
1341
|
_Pragma("omp simd")
|
|
1342
1342
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1343
|
-
|
|
1343
|
+
ca_u_int16_t v = __li_row[__j];
|
|
1344
1344
|
__li_buf_acc[__j] += v;
|
|
1345
1345
|
(void) v;
|
|
1346
1346
|
}
|
|
1347
1347
|
}
|
|
1348
1348
|
|
|
1349
1349
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1350
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
1350
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int16_t) (__li_buf_acc[__j]);
|
|
1351
1351
|
}
|
|
1352
1352
|
}
|
|
1353
1353
|
}
|
|
@@ -1408,7 +1408,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1408
1408
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1409
1409
|
CArray *co;
|
|
1410
1410
|
GetCArray(vout, co);
|
|
1411
|
-
|
|
1411
|
+
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1412
1412
|
|
|
1413
1413
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
1414
1414
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -1437,13 +1437,13 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1437
1437
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
1438
1438
|
}
|
|
1439
1439
|
|
|
1440
|
-
|
|
1440
|
+
ca_u_int16_t __li_buf_acc[512];
|
|
1441
1441
|
|
|
1442
1442
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1443
1443
|
const ca_size_t __li_TILE = 512;
|
|
1444
1444
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1445
1445
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
1446
|
-
const
|
|
1446
|
+
const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
|
|
1447
1447
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1448
1448
|
|
|
1449
1449
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -1454,27 +1454,27 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1454
1454
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
1455
1455
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
1456
1456
|
}
|
|
1457
|
-
|
|
1458
|
-
const
|
|
1457
|
+
ca_u_int16_t *__li_op_k = op + __li_obase;
|
|
1458
|
+
const ca_u_int16_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1459
1459
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
1460
1460
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1461
1461
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1462
1462
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1463
1463
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1464
|
-
__li_buf_acc[__j] = (
|
|
1464
|
+
__li_buf_acc[__j] = (ca_u_int16_t) (0);
|
|
1465
1465
|
}
|
|
1466
1466
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1467
|
-
const
|
|
1467
|
+
const ca_u_int16_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1468
1468
|
_Pragma("omp simd")
|
|
1469
1469
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1470
|
-
|
|
1470
|
+
ca_u_int16_t v = __li_row[__j];
|
|
1471
1471
|
__li_buf_acc[__j] += v;
|
|
1472
1472
|
(void) v;
|
|
1473
1473
|
}
|
|
1474
1474
|
}
|
|
1475
1475
|
|
|
1476
1476
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1477
|
-
__li_op_k[__li_tile + __j] = (
|
|
1477
|
+
__li_op_k[__li_tile + __j] = (ca_u_int16_t) (__li_buf_acc[__j]);
|
|
1478
1478
|
}
|
|
1479
1479
|
}
|
|
1480
1480
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -1555,7 +1555,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1555
1555
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1556
1556
|
CArray *co;
|
|
1557
1557
|
GetCArray(vout, co);
|
|
1558
|
-
|
|
1558
|
+
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1559
1559
|
|
|
1560
1560
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
1561
1561
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -1598,12 +1598,12 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1598
1598
|
ca_size_t __li_OUTER = 1;
|
|
1599
1599
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
1600
1600
|
|
|
1601
|
-
|
|
1601
|
+
ca_u_int16_t __li_buf_acc[512];
|
|
1602
1602
|
|
|
1603
1603
|
const ca_size_t __li_TILE = 512;
|
|
1604
1604
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1605
1605
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
1606
|
-
const
|
|
1606
|
+
const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
|
|
1607
1607
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1608
1608
|
|
|
1609
1609
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -1616,27 +1616,27 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1616
1616
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
1617
1617
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
1618
1618
|
}
|
|
1619
|
-
|
|
1620
|
-
const
|
|
1619
|
+
ca_u_int16_t *__li_op_k = op + __li_obase;
|
|
1620
|
+
const ca_u_int16_t *__li_plane = p_root + __li_pbase;
|
|
1621
1621
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
1622
1622
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
1623
1623
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
1624
1624
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
1625
1625
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1626
|
-
__li_buf_acc[__j] = (
|
|
1626
|
+
__li_buf_acc[__j] = (ca_u_int16_t) (0);
|
|
1627
1627
|
}
|
|
1628
1628
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1629
|
-
const
|
|
1629
|
+
const ca_u_int16_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
1630
1630
|
_Pragma("omp simd")
|
|
1631
1631
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1632
|
-
|
|
1632
|
+
ca_u_int16_t v = __li_row[__j];
|
|
1633
1633
|
__li_buf_acc[__j] += v;
|
|
1634
1634
|
(void) v;
|
|
1635
1635
|
}
|
|
1636
1636
|
}
|
|
1637
1637
|
|
|
1638
1638
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1639
|
-
__li_op_k[__li_tile + __j] = (
|
|
1639
|
+
__li_op_k[__li_tile + __j] = (ca_u_int16_t) (__li_buf_acc[__j]);
|
|
1640
1640
|
}
|
|
1641
1641
|
}
|
|
1642
1642
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -1655,7 +1655,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1655
1655
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1656
1656
|
CArray *co;
|
|
1657
1657
|
GetCArray(vout, co);
|
|
1658
|
-
|
|
1658
|
+
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1659
1659
|
ca_iter_state st;
|
|
1660
1660
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
1661
1661
|
slab_axes, naxes, 0);
|
|
@@ -1668,10 +1668,10 @@ boolean8_t *m;
|
|
|
1668
1668
|
ca_size_t out_i = 0;
|
|
1669
1669
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
1670
1670
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
1671
|
-
|
|
1671
|
+
ca_u_int16_t acc;
|
|
1672
1672
|
ca_size_t masked_cnt = 0;
|
|
1673
|
-
#define __accumulate_i16_EXPR(__x) ((
|
|
1674
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
1673
|
+
#define __accumulate_i16_EXPR(__x) ((ca_u_int16_t)(__x))
|
|
1674
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_int16_t, ca_u_int16_t, st, p, m, acc, 0, __accumulate_i16_EXPR, masked_cnt);
|
|
1675
1675
|
#undef __accumulate_i16_EXPR
|
|
1676
1676
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
1677
1677
|
if ( ! op_mask ) {
|
|
@@ -1679,9 +1679,9 @@ ca_size_t out_i = 0;
|
|
|
1679
1679
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
1680
1680
|
}
|
|
1681
1681
|
op_mask[out_i] = 1;
|
|
1682
|
-
op[out_i++] = (
|
|
1682
|
+
op[out_i++] = (ca_u_int16_t) 0; /* sentinel; mask bit is what counts */
|
|
1683
1683
|
} else {
|
|
1684
|
-
op[out_i++] = (
|
|
1684
|
+
op[out_i++] = (ca_u_int16_t) (acc);
|
|
1685
1685
|
}
|
|
1686
1686
|
}
|
|
1687
1687
|
ca_iter_state_finish(&st);
|
|
@@ -1716,18 +1716,18 @@ accumulate_ki_stream_u16 (VALUE arg)
|
|
|
1716
1716
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
1717
1717
|
ca_size_t __counts[CA_RANK_MAX];
|
|
1718
1718
|
ca_size_t __strides[CA_RANK_MAX];
|
|
1719
|
-
ca_size_t __s = sizeof(
|
|
1719
|
+
ca_size_t __s = sizeof(ca_u_uint16_t);
|
|
1720
1720
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
1721
1721
|
__strides[__k] = __s;
|
|
1722
1722
|
__s *= ca->dim[__k];
|
|
1723
1723
|
}
|
|
1724
1724
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
1725
1725
|
ca_size_t __outer_off = 0;
|
|
1726
|
-
|
|
1726
|
+
ca_u_uint16_t acc;
|
|
1727
1727
|
acc = (0);
|
|
1728
1728
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
1729
1729
|
(void) masked_cnt;
|
|
1730
|
-
|
|
1730
|
+
ca_u_uint16_t *__chunk = (ca_u_uint16_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_uint16_t));
|
|
1731
1731
|
__ctx->chunk = __chunk;
|
|
1732
1732
|
while ( __outer_off < __outer ) {
|
|
1733
1733
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -1737,12 +1737,12 @@ accumulate_ki_stream_u16 (VALUE arg)
|
|
|
1737
1737
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
1738
1738
|
ca_size_t __i;
|
|
1739
1739
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
1740
|
-
|
|
1740
|
+
ca_u_uint16_t v = __chunk[__i];
|
|
1741
1741
|
acc += v;
|
|
1742
1742
|
}
|
|
1743
1743
|
__outer_off += __r;
|
|
1744
1744
|
}
|
|
1745
|
-
return ULL2NUM((
|
|
1745
|
+
return ULL2NUM((ca_u_uint16_t) (acc));
|
|
1746
1746
|
}
|
|
1747
1747
|
|
|
1748
1748
|
static VALUE
|
|
@@ -1786,36 +1786,36 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1786
1786
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
1787
1787
|
CArray *co;
|
|
1788
1788
|
GetCArray(vout, co);
|
|
1789
|
-
|
|
1789
|
+
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
1790
1790
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1791
|
-
const
|
|
1791
|
+
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) ca->ptr;
|
|
1792
1792
|
|
|
1793
1793
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
1794
1794
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
1795
|
-
|
|
1795
|
+
ca_u_uint16_t __li_buf_acc[512];
|
|
1796
1796
|
|
|
1797
1797
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1798
1798
|
const ca_size_t __li_TILE = 512;
|
|
1799
1799
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1800
|
-
const
|
|
1800
|
+
const ca_u_uint16_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1801
1801
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1802
1802
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1803
1803
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1804
1804
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1805
|
-
__li_buf_acc[__j] = (
|
|
1805
|
+
__li_buf_acc[__j] = (ca_u_uint16_t) (0);
|
|
1806
1806
|
}
|
|
1807
1807
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1808
|
-
const
|
|
1808
|
+
const ca_u_uint16_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1809
1809
|
_Pragma("omp simd")
|
|
1810
1810
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1811
|
-
|
|
1811
|
+
ca_u_uint16_t v = __li_row[__j];
|
|
1812
1812
|
__li_buf_acc[__j] += v;
|
|
1813
1813
|
(void) v;
|
|
1814
1814
|
}
|
|
1815
1815
|
}
|
|
1816
1816
|
|
|
1817
1817
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1818
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
1818
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_uint16_t) (__li_buf_acc[__j]);
|
|
1819
1819
|
}
|
|
1820
1820
|
}
|
|
1821
1821
|
}
|
|
@@ -1866,37 +1866,37 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1866
1866
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
1867
1867
|
CArray *co;
|
|
1868
1868
|
GetCArray(vout, co);
|
|
1869
|
-
|
|
1869
|
+
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
1870
1870
|
|
|
1871
|
-
|
|
1871
|
+
ca_u_uint16_t __li_buf_acc[512];
|
|
1872
1872
|
|
|
1873
1873
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1874
1874
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
1875
1875
|
const ca_size_t __li_TILE = 512;
|
|
1876
1876
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1877
1877
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
1878
|
-
const
|
|
1879
|
-
|
|
1878
|
+
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
|
|
1879
|
+
ca_u_uint16_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
1880
1880
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1881
|
-
const
|
|
1881
|
+
const ca_u_uint16_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1882
1882
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1883
1883
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1884
1884
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1885
1885
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1886
|
-
__li_buf_acc[__j] = (
|
|
1886
|
+
__li_buf_acc[__j] = (ca_u_uint16_t) (0);
|
|
1887
1887
|
}
|
|
1888
1888
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1889
|
-
const
|
|
1889
|
+
const ca_u_uint16_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1890
1890
|
_Pragma("omp simd")
|
|
1891
1891
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1892
|
-
|
|
1892
|
+
ca_u_uint16_t v = __li_row[__j];
|
|
1893
1893
|
__li_buf_acc[__j] += v;
|
|
1894
1894
|
(void) v;
|
|
1895
1895
|
}
|
|
1896
1896
|
}
|
|
1897
1897
|
|
|
1898
1898
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1899
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
1899
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_uint16_t) (__li_buf_acc[__j]);
|
|
1900
1900
|
}
|
|
1901
1901
|
}
|
|
1902
1902
|
}
|
|
@@ -1957,7 +1957,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1957
1957
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
1958
1958
|
CArray *co;
|
|
1959
1959
|
GetCArray(vout, co);
|
|
1960
|
-
|
|
1960
|
+
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
1961
1961
|
|
|
1962
1962
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
1963
1963
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -1986,13 +1986,13 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1986
1986
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
1987
1987
|
}
|
|
1988
1988
|
|
|
1989
|
-
|
|
1989
|
+
ca_u_uint16_t __li_buf_acc[512];
|
|
1990
1990
|
|
|
1991
1991
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1992
1992
|
const ca_size_t __li_TILE = 512;
|
|
1993
1993
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1994
1994
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
1995
|
-
const
|
|
1995
|
+
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
|
|
1996
1996
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1997
1997
|
|
|
1998
1998
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -2003,27 +2003,27 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2003
2003
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
2004
2004
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
2005
2005
|
}
|
|
2006
|
-
|
|
2007
|
-
const
|
|
2006
|
+
ca_u_uint16_t *__li_op_k = op + __li_obase;
|
|
2007
|
+
const ca_u_uint16_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
2008
2008
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
2009
2009
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
2010
2010
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
2011
2011
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
2012
2012
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2013
|
-
__li_buf_acc[__j] = (
|
|
2013
|
+
__li_buf_acc[__j] = (ca_u_uint16_t) (0);
|
|
2014
2014
|
}
|
|
2015
2015
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2016
|
-
const
|
|
2016
|
+
const ca_u_uint16_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
2017
2017
|
_Pragma("omp simd")
|
|
2018
2018
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2019
|
-
|
|
2019
|
+
ca_u_uint16_t v = __li_row[__j];
|
|
2020
2020
|
__li_buf_acc[__j] += v;
|
|
2021
2021
|
(void) v;
|
|
2022
2022
|
}
|
|
2023
2023
|
}
|
|
2024
2024
|
|
|
2025
2025
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2026
|
-
__li_op_k[__li_tile + __j] = (
|
|
2026
|
+
__li_op_k[__li_tile + __j] = (ca_u_uint16_t) (__li_buf_acc[__j]);
|
|
2027
2027
|
}
|
|
2028
2028
|
}
|
|
2029
2029
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -2104,7 +2104,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2104
2104
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
2105
2105
|
CArray *co;
|
|
2106
2106
|
GetCArray(vout, co);
|
|
2107
|
-
|
|
2107
|
+
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
2108
2108
|
|
|
2109
2109
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
2110
2110
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -2147,12 +2147,12 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2147
2147
|
ca_size_t __li_OUTER = 1;
|
|
2148
2148
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
2149
2149
|
|
|
2150
|
-
|
|
2150
|
+
ca_u_uint16_t __li_buf_acc[512];
|
|
2151
2151
|
|
|
2152
2152
|
const ca_size_t __li_TILE = 512;
|
|
2153
2153
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2154
2154
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
2155
|
-
const
|
|
2155
|
+
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
|
|
2156
2156
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
2157
2157
|
|
|
2158
2158
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -2165,27 +2165,27 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2165
2165
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
2166
2166
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
2167
2167
|
}
|
|
2168
|
-
|
|
2169
|
-
const
|
|
2168
|
+
ca_u_uint16_t *__li_op_k = op + __li_obase;
|
|
2169
|
+
const ca_u_uint16_t *__li_plane = p_root + __li_pbase;
|
|
2170
2170
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
2171
2171
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
2172
2172
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
2173
2173
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
2174
2174
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2175
|
-
__li_buf_acc[__j] = (
|
|
2175
|
+
__li_buf_acc[__j] = (ca_u_uint16_t) (0);
|
|
2176
2176
|
}
|
|
2177
2177
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2178
|
-
const
|
|
2178
|
+
const ca_u_uint16_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
2179
2179
|
_Pragma("omp simd")
|
|
2180
2180
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2181
|
-
|
|
2181
|
+
ca_u_uint16_t v = __li_row[__j];
|
|
2182
2182
|
__li_buf_acc[__j] += v;
|
|
2183
2183
|
(void) v;
|
|
2184
2184
|
}
|
|
2185
2185
|
}
|
|
2186
2186
|
|
|
2187
2187
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2188
|
-
__li_op_k[__li_tile + __j] = (
|
|
2188
|
+
__li_op_k[__li_tile + __j] = (ca_u_uint16_t) (__li_buf_acc[__j]);
|
|
2189
2189
|
}
|
|
2190
2190
|
}
|
|
2191
2191
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -2204,7 +2204,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2204
2204
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
2205
2205
|
CArray *co;
|
|
2206
2206
|
GetCArray(vout, co);
|
|
2207
|
-
|
|
2207
|
+
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
2208
2208
|
ca_iter_state st;
|
|
2209
2209
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
2210
2210
|
slab_axes, naxes, 0);
|
|
@@ -2217,10 +2217,10 @@ boolean8_t *m;
|
|
|
2217
2217
|
ca_size_t out_i = 0;
|
|
2218
2218
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
2219
2219
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
2220
|
-
|
|
2220
|
+
ca_u_uint16_t acc;
|
|
2221
2221
|
ca_size_t masked_cnt = 0;
|
|
2222
|
-
#define __accumulate_u16_EXPR(__x) ((
|
|
2223
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
2222
|
+
#define __accumulate_u16_EXPR(__x) ((ca_u_uint16_t)(__x))
|
|
2223
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_uint16_t, ca_u_uint16_t, st, p, m, acc, 0, __accumulate_u16_EXPR, masked_cnt);
|
|
2224
2224
|
#undef __accumulate_u16_EXPR
|
|
2225
2225
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
2226
2226
|
if ( ! op_mask ) {
|
|
@@ -2228,9 +2228,9 @@ ca_size_t out_i = 0;
|
|
|
2228
2228
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
2229
2229
|
}
|
|
2230
2230
|
op_mask[out_i] = 1;
|
|
2231
|
-
op[out_i++] = (
|
|
2231
|
+
op[out_i++] = (ca_u_uint16_t) 0; /* sentinel; mask bit is what counts */
|
|
2232
2232
|
} else {
|
|
2233
|
-
op[out_i++] = (
|
|
2233
|
+
op[out_i++] = (ca_u_uint16_t) (acc);
|
|
2234
2234
|
}
|
|
2235
2235
|
}
|
|
2236
2236
|
ca_iter_state_finish(&st);
|
|
@@ -2265,18 +2265,18 @@ accumulate_ki_stream_i32 (VALUE arg)
|
|
|
2265
2265
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
2266
2266
|
ca_size_t __counts[CA_RANK_MAX];
|
|
2267
2267
|
ca_size_t __strides[CA_RANK_MAX];
|
|
2268
|
-
ca_size_t __s = sizeof(
|
|
2268
|
+
ca_size_t __s = sizeof(ca_u_int32_t);
|
|
2269
2269
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
2270
2270
|
__strides[__k] = __s;
|
|
2271
2271
|
__s *= ca->dim[__k];
|
|
2272
2272
|
}
|
|
2273
2273
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
2274
2274
|
ca_size_t __outer_off = 0;
|
|
2275
|
-
|
|
2275
|
+
ca_u_int32_t acc;
|
|
2276
2276
|
acc = (0);
|
|
2277
2277
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
2278
2278
|
(void) masked_cnt;
|
|
2279
|
-
|
|
2279
|
+
ca_u_int32_t *__chunk = (ca_u_int32_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_int32_t));
|
|
2280
2280
|
__ctx->chunk = __chunk;
|
|
2281
2281
|
while ( __outer_off < __outer ) {
|
|
2282
2282
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -2286,12 +2286,12 @@ accumulate_ki_stream_i32 (VALUE arg)
|
|
|
2286
2286
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
2287
2287
|
ca_size_t __i;
|
|
2288
2288
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
2289
|
-
|
|
2289
|
+
ca_u_int32_t v = __chunk[__i];
|
|
2290
2290
|
acc += v;
|
|
2291
2291
|
}
|
|
2292
2292
|
__outer_off += __r;
|
|
2293
2293
|
}
|
|
2294
|
-
return LL2NUM((
|
|
2294
|
+
return LL2NUM((ca_u_int32_t) (acc));
|
|
2295
2295
|
}
|
|
2296
2296
|
|
|
2297
2297
|
static VALUE
|
|
@@ -2335,36 +2335,36 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2335
2335
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
2336
2336
|
CArray *co;
|
|
2337
2337
|
GetCArray(vout, co);
|
|
2338
|
-
|
|
2338
|
+
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
2339
2339
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
2340
|
-
const
|
|
2340
|
+
const ca_u_int32_t *p_root = (const ca_u_int32_t *) ca->ptr;
|
|
2341
2341
|
|
|
2342
2342
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
2343
2343
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
2344
|
-
|
|
2344
|
+
ca_u_int32_t __li_buf_acc[512];
|
|
2345
2345
|
|
|
2346
2346
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
2347
2347
|
const ca_size_t __li_TILE = 512;
|
|
2348
2348
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
2349
|
-
const
|
|
2349
|
+
const ca_u_int32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
2350
2350
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
2351
2351
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
2352
2352
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
2353
2353
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2354
|
-
__li_buf_acc[__j] = (
|
|
2354
|
+
__li_buf_acc[__j] = (ca_u_int32_t) (0);
|
|
2355
2355
|
}
|
|
2356
2356
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2357
|
-
const
|
|
2357
|
+
const ca_u_int32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
2358
2358
|
_Pragma("omp simd")
|
|
2359
2359
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2360
|
-
|
|
2360
|
+
ca_u_int32_t v = __li_row[__j];
|
|
2361
2361
|
__li_buf_acc[__j] += v;
|
|
2362
2362
|
(void) v;
|
|
2363
2363
|
}
|
|
2364
2364
|
}
|
|
2365
2365
|
|
|
2366
2366
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2367
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
2367
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int32_t) (__li_buf_acc[__j]);
|
|
2368
2368
|
}
|
|
2369
2369
|
}
|
|
2370
2370
|
}
|
|
@@ -2415,37 +2415,37 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2415
2415
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
2416
2416
|
CArray *co;
|
|
2417
2417
|
GetCArray(vout, co);
|
|
2418
|
-
|
|
2418
|
+
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
2419
2419
|
|
|
2420
|
-
|
|
2420
|
+
ca_u_int32_t __li_buf_acc[512];
|
|
2421
2421
|
|
|
2422
2422
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
2423
2423
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
2424
2424
|
const ca_size_t __li_TILE = 512;
|
|
2425
2425
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2426
2426
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
2427
|
-
const
|
|
2428
|
-
|
|
2427
|
+
const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
|
|
2428
|
+
ca_u_int32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
2429
2429
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
2430
|
-
const
|
|
2430
|
+
const ca_u_int32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
2431
2431
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
2432
2432
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
2433
2433
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
2434
2434
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2435
|
-
__li_buf_acc[__j] = (
|
|
2435
|
+
__li_buf_acc[__j] = (ca_u_int32_t) (0);
|
|
2436
2436
|
}
|
|
2437
2437
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2438
|
-
const
|
|
2438
|
+
const ca_u_int32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
2439
2439
|
_Pragma("omp simd")
|
|
2440
2440
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2441
|
-
|
|
2441
|
+
ca_u_int32_t v = __li_row[__j];
|
|
2442
2442
|
__li_buf_acc[__j] += v;
|
|
2443
2443
|
(void) v;
|
|
2444
2444
|
}
|
|
2445
2445
|
}
|
|
2446
2446
|
|
|
2447
2447
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2448
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
2448
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int32_t) (__li_buf_acc[__j]);
|
|
2449
2449
|
}
|
|
2450
2450
|
}
|
|
2451
2451
|
}
|
|
@@ -2506,7 +2506,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2506
2506
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
2507
2507
|
CArray *co;
|
|
2508
2508
|
GetCArray(vout, co);
|
|
2509
|
-
|
|
2509
|
+
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
2510
2510
|
|
|
2511
2511
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
2512
2512
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -2535,13 +2535,13 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2535
2535
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
2536
2536
|
}
|
|
2537
2537
|
|
|
2538
|
-
|
|
2538
|
+
ca_u_int32_t __li_buf_acc[512];
|
|
2539
2539
|
|
|
2540
2540
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
2541
2541
|
const ca_size_t __li_TILE = 512;
|
|
2542
2542
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2543
2543
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
2544
|
-
const
|
|
2544
|
+
const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
|
|
2545
2545
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
2546
2546
|
|
|
2547
2547
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -2552,27 +2552,27 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2552
2552
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
2553
2553
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
2554
2554
|
}
|
|
2555
|
-
|
|
2556
|
-
const
|
|
2555
|
+
ca_u_int32_t *__li_op_k = op + __li_obase;
|
|
2556
|
+
const ca_u_int32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
2557
2557
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
2558
2558
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
2559
2559
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
2560
2560
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
2561
2561
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2562
|
-
__li_buf_acc[__j] = (
|
|
2562
|
+
__li_buf_acc[__j] = (ca_u_int32_t) (0);
|
|
2563
2563
|
}
|
|
2564
2564
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2565
|
-
const
|
|
2565
|
+
const ca_u_int32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
2566
2566
|
_Pragma("omp simd")
|
|
2567
2567
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2568
|
-
|
|
2568
|
+
ca_u_int32_t v = __li_row[__j];
|
|
2569
2569
|
__li_buf_acc[__j] += v;
|
|
2570
2570
|
(void) v;
|
|
2571
2571
|
}
|
|
2572
2572
|
}
|
|
2573
2573
|
|
|
2574
2574
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2575
|
-
__li_op_k[__li_tile + __j] = (
|
|
2575
|
+
__li_op_k[__li_tile + __j] = (ca_u_int32_t) (__li_buf_acc[__j]);
|
|
2576
2576
|
}
|
|
2577
2577
|
}
|
|
2578
2578
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -2653,7 +2653,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2653
2653
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
2654
2654
|
CArray *co;
|
|
2655
2655
|
GetCArray(vout, co);
|
|
2656
|
-
|
|
2656
|
+
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
2657
2657
|
|
|
2658
2658
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
2659
2659
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -2696,12 +2696,12 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2696
2696
|
ca_size_t __li_OUTER = 1;
|
|
2697
2697
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
2698
2698
|
|
|
2699
|
-
|
|
2699
|
+
ca_u_int32_t __li_buf_acc[512];
|
|
2700
2700
|
|
|
2701
2701
|
const ca_size_t __li_TILE = 512;
|
|
2702
2702
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2703
2703
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
2704
|
-
const
|
|
2704
|
+
const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
|
|
2705
2705
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
2706
2706
|
|
|
2707
2707
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -2714,27 +2714,27 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2714
2714
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
2715
2715
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
2716
2716
|
}
|
|
2717
|
-
|
|
2718
|
-
const
|
|
2717
|
+
ca_u_int32_t *__li_op_k = op + __li_obase;
|
|
2718
|
+
const ca_u_int32_t *__li_plane = p_root + __li_pbase;
|
|
2719
2719
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
2720
2720
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
2721
2721
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
2722
2722
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
2723
2723
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2724
|
-
__li_buf_acc[__j] = (
|
|
2724
|
+
__li_buf_acc[__j] = (ca_u_int32_t) (0);
|
|
2725
2725
|
}
|
|
2726
2726
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2727
|
-
const
|
|
2727
|
+
const ca_u_int32_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
2728
2728
|
_Pragma("omp simd")
|
|
2729
2729
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2730
|
-
|
|
2730
|
+
ca_u_int32_t v = __li_row[__j];
|
|
2731
2731
|
__li_buf_acc[__j] += v;
|
|
2732
2732
|
(void) v;
|
|
2733
2733
|
}
|
|
2734
2734
|
}
|
|
2735
2735
|
|
|
2736
2736
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2737
|
-
__li_op_k[__li_tile + __j] = (
|
|
2737
|
+
__li_op_k[__li_tile + __j] = (ca_u_int32_t) (__li_buf_acc[__j]);
|
|
2738
2738
|
}
|
|
2739
2739
|
}
|
|
2740
2740
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -2753,7 +2753,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2753
2753
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
2754
2754
|
CArray *co;
|
|
2755
2755
|
GetCArray(vout, co);
|
|
2756
|
-
|
|
2756
|
+
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
2757
2757
|
ca_iter_state st;
|
|
2758
2758
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
2759
2759
|
slab_axes, naxes, 0);
|
|
@@ -2766,10 +2766,10 @@ boolean8_t *m;
|
|
|
2766
2766
|
ca_size_t out_i = 0;
|
|
2767
2767
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
2768
2768
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
2769
|
-
|
|
2769
|
+
ca_u_int32_t acc;
|
|
2770
2770
|
ca_size_t masked_cnt = 0;
|
|
2771
|
-
#define __accumulate_i32_EXPR(__x) ((
|
|
2772
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
2771
|
+
#define __accumulate_i32_EXPR(__x) ((ca_u_int32_t)(__x))
|
|
2772
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_int32_t, ca_u_int32_t, st, p, m, acc, 0, __accumulate_i32_EXPR, masked_cnt);
|
|
2773
2773
|
#undef __accumulate_i32_EXPR
|
|
2774
2774
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
2775
2775
|
if ( ! op_mask ) {
|
|
@@ -2777,9 +2777,9 @@ ca_size_t out_i = 0;
|
|
|
2777
2777
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
2778
2778
|
}
|
|
2779
2779
|
op_mask[out_i] = 1;
|
|
2780
|
-
op[out_i++] = (
|
|
2780
|
+
op[out_i++] = (ca_u_int32_t) 0; /* sentinel; mask bit is what counts */
|
|
2781
2781
|
} else {
|
|
2782
|
-
op[out_i++] = (
|
|
2782
|
+
op[out_i++] = (ca_u_int32_t) (acc);
|
|
2783
2783
|
}
|
|
2784
2784
|
}
|
|
2785
2785
|
ca_iter_state_finish(&st);
|
|
@@ -2814,18 +2814,18 @@ accumulate_ki_stream_u32 (VALUE arg)
|
|
|
2814
2814
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
2815
2815
|
ca_size_t __counts[CA_RANK_MAX];
|
|
2816
2816
|
ca_size_t __strides[CA_RANK_MAX];
|
|
2817
|
-
ca_size_t __s = sizeof(
|
|
2817
|
+
ca_size_t __s = sizeof(ca_u_uint32_t);
|
|
2818
2818
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
2819
2819
|
__strides[__k] = __s;
|
|
2820
2820
|
__s *= ca->dim[__k];
|
|
2821
2821
|
}
|
|
2822
2822
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
2823
2823
|
ca_size_t __outer_off = 0;
|
|
2824
|
-
|
|
2824
|
+
ca_u_uint32_t acc;
|
|
2825
2825
|
acc = (0);
|
|
2826
2826
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
2827
2827
|
(void) masked_cnt;
|
|
2828
|
-
|
|
2828
|
+
ca_u_uint32_t *__chunk = (ca_u_uint32_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_uint32_t));
|
|
2829
2829
|
__ctx->chunk = __chunk;
|
|
2830
2830
|
while ( __outer_off < __outer ) {
|
|
2831
2831
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -2835,12 +2835,12 @@ accumulate_ki_stream_u32 (VALUE arg)
|
|
|
2835
2835
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
2836
2836
|
ca_size_t __i;
|
|
2837
2837
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
2838
|
-
|
|
2838
|
+
ca_u_uint32_t v = __chunk[__i];
|
|
2839
2839
|
acc += v;
|
|
2840
2840
|
}
|
|
2841
2841
|
__outer_off += __r;
|
|
2842
2842
|
}
|
|
2843
|
-
return ULL2NUM((
|
|
2843
|
+
return ULL2NUM((ca_u_uint32_t) (acc));
|
|
2844
2844
|
}
|
|
2845
2845
|
|
|
2846
2846
|
static VALUE
|
|
@@ -2884,36 +2884,36 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2884
2884
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
2885
2885
|
CArray *co;
|
|
2886
2886
|
GetCArray(vout, co);
|
|
2887
|
-
|
|
2887
|
+
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
2888
2888
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
2889
|
-
const
|
|
2889
|
+
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) ca->ptr;
|
|
2890
2890
|
|
|
2891
2891
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
2892
2892
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
2893
|
-
|
|
2893
|
+
ca_u_uint32_t __li_buf_acc[512];
|
|
2894
2894
|
|
|
2895
2895
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
2896
2896
|
const ca_size_t __li_TILE = 512;
|
|
2897
2897
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
2898
|
-
const
|
|
2898
|
+
const ca_u_uint32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
2899
2899
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
2900
2900
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
2901
2901
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
2902
2902
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2903
|
-
__li_buf_acc[__j] = (
|
|
2903
|
+
__li_buf_acc[__j] = (ca_u_uint32_t) (0);
|
|
2904
2904
|
}
|
|
2905
2905
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2906
|
-
const
|
|
2906
|
+
const ca_u_uint32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
2907
2907
|
_Pragma("omp simd")
|
|
2908
2908
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2909
|
-
|
|
2909
|
+
ca_u_uint32_t v = __li_row[__j];
|
|
2910
2910
|
__li_buf_acc[__j] += v;
|
|
2911
2911
|
(void) v;
|
|
2912
2912
|
}
|
|
2913
2913
|
}
|
|
2914
2914
|
|
|
2915
2915
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2916
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
2916
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_uint32_t) (__li_buf_acc[__j]);
|
|
2917
2917
|
}
|
|
2918
2918
|
}
|
|
2919
2919
|
}
|
|
@@ -2964,37 +2964,37 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2964
2964
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
2965
2965
|
CArray *co;
|
|
2966
2966
|
GetCArray(vout, co);
|
|
2967
|
-
|
|
2967
|
+
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
2968
2968
|
|
|
2969
|
-
|
|
2969
|
+
ca_u_uint32_t __li_buf_acc[512];
|
|
2970
2970
|
|
|
2971
2971
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
2972
2972
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
2973
2973
|
const ca_size_t __li_TILE = 512;
|
|
2974
2974
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2975
2975
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
2976
|
-
const
|
|
2977
|
-
|
|
2976
|
+
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
|
|
2977
|
+
ca_u_uint32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
2978
2978
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
2979
|
-
const
|
|
2979
|
+
const ca_u_uint32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
2980
2980
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
2981
2981
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
2982
2982
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
2983
2983
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2984
|
-
__li_buf_acc[__j] = (
|
|
2984
|
+
__li_buf_acc[__j] = (ca_u_uint32_t) (0);
|
|
2985
2985
|
}
|
|
2986
2986
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
2987
|
-
const
|
|
2987
|
+
const ca_u_uint32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
2988
2988
|
_Pragma("omp simd")
|
|
2989
2989
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2990
|
-
|
|
2990
|
+
ca_u_uint32_t v = __li_row[__j];
|
|
2991
2991
|
__li_buf_acc[__j] += v;
|
|
2992
2992
|
(void) v;
|
|
2993
2993
|
}
|
|
2994
2994
|
}
|
|
2995
2995
|
|
|
2996
2996
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2997
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
2997
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_uint32_t) (__li_buf_acc[__j]);
|
|
2998
2998
|
}
|
|
2999
2999
|
}
|
|
3000
3000
|
}
|
|
@@ -3055,7 +3055,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3055
3055
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
3056
3056
|
CArray *co;
|
|
3057
3057
|
GetCArray(vout, co);
|
|
3058
|
-
|
|
3058
|
+
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
3059
3059
|
|
|
3060
3060
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
3061
3061
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -3084,13 +3084,13 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3084
3084
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
3085
3085
|
}
|
|
3086
3086
|
|
|
3087
|
-
|
|
3087
|
+
ca_u_uint32_t __li_buf_acc[512];
|
|
3088
3088
|
|
|
3089
3089
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3090
3090
|
const ca_size_t __li_TILE = 512;
|
|
3091
3091
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3092
3092
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
3093
|
-
const
|
|
3093
|
+
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
|
|
3094
3094
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3095
3095
|
|
|
3096
3096
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -3101,27 +3101,27 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3101
3101
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
3102
3102
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
3103
3103
|
}
|
|
3104
|
-
|
|
3105
|
-
const
|
|
3104
|
+
ca_u_uint32_t *__li_op_k = op + __li_obase;
|
|
3105
|
+
const ca_u_uint32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
3106
3106
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
3107
3107
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
3108
3108
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
3109
3109
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
3110
3110
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3111
|
-
__li_buf_acc[__j] = (
|
|
3111
|
+
__li_buf_acc[__j] = (ca_u_uint32_t) (0);
|
|
3112
3112
|
}
|
|
3113
3113
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
3114
|
-
const
|
|
3114
|
+
const ca_u_uint32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
3115
3115
|
_Pragma("omp simd")
|
|
3116
3116
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3117
|
-
|
|
3117
|
+
ca_u_uint32_t v = __li_row[__j];
|
|
3118
3118
|
__li_buf_acc[__j] += v;
|
|
3119
3119
|
(void) v;
|
|
3120
3120
|
}
|
|
3121
3121
|
}
|
|
3122
3122
|
|
|
3123
3123
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3124
|
-
__li_op_k[__li_tile + __j] = (
|
|
3124
|
+
__li_op_k[__li_tile + __j] = (ca_u_uint32_t) (__li_buf_acc[__j]);
|
|
3125
3125
|
}
|
|
3126
3126
|
}
|
|
3127
3127
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -3202,7 +3202,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3202
3202
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
3203
3203
|
CArray *co;
|
|
3204
3204
|
GetCArray(vout, co);
|
|
3205
|
-
|
|
3205
|
+
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
3206
3206
|
|
|
3207
3207
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
3208
3208
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -3245,12 +3245,12 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3245
3245
|
ca_size_t __li_OUTER = 1;
|
|
3246
3246
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
3247
3247
|
|
|
3248
|
-
|
|
3248
|
+
ca_u_uint32_t __li_buf_acc[512];
|
|
3249
3249
|
|
|
3250
3250
|
const ca_size_t __li_TILE = 512;
|
|
3251
3251
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3252
3252
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
3253
|
-
const
|
|
3253
|
+
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
|
|
3254
3254
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3255
3255
|
|
|
3256
3256
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -3263,27 +3263,27 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3263
3263
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
3264
3264
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
3265
3265
|
}
|
|
3266
|
-
|
|
3267
|
-
const
|
|
3266
|
+
ca_u_uint32_t *__li_op_k = op + __li_obase;
|
|
3267
|
+
const ca_u_uint32_t *__li_plane = p_root + __li_pbase;
|
|
3268
3268
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
3269
3269
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
3270
3270
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
3271
3271
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
3272
3272
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3273
|
-
__li_buf_acc[__j] = (
|
|
3273
|
+
__li_buf_acc[__j] = (ca_u_uint32_t) (0);
|
|
3274
3274
|
}
|
|
3275
3275
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
3276
|
-
const
|
|
3276
|
+
const ca_u_uint32_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
3277
3277
|
_Pragma("omp simd")
|
|
3278
3278
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3279
|
-
|
|
3279
|
+
ca_u_uint32_t v = __li_row[__j];
|
|
3280
3280
|
__li_buf_acc[__j] += v;
|
|
3281
3281
|
(void) v;
|
|
3282
3282
|
}
|
|
3283
3283
|
}
|
|
3284
3284
|
|
|
3285
3285
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3286
|
-
__li_op_k[__li_tile + __j] = (
|
|
3286
|
+
__li_op_k[__li_tile + __j] = (ca_u_uint32_t) (__li_buf_acc[__j]);
|
|
3287
3287
|
}
|
|
3288
3288
|
}
|
|
3289
3289
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -3302,7 +3302,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3302
3302
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
3303
3303
|
CArray *co;
|
|
3304
3304
|
GetCArray(vout, co);
|
|
3305
|
-
|
|
3305
|
+
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
3306
3306
|
ca_iter_state st;
|
|
3307
3307
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
3308
3308
|
slab_axes, naxes, 0);
|
|
@@ -3315,10 +3315,10 @@ boolean8_t *m;
|
|
|
3315
3315
|
ca_size_t out_i = 0;
|
|
3316
3316
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
3317
3317
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
3318
|
-
|
|
3318
|
+
ca_u_uint32_t acc;
|
|
3319
3319
|
ca_size_t masked_cnt = 0;
|
|
3320
|
-
#define __accumulate_u32_EXPR(__x) ((
|
|
3321
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
3320
|
+
#define __accumulate_u32_EXPR(__x) ((ca_u_uint32_t)(__x))
|
|
3321
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_uint32_t, ca_u_uint32_t, st, p, m, acc, 0, __accumulate_u32_EXPR, masked_cnt);
|
|
3322
3322
|
#undef __accumulate_u32_EXPR
|
|
3323
3323
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
3324
3324
|
if ( ! op_mask ) {
|
|
@@ -3326,9 +3326,9 @@ ca_size_t out_i = 0;
|
|
|
3326
3326
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
3327
3327
|
}
|
|
3328
3328
|
op_mask[out_i] = 1;
|
|
3329
|
-
op[out_i++] = (
|
|
3329
|
+
op[out_i++] = (ca_u_uint32_t) 0; /* sentinel; mask bit is what counts */
|
|
3330
3330
|
} else {
|
|
3331
|
-
op[out_i++] = (
|
|
3331
|
+
op[out_i++] = (ca_u_uint32_t) (acc);
|
|
3332
3332
|
}
|
|
3333
3333
|
}
|
|
3334
3334
|
ca_iter_state_finish(&st);
|
|
@@ -3363,18 +3363,18 @@ accumulate_ki_stream_i64 (VALUE arg)
|
|
|
3363
3363
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
3364
3364
|
ca_size_t __counts[CA_RANK_MAX];
|
|
3365
3365
|
ca_size_t __strides[CA_RANK_MAX];
|
|
3366
|
-
ca_size_t __s = sizeof(
|
|
3366
|
+
ca_size_t __s = sizeof(ca_u_int64_t);
|
|
3367
3367
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
3368
3368
|
__strides[__k] = __s;
|
|
3369
3369
|
__s *= ca->dim[__k];
|
|
3370
3370
|
}
|
|
3371
3371
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
3372
3372
|
ca_size_t __outer_off = 0;
|
|
3373
|
-
|
|
3373
|
+
ca_u_int64_t acc;
|
|
3374
3374
|
acc = (0);
|
|
3375
3375
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
3376
3376
|
(void) masked_cnt;
|
|
3377
|
-
|
|
3377
|
+
ca_u_int64_t *__chunk = (ca_u_int64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_int64_t));
|
|
3378
3378
|
__ctx->chunk = __chunk;
|
|
3379
3379
|
while ( __outer_off < __outer ) {
|
|
3380
3380
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -3384,12 +3384,12 @@ accumulate_ki_stream_i64 (VALUE arg)
|
|
|
3384
3384
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
3385
3385
|
ca_size_t __i;
|
|
3386
3386
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
3387
|
-
|
|
3387
|
+
ca_u_int64_t v = __chunk[__i];
|
|
3388
3388
|
acc += v;
|
|
3389
3389
|
}
|
|
3390
3390
|
__outer_off += __r;
|
|
3391
3391
|
}
|
|
3392
|
-
return LL2NUM((
|
|
3392
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
3393
3393
|
}
|
|
3394
3394
|
|
|
3395
3395
|
static VALUE
|
|
@@ -3433,36 +3433,36 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3433
3433
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
3434
3434
|
CArray *co;
|
|
3435
3435
|
GetCArray(vout, co);
|
|
3436
|
-
|
|
3436
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
3437
3437
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
3438
|
-
const
|
|
3438
|
+
const ca_u_int64_t *p_root = (const ca_u_int64_t *) ca->ptr;
|
|
3439
3439
|
|
|
3440
3440
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
3441
3441
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
3442
|
-
|
|
3442
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
3443
3443
|
|
|
3444
3444
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3445
3445
|
const ca_size_t __li_TILE = 512;
|
|
3446
3446
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
3447
|
-
const
|
|
3447
|
+
const ca_u_int64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
3448
3448
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
3449
3449
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
3450
3450
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
3451
3451
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3452
|
-
__li_buf_acc[__j] = (
|
|
3452
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
3453
3453
|
}
|
|
3454
3454
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
3455
|
-
const
|
|
3455
|
+
const ca_u_int64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
3456
3456
|
_Pragma("omp simd")
|
|
3457
3457
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3458
|
-
|
|
3458
|
+
ca_u_int64_t v = __li_row[__j];
|
|
3459
3459
|
__li_buf_acc[__j] += v;
|
|
3460
3460
|
(void) v;
|
|
3461
3461
|
}
|
|
3462
3462
|
}
|
|
3463
3463
|
|
|
3464
3464
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3465
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
3465
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
3466
3466
|
}
|
|
3467
3467
|
}
|
|
3468
3468
|
}
|
|
@@ -3513,37 +3513,37 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3513
3513
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
3514
3514
|
CArray *co;
|
|
3515
3515
|
GetCArray(vout, co);
|
|
3516
|
-
|
|
3516
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
3517
3517
|
|
|
3518
|
-
|
|
3518
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
3519
3519
|
|
|
3520
3520
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3521
3521
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
3522
3522
|
const ca_size_t __li_TILE = 512;
|
|
3523
3523
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3524
3524
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
3525
|
-
const
|
|
3526
|
-
|
|
3525
|
+
const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
|
|
3526
|
+
ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
3527
3527
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
3528
|
-
const
|
|
3528
|
+
const ca_u_int64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
3529
3529
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
3530
3530
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
3531
3531
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
3532
3532
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3533
|
-
__li_buf_acc[__j] = (
|
|
3533
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
3534
3534
|
}
|
|
3535
3535
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
3536
|
-
const
|
|
3536
|
+
const ca_u_int64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
3537
3537
|
_Pragma("omp simd")
|
|
3538
3538
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3539
|
-
|
|
3539
|
+
ca_u_int64_t v = __li_row[__j];
|
|
3540
3540
|
__li_buf_acc[__j] += v;
|
|
3541
3541
|
(void) v;
|
|
3542
3542
|
}
|
|
3543
3543
|
}
|
|
3544
3544
|
|
|
3545
3545
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3546
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
3546
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
3547
3547
|
}
|
|
3548
3548
|
}
|
|
3549
3549
|
}
|
|
@@ -3604,7 +3604,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3604
3604
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
3605
3605
|
CArray *co;
|
|
3606
3606
|
GetCArray(vout, co);
|
|
3607
|
-
|
|
3607
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
3608
3608
|
|
|
3609
3609
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
3610
3610
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -3633,13 +3633,13 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3633
3633
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
3634
3634
|
}
|
|
3635
3635
|
|
|
3636
|
-
|
|
3636
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
3637
3637
|
|
|
3638
3638
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3639
3639
|
const ca_size_t __li_TILE = 512;
|
|
3640
3640
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3641
3641
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
3642
|
-
const
|
|
3642
|
+
const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
|
|
3643
3643
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3644
3644
|
|
|
3645
3645
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -3650,27 +3650,27 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3650
3650
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
3651
3651
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
3652
3652
|
}
|
|
3653
|
-
|
|
3654
|
-
const
|
|
3653
|
+
ca_u_int64_t *__li_op_k = op + __li_obase;
|
|
3654
|
+
const ca_u_int64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
3655
3655
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
3656
3656
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
3657
3657
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
3658
3658
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
3659
3659
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3660
|
-
__li_buf_acc[__j] = (
|
|
3660
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
3661
3661
|
}
|
|
3662
3662
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
3663
|
-
const
|
|
3663
|
+
const ca_u_int64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
3664
3664
|
_Pragma("omp simd")
|
|
3665
3665
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3666
|
-
|
|
3666
|
+
ca_u_int64_t v = __li_row[__j];
|
|
3667
3667
|
__li_buf_acc[__j] += v;
|
|
3668
3668
|
(void) v;
|
|
3669
3669
|
}
|
|
3670
3670
|
}
|
|
3671
3671
|
|
|
3672
3672
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3673
|
-
__li_op_k[__li_tile + __j] = (
|
|
3673
|
+
__li_op_k[__li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
3674
3674
|
}
|
|
3675
3675
|
}
|
|
3676
3676
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -3751,7 +3751,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3751
3751
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
3752
3752
|
CArray *co;
|
|
3753
3753
|
GetCArray(vout, co);
|
|
3754
|
-
|
|
3754
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
3755
3755
|
|
|
3756
3756
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
3757
3757
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -3794,12 +3794,12 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3794
3794
|
ca_size_t __li_OUTER = 1;
|
|
3795
3795
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
3796
3796
|
|
|
3797
|
-
|
|
3797
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
3798
3798
|
|
|
3799
3799
|
const ca_size_t __li_TILE = 512;
|
|
3800
3800
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3801
3801
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
3802
|
-
const
|
|
3802
|
+
const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
|
|
3803
3803
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3804
3804
|
|
|
3805
3805
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -3812,27 +3812,27 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3812
3812
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
3813
3813
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
3814
3814
|
}
|
|
3815
|
-
|
|
3816
|
-
const
|
|
3815
|
+
ca_u_int64_t *__li_op_k = op + __li_obase;
|
|
3816
|
+
const ca_u_int64_t *__li_plane = p_root + __li_pbase;
|
|
3817
3817
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
3818
3818
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
3819
3819
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
3820
3820
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
3821
3821
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3822
|
-
__li_buf_acc[__j] = (
|
|
3822
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
3823
3823
|
}
|
|
3824
3824
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
3825
|
-
const
|
|
3825
|
+
const ca_u_int64_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
3826
3826
|
_Pragma("omp simd")
|
|
3827
3827
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3828
|
-
|
|
3828
|
+
ca_u_int64_t v = __li_row[__j];
|
|
3829
3829
|
__li_buf_acc[__j] += v;
|
|
3830
3830
|
(void) v;
|
|
3831
3831
|
}
|
|
3832
3832
|
}
|
|
3833
3833
|
|
|
3834
3834
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3835
|
-
__li_op_k[__li_tile + __j] = (
|
|
3835
|
+
__li_op_k[__li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
3836
3836
|
}
|
|
3837
3837
|
}
|
|
3838
3838
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -3851,7 +3851,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3851
3851
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
3852
3852
|
CArray *co;
|
|
3853
3853
|
GetCArray(vout, co);
|
|
3854
|
-
|
|
3854
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
3855
3855
|
ca_iter_state st;
|
|
3856
3856
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
3857
3857
|
slab_axes, naxes, 0);
|
|
@@ -3864,10 +3864,10 @@ boolean8_t *m;
|
|
|
3864
3864
|
ca_size_t out_i = 0;
|
|
3865
3865
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
3866
3866
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
3867
|
-
|
|
3867
|
+
ca_u_int64_t acc;
|
|
3868
3868
|
ca_size_t masked_cnt = 0;
|
|
3869
|
-
#define __accumulate_i64_EXPR(__x) ((
|
|
3870
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
3869
|
+
#define __accumulate_i64_EXPR(__x) ((ca_u_int64_t)(__x))
|
|
3870
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_int64_t, ca_u_int64_t, st, p, m, acc, 0, __accumulate_i64_EXPR, masked_cnt);
|
|
3871
3871
|
#undef __accumulate_i64_EXPR
|
|
3872
3872
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
3873
3873
|
if ( ! op_mask ) {
|
|
@@ -3875,9 +3875,9 @@ ca_size_t out_i = 0;
|
|
|
3875
3875
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
3876
3876
|
}
|
|
3877
3877
|
op_mask[out_i] = 1;
|
|
3878
|
-
op[out_i++] = (
|
|
3878
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
3879
3879
|
} else {
|
|
3880
|
-
op[out_i++] = (
|
|
3880
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
3881
3881
|
}
|
|
3882
3882
|
}
|
|
3883
3883
|
ca_iter_state_finish(&st);
|
|
@@ -3912,18 +3912,18 @@ accumulate_ki_stream_u64 (VALUE arg)
|
|
|
3912
3912
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
3913
3913
|
ca_size_t __counts[CA_RANK_MAX];
|
|
3914
3914
|
ca_size_t __strides[CA_RANK_MAX];
|
|
3915
|
-
ca_size_t __s = sizeof(
|
|
3915
|
+
ca_size_t __s = sizeof(ca_u_uint64_t);
|
|
3916
3916
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
3917
3917
|
__strides[__k] = __s;
|
|
3918
3918
|
__s *= ca->dim[__k];
|
|
3919
3919
|
}
|
|
3920
3920
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
3921
3921
|
ca_size_t __outer_off = 0;
|
|
3922
|
-
|
|
3922
|
+
ca_u_uint64_t acc;
|
|
3923
3923
|
acc = (0);
|
|
3924
3924
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
3925
3925
|
(void) masked_cnt;
|
|
3926
|
-
|
|
3926
|
+
ca_u_uint64_t *__chunk = (ca_u_uint64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_uint64_t));
|
|
3927
3927
|
__ctx->chunk = __chunk;
|
|
3928
3928
|
while ( __outer_off < __outer ) {
|
|
3929
3929
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -3933,12 +3933,12 @@ accumulate_ki_stream_u64 (VALUE arg)
|
|
|
3933
3933
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
3934
3934
|
ca_size_t __i;
|
|
3935
3935
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
3936
|
-
|
|
3936
|
+
ca_u_uint64_t v = __chunk[__i];
|
|
3937
3937
|
acc += v;
|
|
3938
3938
|
}
|
|
3939
3939
|
__outer_off += __r;
|
|
3940
3940
|
}
|
|
3941
|
-
return ULL2NUM((
|
|
3941
|
+
return ULL2NUM((ca_u_uint64_t) (acc));
|
|
3942
3942
|
}
|
|
3943
3943
|
|
|
3944
3944
|
static VALUE
|
|
@@ -3982,36 +3982,36 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3982
3982
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
3983
3983
|
CArray *co;
|
|
3984
3984
|
GetCArray(vout, co);
|
|
3985
|
-
|
|
3985
|
+
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
3986
3986
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
3987
|
-
const
|
|
3987
|
+
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) ca->ptr;
|
|
3988
3988
|
|
|
3989
3989
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
3990
3990
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
3991
|
-
|
|
3991
|
+
ca_u_uint64_t __li_buf_acc[512];
|
|
3992
3992
|
|
|
3993
3993
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3994
3994
|
const ca_size_t __li_TILE = 512;
|
|
3995
3995
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
3996
|
-
const
|
|
3996
|
+
const ca_u_uint64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
3997
3997
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
3998
3998
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
3999
3999
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
4000
4000
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4001
|
-
__li_buf_acc[__j] = (
|
|
4001
|
+
__li_buf_acc[__j] = (ca_u_uint64_t) (0);
|
|
4002
4002
|
}
|
|
4003
4003
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4004
|
-
const
|
|
4004
|
+
const ca_u_uint64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4005
4005
|
_Pragma("omp simd")
|
|
4006
4006
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4007
|
-
|
|
4007
|
+
ca_u_uint64_t v = __li_row[__j];
|
|
4008
4008
|
__li_buf_acc[__j] += v;
|
|
4009
4009
|
(void) v;
|
|
4010
4010
|
}
|
|
4011
4011
|
}
|
|
4012
4012
|
|
|
4013
4013
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4014
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
4014
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_uint64_t) (__li_buf_acc[__j]);
|
|
4015
4015
|
}
|
|
4016
4016
|
}
|
|
4017
4017
|
}
|
|
@@ -4062,37 +4062,37 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4062
4062
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
4063
4063
|
CArray *co;
|
|
4064
4064
|
GetCArray(vout, co);
|
|
4065
|
-
|
|
4065
|
+
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
4066
4066
|
|
|
4067
|
-
|
|
4067
|
+
ca_u_uint64_t __li_buf_acc[512];
|
|
4068
4068
|
|
|
4069
4069
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4070
4070
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
4071
4071
|
const ca_size_t __li_TILE = 512;
|
|
4072
4072
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4073
4073
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
4074
|
-
const
|
|
4075
|
-
|
|
4074
|
+
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
|
|
4075
|
+
ca_u_uint64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
4076
4076
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
4077
|
-
const
|
|
4077
|
+
const ca_u_uint64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
4078
4078
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
4079
4079
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
4080
4080
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
4081
4081
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4082
|
-
__li_buf_acc[__j] = (
|
|
4082
|
+
__li_buf_acc[__j] = (ca_u_uint64_t) (0);
|
|
4083
4083
|
}
|
|
4084
4084
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4085
|
-
const
|
|
4085
|
+
const ca_u_uint64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4086
4086
|
_Pragma("omp simd")
|
|
4087
4087
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4088
|
-
|
|
4088
|
+
ca_u_uint64_t v = __li_row[__j];
|
|
4089
4089
|
__li_buf_acc[__j] += v;
|
|
4090
4090
|
(void) v;
|
|
4091
4091
|
}
|
|
4092
4092
|
}
|
|
4093
4093
|
|
|
4094
4094
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4095
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
4095
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_uint64_t) (__li_buf_acc[__j]);
|
|
4096
4096
|
}
|
|
4097
4097
|
}
|
|
4098
4098
|
}
|
|
@@ -4153,7 +4153,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4153
4153
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
4154
4154
|
CArray *co;
|
|
4155
4155
|
GetCArray(vout, co);
|
|
4156
|
-
|
|
4156
|
+
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
4157
4157
|
|
|
4158
4158
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
4159
4159
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -4182,13 +4182,13 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4182
4182
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
4183
4183
|
}
|
|
4184
4184
|
|
|
4185
|
-
|
|
4185
|
+
ca_u_uint64_t __li_buf_acc[512];
|
|
4186
4186
|
|
|
4187
4187
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4188
4188
|
const ca_size_t __li_TILE = 512;
|
|
4189
4189
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4190
4190
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
4191
|
-
const
|
|
4191
|
+
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
|
|
4192
4192
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4193
4193
|
|
|
4194
4194
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -4199,27 +4199,27 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4199
4199
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
4200
4200
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
4201
4201
|
}
|
|
4202
|
-
|
|
4203
|
-
const
|
|
4202
|
+
ca_u_uint64_t *__li_op_k = op + __li_obase;
|
|
4203
|
+
const ca_u_uint64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
4204
4204
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
4205
4205
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
4206
4206
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
4207
4207
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
4208
4208
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4209
|
-
__li_buf_acc[__j] = (
|
|
4209
|
+
__li_buf_acc[__j] = (ca_u_uint64_t) (0);
|
|
4210
4210
|
}
|
|
4211
4211
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4212
|
-
const
|
|
4212
|
+
const ca_u_uint64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4213
4213
|
_Pragma("omp simd")
|
|
4214
4214
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4215
|
-
|
|
4215
|
+
ca_u_uint64_t v = __li_row[__j];
|
|
4216
4216
|
__li_buf_acc[__j] += v;
|
|
4217
4217
|
(void) v;
|
|
4218
4218
|
}
|
|
4219
4219
|
}
|
|
4220
4220
|
|
|
4221
4221
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4222
|
-
__li_op_k[__li_tile + __j] = (
|
|
4222
|
+
__li_op_k[__li_tile + __j] = (ca_u_uint64_t) (__li_buf_acc[__j]);
|
|
4223
4223
|
}
|
|
4224
4224
|
}
|
|
4225
4225
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -4300,7 +4300,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4300
4300
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
4301
4301
|
CArray *co;
|
|
4302
4302
|
GetCArray(vout, co);
|
|
4303
|
-
|
|
4303
|
+
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
4304
4304
|
|
|
4305
4305
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
4306
4306
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -4343,12 +4343,12 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4343
4343
|
ca_size_t __li_OUTER = 1;
|
|
4344
4344
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
4345
4345
|
|
|
4346
|
-
|
|
4346
|
+
ca_u_uint64_t __li_buf_acc[512];
|
|
4347
4347
|
|
|
4348
4348
|
const ca_size_t __li_TILE = 512;
|
|
4349
4349
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4350
4350
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
4351
|
-
const
|
|
4351
|
+
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
|
|
4352
4352
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4353
4353
|
|
|
4354
4354
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -4361,27 +4361,27 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4361
4361
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
4362
4362
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
4363
4363
|
}
|
|
4364
|
-
|
|
4365
|
-
const
|
|
4364
|
+
ca_u_uint64_t *__li_op_k = op + __li_obase;
|
|
4365
|
+
const ca_u_uint64_t *__li_plane = p_root + __li_pbase;
|
|
4366
4366
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
4367
4367
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
4368
4368
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
4369
4369
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
4370
4370
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4371
|
-
__li_buf_acc[__j] = (
|
|
4371
|
+
__li_buf_acc[__j] = (ca_u_uint64_t) (0);
|
|
4372
4372
|
}
|
|
4373
4373
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4374
|
-
const
|
|
4374
|
+
const ca_u_uint64_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
4375
4375
|
_Pragma("omp simd")
|
|
4376
4376
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4377
|
-
|
|
4377
|
+
ca_u_uint64_t v = __li_row[__j];
|
|
4378
4378
|
__li_buf_acc[__j] += v;
|
|
4379
4379
|
(void) v;
|
|
4380
4380
|
}
|
|
4381
4381
|
}
|
|
4382
4382
|
|
|
4383
4383
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4384
|
-
__li_op_k[__li_tile + __j] = (
|
|
4384
|
+
__li_op_k[__li_tile + __j] = (ca_u_uint64_t) (__li_buf_acc[__j]);
|
|
4385
4385
|
}
|
|
4386
4386
|
}
|
|
4387
4387
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -4400,7 +4400,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4400
4400
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
4401
4401
|
CArray *co;
|
|
4402
4402
|
GetCArray(vout, co);
|
|
4403
|
-
|
|
4403
|
+
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
4404
4404
|
ca_iter_state st;
|
|
4405
4405
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
4406
4406
|
slab_axes, naxes, 0);
|
|
@@ -4413,10 +4413,10 @@ boolean8_t *m;
|
|
|
4413
4413
|
ca_size_t out_i = 0;
|
|
4414
4414
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
4415
4415
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
4416
|
-
|
|
4416
|
+
ca_u_uint64_t acc;
|
|
4417
4417
|
ca_size_t masked_cnt = 0;
|
|
4418
|
-
#define __accumulate_u64_EXPR(__x) ((
|
|
4419
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
4418
|
+
#define __accumulate_u64_EXPR(__x) ((ca_u_uint64_t)(__x))
|
|
4419
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_uint64_t, ca_u_uint64_t, st, p, m, acc, 0, __accumulate_u64_EXPR, masked_cnt);
|
|
4420
4420
|
#undef __accumulate_u64_EXPR
|
|
4421
4421
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
4422
4422
|
if ( ! op_mask ) {
|
|
@@ -4424,9 +4424,9 @@ ca_size_t out_i = 0;
|
|
|
4424
4424
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
4425
4425
|
}
|
|
4426
4426
|
op_mask[out_i] = 1;
|
|
4427
|
-
op[out_i++] = (
|
|
4427
|
+
op[out_i++] = (ca_u_uint64_t) 0; /* sentinel; mask bit is what counts */
|
|
4428
4428
|
} else {
|
|
4429
|
-
op[out_i++] = (
|
|
4429
|
+
op[out_i++] = (ca_u_uint64_t) (acc);
|
|
4430
4430
|
}
|
|
4431
4431
|
}
|
|
4432
4432
|
ca_iter_state_finish(&st);
|
|
@@ -4461,18 +4461,18 @@ accumulate_ki_stream_f32 (VALUE arg)
|
|
|
4461
4461
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
4462
4462
|
ca_size_t __counts[CA_RANK_MAX];
|
|
4463
4463
|
ca_size_t __strides[CA_RANK_MAX];
|
|
4464
|
-
ca_size_t __s = sizeof(
|
|
4464
|
+
ca_size_t __s = sizeof(ca_u_float32_t);
|
|
4465
4465
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
4466
4466
|
__strides[__k] = __s;
|
|
4467
4467
|
__s *= ca->dim[__k];
|
|
4468
4468
|
}
|
|
4469
4469
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
4470
4470
|
ca_size_t __outer_off = 0;
|
|
4471
|
-
|
|
4471
|
+
ca_u_float32_t acc;
|
|
4472
4472
|
acc = (0);
|
|
4473
4473
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
4474
4474
|
(void) masked_cnt;
|
|
4475
|
-
|
|
4475
|
+
ca_u_float32_t *__chunk = (ca_u_float32_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_float32_t));
|
|
4476
4476
|
__ctx->chunk = __chunk;
|
|
4477
4477
|
while ( __outer_off < __outer ) {
|
|
4478
4478
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -4482,12 +4482,12 @@ accumulate_ki_stream_f32 (VALUE arg)
|
|
|
4482
4482
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
4483
4483
|
ca_size_t __i;
|
|
4484
4484
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
4485
|
-
|
|
4485
|
+
ca_u_float32_t v = __chunk[__i];
|
|
4486
4486
|
acc += v;
|
|
4487
4487
|
}
|
|
4488
4488
|
__outer_off += __r;
|
|
4489
4489
|
}
|
|
4490
|
-
return rb_float_new((
|
|
4490
|
+
return rb_float_new((ca_u_float32_t) (acc));
|
|
4491
4491
|
}
|
|
4492
4492
|
|
|
4493
4493
|
static VALUE
|
|
@@ -4531,36 +4531,36 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4531
4531
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
4532
4532
|
CArray *co;
|
|
4533
4533
|
GetCArray(vout, co);
|
|
4534
|
-
|
|
4534
|
+
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
4535
4535
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
4536
|
-
const
|
|
4536
|
+
const ca_u_float32_t *p_root = (const ca_u_float32_t *) ca->ptr;
|
|
4537
4537
|
|
|
4538
4538
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
4539
4539
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
4540
|
-
|
|
4540
|
+
ca_u_float32_t __li_buf_acc[512];
|
|
4541
4541
|
|
|
4542
4542
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4543
4543
|
const ca_size_t __li_TILE = 512;
|
|
4544
4544
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
4545
|
-
const
|
|
4545
|
+
const ca_u_float32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
4546
4546
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
4547
4547
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
4548
4548
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
4549
4549
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4550
|
-
__li_buf_acc[__j] = (
|
|
4550
|
+
__li_buf_acc[__j] = (ca_u_float32_t) (0);
|
|
4551
4551
|
}
|
|
4552
4552
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4553
|
-
const
|
|
4553
|
+
const ca_u_float32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4554
4554
|
_Pragma("omp simd")
|
|
4555
4555
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4556
|
-
|
|
4556
|
+
ca_u_float32_t v = __li_row[__j];
|
|
4557
4557
|
__li_buf_acc[__j] += v;
|
|
4558
4558
|
(void) v;
|
|
4559
4559
|
}
|
|
4560
4560
|
}
|
|
4561
4561
|
|
|
4562
4562
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4563
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
4563
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_float32_t) (__li_buf_acc[__j]);
|
|
4564
4564
|
}
|
|
4565
4565
|
}
|
|
4566
4566
|
}
|
|
@@ -4611,37 +4611,37 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4611
4611
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
4612
4612
|
CArray *co;
|
|
4613
4613
|
GetCArray(vout, co);
|
|
4614
|
-
|
|
4614
|
+
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
4615
4615
|
|
|
4616
|
-
|
|
4616
|
+
ca_u_float32_t __li_buf_acc[512];
|
|
4617
4617
|
|
|
4618
4618
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4619
4619
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
4620
4620
|
const ca_size_t __li_TILE = 512;
|
|
4621
4621
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4622
4622
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
4623
|
-
const
|
|
4624
|
-
|
|
4623
|
+
const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
|
|
4624
|
+
ca_u_float32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
4625
4625
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
4626
|
-
const
|
|
4626
|
+
const ca_u_float32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
4627
4627
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
4628
4628
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
4629
4629
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
4630
4630
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4631
|
-
__li_buf_acc[__j] = (
|
|
4631
|
+
__li_buf_acc[__j] = (ca_u_float32_t) (0);
|
|
4632
4632
|
}
|
|
4633
4633
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4634
|
-
const
|
|
4634
|
+
const ca_u_float32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4635
4635
|
_Pragma("omp simd")
|
|
4636
4636
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4637
|
-
|
|
4637
|
+
ca_u_float32_t v = __li_row[__j];
|
|
4638
4638
|
__li_buf_acc[__j] += v;
|
|
4639
4639
|
(void) v;
|
|
4640
4640
|
}
|
|
4641
4641
|
}
|
|
4642
4642
|
|
|
4643
4643
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4644
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
4644
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_float32_t) (__li_buf_acc[__j]);
|
|
4645
4645
|
}
|
|
4646
4646
|
}
|
|
4647
4647
|
}
|
|
@@ -4702,7 +4702,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4702
4702
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
4703
4703
|
CArray *co;
|
|
4704
4704
|
GetCArray(vout, co);
|
|
4705
|
-
|
|
4705
|
+
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
4706
4706
|
|
|
4707
4707
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
4708
4708
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -4731,13 +4731,13 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4731
4731
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
4732
4732
|
}
|
|
4733
4733
|
|
|
4734
|
-
|
|
4734
|
+
ca_u_float32_t __li_buf_acc[512];
|
|
4735
4735
|
|
|
4736
4736
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4737
4737
|
const ca_size_t __li_TILE = 512;
|
|
4738
4738
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4739
4739
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
4740
|
-
const
|
|
4740
|
+
const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
|
|
4741
4741
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4742
4742
|
|
|
4743
4743
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -4748,27 +4748,27 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4748
4748
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
4749
4749
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
4750
4750
|
}
|
|
4751
|
-
|
|
4752
|
-
const
|
|
4751
|
+
ca_u_float32_t *__li_op_k = op + __li_obase;
|
|
4752
|
+
const ca_u_float32_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
4753
4753
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
4754
4754
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
4755
4755
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
4756
4756
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
4757
4757
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4758
|
-
__li_buf_acc[__j] = (
|
|
4758
|
+
__li_buf_acc[__j] = (ca_u_float32_t) (0);
|
|
4759
4759
|
}
|
|
4760
4760
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4761
|
-
const
|
|
4761
|
+
const ca_u_float32_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4762
4762
|
_Pragma("omp simd")
|
|
4763
4763
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4764
|
-
|
|
4764
|
+
ca_u_float32_t v = __li_row[__j];
|
|
4765
4765
|
__li_buf_acc[__j] += v;
|
|
4766
4766
|
(void) v;
|
|
4767
4767
|
}
|
|
4768
4768
|
}
|
|
4769
4769
|
|
|
4770
4770
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4771
|
-
__li_op_k[__li_tile + __j] = (
|
|
4771
|
+
__li_op_k[__li_tile + __j] = (ca_u_float32_t) (__li_buf_acc[__j]);
|
|
4772
4772
|
}
|
|
4773
4773
|
}
|
|
4774
4774
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -4849,7 +4849,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4849
4849
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
4850
4850
|
CArray *co;
|
|
4851
4851
|
GetCArray(vout, co);
|
|
4852
|
-
|
|
4852
|
+
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
4853
4853
|
|
|
4854
4854
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
4855
4855
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -4892,12 +4892,12 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4892
4892
|
ca_size_t __li_OUTER = 1;
|
|
4893
4893
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
4894
4894
|
|
|
4895
|
-
|
|
4895
|
+
ca_u_float32_t __li_buf_acc[512];
|
|
4896
4896
|
|
|
4897
4897
|
const ca_size_t __li_TILE = 512;
|
|
4898
4898
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4899
4899
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
4900
|
-
const
|
|
4900
|
+
const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
|
|
4901
4901
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4902
4902
|
|
|
4903
4903
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -4910,27 +4910,27 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4910
4910
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
4911
4911
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
4912
4912
|
}
|
|
4913
|
-
|
|
4914
|
-
const
|
|
4913
|
+
ca_u_float32_t *__li_op_k = op + __li_obase;
|
|
4914
|
+
const ca_u_float32_t *__li_plane = p_root + __li_pbase;
|
|
4915
4915
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
4916
4916
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
4917
4917
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
4918
4918
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
4919
4919
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4920
|
-
__li_buf_acc[__j] = (
|
|
4920
|
+
__li_buf_acc[__j] = (ca_u_float32_t) (0);
|
|
4921
4921
|
}
|
|
4922
4922
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4923
|
-
const
|
|
4923
|
+
const ca_u_float32_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
4924
4924
|
_Pragma("omp simd")
|
|
4925
4925
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4926
|
-
|
|
4926
|
+
ca_u_float32_t v = __li_row[__j];
|
|
4927
4927
|
__li_buf_acc[__j] += v;
|
|
4928
4928
|
(void) v;
|
|
4929
4929
|
}
|
|
4930
4930
|
}
|
|
4931
4931
|
|
|
4932
4932
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4933
|
-
__li_op_k[__li_tile + __j] = (
|
|
4933
|
+
__li_op_k[__li_tile + __j] = (ca_u_float32_t) (__li_buf_acc[__j]);
|
|
4934
4934
|
}
|
|
4935
4935
|
}
|
|
4936
4936
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -4949,7 +4949,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4949
4949
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
4950
4950
|
CArray *co;
|
|
4951
4951
|
GetCArray(vout, co);
|
|
4952
|
-
|
|
4952
|
+
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
4953
4953
|
ca_iter_state st;
|
|
4954
4954
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
4955
4955
|
slab_axes, naxes, 0);
|
|
@@ -4962,10 +4962,10 @@ boolean8_t *m;
|
|
|
4962
4962
|
ca_size_t out_i = 0;
|
|
4963
4963
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
4964
4964
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
4965
|
-
|
|
4965
|
+
ca_u_float32_t acc;
|
|
4966
4966
|
ca_size_t masked_cnt = 0;
|
|
4967
|
-
#define __accumulate_f32_EXPR(__x) ((
|
|
4968
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
4967
|
+
#define __accumulate_f32_EXPR(__x) ((ca_u_float32_t)(__x))
|
|
4968
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_float32_t, ca_u_float32_t, st, p, m, acc, 0, __accumulate_f32_EXPR, masked_cnt);
|
|
4969
4969
|
#undef __accumulate_f32_EXPR
|
|
4970
4970
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
4971
4971
|
if ( ! op_mask ) {
|
|
@@ -4973,9 +4973,9 @@ ca_size_t out_i = 0;
|
|
|
4973
4973
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
4974
4974
|
}
|
|
4975
4975
|
op_mask[out_i] = 1;
|
|
4976
|
-
op[out_i++] = (
|
|
4976
|
+
op[out_i++] = (ca_u_float32_t) 0; /* sentinel; mask bit is what counts */
|
|
4977
4977
|
} else {
|
|
4978
|
-
op[out_i++] = (
|
|
4978
|
+
op[out_i++] = (ca_u_float32_t) (acc);
|
|
4979
4979
|
}
|
|
4980
4980
|
}
|
|
4981
4981
|
ca_iter_state_finish(&st);
|
|
@@ -5010,18 +5010,18 @@ accumulate_ki_stream_f64 (VALUE arg)
|
|
|
5010
5010
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
5011
5011
|
ca_size_t __counts[CA_RANK_MAX];
|
|
5012
5012
|
ca_size_t __strides[CA_RANK_MAX];
|
|
5013
|
-
ca_size_t __s = sizeof(
|
|
5013
|
+
ca_size_t __s = sizeof(ca_u_float64_t);
|
|
5014
5014
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
5015
5015
|
__strides[__k] = __s;
|
|
5016
5016
|
__s *= ca->dim[__k];
|
|
5017
5017
|
}
|
|
5018
5018
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
5019
5019
|
ca_size_t __outer_off = 0;
|
|
5020
|
-
|
|
5020
|
+
ca_u_float64_t acc;
|
|
5021
5021
|
acc = (0);
|
|
5022
5022
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
5023
5023
|
(void) masked_cnt;
|
|
5024
|
-
|
|
5024
|
+
ca_u_float64_t *__chunk = (ca_u_float64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_float64_t));
|
|
5025
5025
|
__ctx->chunk = __chunk;
|
|
5026
5026
|
while ( __outer_off < __outer ) {
|
|
5027
5027
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -5031,12 +5031,12 @@ accumulate_ki_stream_f64 (VALUE arg)
|
|
|
5031
5031
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
5032
5032
|
ca_size_t __i;
|
|
5033
5033
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
5034
|
-
|
|
5034
|
+
ca_u_float64_t v = __chunk[__i];
|
|
5035
5035
|
acc += v;
|
|
5036
5036
|
}
|
|
5037
5037
|
__outer_off += __r;
|
|
5038
5038
|
}
|
|
5039
|
-
return rb_float_new((
|
|
5039
|
+
return rb_float_new((ca_u_float64_t) (acc));
|
|
5040
5040
|
}
|
|
5041
5041
|
|
|
5042
5042
|
static VALUE
|
|
@@ -5080,36 +5080,36 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5080
5080
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
5081
5081
|
CArray *co;
|
|
5082
5082
|
GetCArray(vout, co);
|
|
5083
|
-
|
|
5083
|
+
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
5084
5084
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
5085
|
-
const
|
|
5085
|
+
const ca_u_float64_t *p_root = (const ca_u_float64_t *) ca->ptr;
|
|
5086
5086
|
|
|
5087
5087
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
5088
5088
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
5089
|
-
|
|
5089
|
+
ca_u_float64_t __li_buf_acc[512];
|
|
5090
5090
|
|
|
5091
5091
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5092
5092
|
const ca_size_t __li_TILE = 512;
|
|
5093
5093
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
5094
|
-
const
|
|
5094
|
+
const ca_u_float64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
5095
5095
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
5096
5096
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
5097
5097
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
5098
5098
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5099
|
-
__li_buf_acc[__j] = (
|
|
5099
|
+
__li_buf_acc[__j] = (ca_u_float64_t) (0);
|
|
5100
5100
|
}
|
|
5101
5101
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5102
|
-
const
|
|
5102
|
+
const ca_u_float64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5103
5103
|
_Pragma("omp simd")
|
|
5104
5104
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5105
|
-
|
|
5105
|
+
ca_u_float64_t v = __li_row[__j];
|
|
5106
5106
|
__li_buf_acc[__j] += v;
|
|
5107
5107
|
(void) v;
|
|
5108
5108
|
}
|
|
5109
5109
|
}
|
|
5110
5110
|
|
|
5111
5111
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5112
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
5112
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_float64_t) (__li_buf_acc[__j]);
|
|
5113
5113
|
}
|
|
5114
5114
|
}
|
|
5115
5115
|
}
|
|
@@ -5160,37 +5160,37 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5160
5160
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
5161
5161
|
CArray *co;
|
|
5162
5162
|
GetCArray(vout, co);
|
|
5163
|
-
|
|
5163
|
+
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
5164
5164
|
|
|
5165
|
-
|
|
5165
|
+
ca_u_float64_t __li_buf_acc[512];
|
|
5166
5166
|
|
|
5167
5167
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5168
5168
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
5169
5169
|
const ca_size_t __li_TILE = 512;
|
|
5170
5170
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5171
5171
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
5172
|
-
const
|
|
5173
|
-
|
|
5172
|
+
const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
|
|
5173
|
+
ca_u_float64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
5174
5174
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
5175
|
-
const
|
|
5175
|
+
const ca_u_float64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
5176
5176
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
5177
5177
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
5178
5178
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
5179
5179
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5180
|
-
__li_buf_acc[__j] = (
|
|
5180
|
+
__li_buf_acc[__j] = (ca_u_float64_t) (0);
|
|
5181
5181
|
}
|
|
5182
5182
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5183
|
-
const
|
|
5183
|
+
const ca_u_float64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5184
5184
|
_Pragma("omp simd")
|
|
5185
5185
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5186
|
-
|
|
5186
|
+
ca_u_float64_t v = __li_row[__j];
|
|
5187
5187
|
__li_buf_acc[__j] += v;
|
|
5188
5188
|
(void) v;
|
|
5189
5189
|
}
|
|
5190
5190
|
}
|
|
5191
5191
|
|
|
5192
5192
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5193
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
5193
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_float64_t) (__li_buf_acc[__j]);
|
|
5194
5194
|
}
|
|
5195
5195
|
}
|
|
5196
5196
|
}
|
|
@@ -5251,7 +5251,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5251
5251
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
5252
5252
|
CArray *co;
|
|
5253
5253
|
GetCArray(vout, co);
|
|
5254
|
-
|
|
5254
|
+
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
5255
5255
|
|
|
5256
5256
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
5257
5257
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -5280,13 +5280,13 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5280
5280
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
5281
5281
|
}
|
|
5282
5282
|
|
|
5283
|
-
|
|
5283
|
+
ca_u_float64_t __li_buf_acc[512];
|
|
5284
5284
|
|
|
5285
5285
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5286
5286
|
const ca_size_t __li_TILE = 512;
|
|
5287
5287
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5288
5288
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
5289
|
-
const
|
|
5289
|
+
const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
|
|
5290
5290
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
5291
5291
|
|
|
5292
5292
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -5297,27 +5297,27 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5297
5297
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
5298
5298
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
5299
5299
|
}
|
|
5300
|
-
|
|
5301
|
-
const
|
|
5300
|
+
ca_u_float64_t *__li_op_k = op + __li_obase;
|
|
5301
|
+
const ca_u_float64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
5302
5302
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
5303
5303
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
5304
5304
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
5305
5305
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
5306
5306
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5307
|
-
__li_buf_acc[__j] = (
|
|
5307
|
+
__li_buf_acc[__j] = (ca_u_float64_t) (0);
|
|
5308
5308
|
}
|
|
5309
5309
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5310
|
-
const
|
|
5310
|
+
const ca_u_float64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5311
5311
|
_Pragma("omp simd")
|
|
5312
5312
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5313
|
-
|
|
5313
|
+
ca_u_float64_t v = __li_row[__j];
|
|
5314
5314
|
__li_buf_acc[__j] += v;
|
|
5315
5315
|
(void) v;
|
|
5316
5316
|
}
|
|
5317
5317
|
}
|
|
5318
5318
|
|
|
5319
5319
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5320
|
-
__li_op_k[__li_tile + __j] = (
|
|
5320
|
+
__li_op_k[__li_tile + __j] = (ca_u_float64_t) (__li_buf_acc[__j]);
|
|
5321
5321
|
}
|
|
5322
5322
|
}
|
|
5323
5323
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -5398,7 +5398,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5398
5398
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
5399
5399
|
CArray *co;
|
|
5400
5400
|
GetCArray(vout, co);
|
|
5401
|
-
|
|
5401
|
+
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
5402
5402
|
|
|
5403
5403
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
5404
5404
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -5441,12 +5441,12 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5441
5441
|
ca_size_t __li_OUTER = 1;
|
|
5442
5442
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
5443
5443
|
|
|
5444
|
-
|
|
5444
|
+
ca_u_float64_t __li_buf_acc[512];
|
|
5445
5445
|
|
|
5446
5446
|
const ca_size_t __li_TILE = 512;
|
|
5447
5447
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5448
5448
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
5449
|
-
const
|
|
5449
|
+
const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
|
|
5450
5450
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
5451
5451
|
|
|
5452
5452
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -5459,27 +5459,27 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5459
5459
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
5460
5460
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
5461
5461
|
}
|
|
5462
|
-
|
|
5463
|
-
const
|
|
5462
|
+
ca_u_float64_t *__li_op_k = op + __li_obase;
|
|
5463
|
+
const ca_u_float64_t *__li_plane = p_root + __li_pbase;
|
|
5464
5464
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
5465
5465
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
5466
5466
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
5467
5467
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
5468
5468
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5469
|
-
__li_buf_acc[__j] = (
|
|
5469
|
+
__li_buf_acc[__j] = (ca_u_float64_t) (0);
|
|
5470
5470
|
}
|
|
5471
5471
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5472
|
-
const
|
|
5472
|
+
const ca_u_float64_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
5473
5473
|
_Pragma("omp simd")
|
|
5474
5474
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5475
|
-
|
|
5475
|
+
ca_u_float64_t v = __li_row[__j];
|
|
5476
5476
|
__li_buf_acc[__j] += v;
|
|
5477
5477
|
(void) v;
|
|
5478
5478
|
}
|
|
5479
5479
|
}
|
|
5480
5480
|
|
|
5481
5481
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5482
|
-
__li_op_k[__li_tile + __j] = (
|
|
5482
|
+
__li_op_k[__li_tile + __j] = (ca_u_float64_t) (__li_buf_acc[__j]);
|
|
5483
5483
|
}
|
|
5484
5484
|
}
|
|
5485
5485
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -5498,7 +5498,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5498
5498
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
5499
5499
|
CArray *co;
|
|
5500
5500
|
GetCArray(vout, co);
|
|
5501
|
-
|
|
5501
|
+
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
5502
5502
|
ca_iter_state st;
|
|
5503
5503
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
5504
5504
|
slab_axes, naxes, 0);
|
|
@@ -5511,10 +5511,10 @@ boolean8_t *m;
|
|
|
5511
5511
|
ca_size_t out_i = 0;
|
|
5512
5512
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
5513
5513
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
5514
|
-
|
|
5514
|
+
ca_u_float64_t acc;
|
|
5515
5515
|
ca_size_t masked_cnt = 0;
|
|
5516
|
-
#define __accumulate_f64_EXPR(__x) ((
|
|
5517
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
5516
|
+
#define __accumulate_f64_EXPR(__x) ((ca_u_float64_t)(__x))
|
|
5517
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_float64_t, ca_u_float64_t, st, p, m, acc, 0, __accumulate_f64_EXPR, masked_cnt);
|
|
5518
5518
|
#undef __accumulate_f64_EXPR
|
|
5519
5519
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
5520
5520
|
if ( ! op_mask ) {
|
|
@@ -5522,9 +5522,9 @@ ca_size_t out_i = 0;
|
|
|
5522
5522
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
5523
5523
|
}
|
|
5524
5524
|
op_mask[out_i] = 1;
|
|
5525
|
-
op[out_i++] = (
|
|
5525
|
+
op[out_i++] = (ca_u_float64_t) 0; /* sentinel; mask bit is what counts */
|
|
5526
5526
|
} else {
|
|
5527
|
-
op[out_i++] = (
|
|
5527
|
+
op[out_i++] = (ca_u_float64_t) (acc);
|
|
5528
5528
|
}
|
|
5529
5529
|
}
|
|
5530
5530
|
ca_iter_state_finish(&st);
|
|
@@ -5559,18 +5559,18 @@ accumulate_ki_stream_cmplx64 (VALUE arg)
|
|
|
5559
5559
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
5560
5560
|
ca_size_t __counts[CA_RANK_MAX];
|
|
5561
5561
|
ca_size_t __strides[CA_RANK_MAX];
|
|
5562
|
-
ca_size_t __s = sizeof(
|
|
5562
|
+
ca_size_t __s = sizeof(ca_u_cmplx64_t);
|
|
5563
5563
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
5564
5564
|
__strides[__k] = __s;
|
|
5565
5565
|
__s *= ca->dim[__k];
|
|
5566
5566
|
}
|
|
5567
5567
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
5568
5568
|
ca_size_t __outer_off = 0;
|
|
5569
|
-
|
|
5569
|
+
ca_u_cmplx64_t acc;
|
|
5570
5570
|
acc = (0);
|
|
5571
5571
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
5572
5572
|
(void) masked_cnt;
|
|
5573
|
-
|
|
5573
|
+
ca_u_cmplx64_t *__chunk = (ca_u_cmplx64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_cmplx64_t));
|
|
5574
5574
|
__ctx->chunk = __chunk;
|
|
5575
5575
|
while ( __outer_off < __outer ) {
|
|
5576
5576
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -5580,12 +5580,12 @@ accumulate_ki_stream_cmplx64 (VALUE arg)
|
|
|
5580
5580
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
5581
5581
|
ca_size_t __i;
|
|
5582
5582
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
5583
|
-
|
|
5583
|
+
ca_u_cmplx64_t v = __chunk[__i];
|
|
5584
5584
|
acc += v;
|
|
5585
5585
|
}
|
|
5586
5586
|
__outer_off += __r;
|
|
5587
5587
|
}
|
|
5588
|
-
return CC2NUM((
|
|
5588
|
+
return CC2NUM((ca_u_cmplx64_t) (acc));
|
|
5589
5589
|
}
|
|
5590
5590
|
|
|
5591
5591
|
static VALUE
|
|
@@ -5629,36 +5629,36 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5629
5629
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
5630
5630
|
CArray *co;
|
|
5631
5631
|
GetCArray(vout, co);
|
|
5632
|
-
|
|
5632
|
+
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
5633
5633
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
5634
|
-
const
|
|
5634
|
+
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) ca->ptr;
|
|
5635
5635
|
|
|
5636
5636
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
5637
5637
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
5638
|
-
|
|
5638
|
+
ca_u_cmplx64_t __li_buf_acc[512];
|
|
5639
5639
|
|
|
5640
5640
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5641
5641
|
const ca_size_t __li_TILE = 512;
|
|
5642
5642
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
5643
|
-
const
|
|
5643
|
+
const ca_u_cmplx64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
5644
5644
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
5645
5645
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
5646
5646
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
5647
5647
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5648
|
-
__li_buf_acc[__j] = (
|
|
5648
|
+
__li_buf_acc[__j] = (ca_u_cmplx64_t) (0);
|
|
5649
5649
|
}
|
|
5650
5650
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5651
|
-
const
|
|
5651
|
+
const ca_u_cmplx64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5652
5652
|
_Pragma("omp simd")
|
|
5653
5653
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5654
|
-
|
|
5654
|
+
ca_u_cmplx64_t v = __li_row[__j];
|
|
5655
5655
|
__li_buf_acc[__j] += v;
|
|
5656
5656
|
(void) v;
|
|
5657
5657
|
}
|
|
5658
5658
|
}
|
|
5659
5659
|
|
|
5660
5660
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5661
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
5661
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_cmplx64_t) (__li_buf_acc[__j]);
|
|
5662
5662
|
}
|
|
5663
5663
|
}
|
|
5664
5664
|
}
|
|
@@ -5709,37 +5709,37 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5709
5709
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
5710
5710
|
CArray *co;
|
|
5711
5711
|
GetCArray(vout, co);
|
|
5712
|
-
|
|
5712
|
+
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
5713
5713
|
|
|
5714
|
-
|
|
5714
|
+
ca_u_cmplx64_t __li_buf_acc[512];
|
|
5715
5715
|
|
|
5716
5716
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5717
5717
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
5718
5718
|
const ca_size_t __li_TILE = 512;
|
|
5719
5719
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5720
5720
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
5721
|
-
const
|
|
5722
|
-
|
|
5721
|
+
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
|
|
5722
|
+
ca_u_cmplx64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
5723
5723
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
5724
|
-
const
|
|
5724
|
+
const ca_u_cmplx64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
5725
5725
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
5726
5726
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
5727
5727
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
5728
5728
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5729
|
-
__li_buf_acc[__j] = (
|
|
5729
|
+
__li_buf_acc[__j] = (ca_u_cmplx64_t) (0);
|
|
5730
5730
|
}
|
|
5731
5731
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5732
|
-
const
|
|
5732
|
+
const ca_u_cmplx64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5733
5733
|
_Pragma("omp simd")
|
|
5734
5734
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5735
|
-
|
|
5735
|
+
ca_u_cmplx64_t v = __li_row[__j];
|
|
5736
5736
|
__li_buf_acc[__j] += v;
|
|
5737
5737
|
(void) v;
|
|
5738
5738
|
}
|
|
5739
5739
|
}
|
|
5740
5740
|
|
|
5741
5741
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5742
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
5742
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_cmplx64_t) (__li_buf_acc[__j]);
|
|
5743
5743
|
}
|
|
5744
5744
|
}
|
|
5745
5745
|
}
|
|
@@ -5800,7 +5800,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5800
5800
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
5801
5801
|
CArray *co;
|
|
5802
5802
|
GetCArray(vout, co);
|
|
5803
|
-
|
|
5803
|
+
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
5804
5804
|
|
|
5805
5805
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
5806
5806
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -5829,13 +5829,13 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5829
5829
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
5830
5830
|
}
|
|
5831
5831
|
|
|
5832
|
-
|
|
5832
|
+
ca_u_cmplx64_t __li_buf_acc[512];
|
|
5833
5833
|
|
|
5834
5834
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5835
5835
|
const ca_size_t __li_TILE = 512;
|
|
5836
5836
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5837
5837
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
5838
|
-
const
|
|
5838
|
+
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
|
|
5839
5839
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
5840
5840
|
|
|
5841
5841
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -5846,27 +5846,27 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5846
5846
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
5847
5847
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
5848
5848
|
}
|
|
5849
|
-
|
|
5850
|
-
const
|
|
5849
|
+
ca_u_cmplx64_t *__li_op_k = op + __li_obase;
|
|
5850
|
+
const ca_u_cmplx64_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
5851
5851
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
5852
5852
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
5853
5853
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
5854
5854
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
5855
5855
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5856
|
-
__li_buf_acc[__j] = (
|
|
5856
|
+
__li_buf_acc[__j] = (ca_u_cmplx64_t) (0);
|
|
5857
5857
|
}
|
|
5858
5858
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5859
|
-
const
|
|
5859
|
+
const ca_u_cmplx64_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5860
5860
|
_Pragma("omp simd")
|
|
5861
5861
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5862
|
-
|
|
5862
|
+
ca_u_cmplx64_t v = __li_row[__j];
|
|
5863
5863
|
__li_buf_acc[__j] += v;
|
|
5864
5864
|
(void) v;
|
|
5865
5865
|
}
|
|
5866
5866
|
}
|
|
5867
5867
|
|
|
5868
5868
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5869
|
-
__li_op_k[__li_tile + __j] = (
|
|
5869
|
+
__li_op_k[__li_tile + __j] = (ca_u_cmplx64_t) (__li_buf_acc[__j]);
|
|
5870
5870
|
}
|
|
5871
5871
|
}
|
|
5872
5872
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -5947,7 +5947,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5947
5947
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
5948
5948
|
CArray *co;
|
|
5949
5949
|
GetCArray(vout, co);
|
|
5950
|
-
|
|
5950
|
+
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
5951
5951
|
|
|
5952
5952
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
5953
5953
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -5990,12 +5990,12 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5990
5990
|
ca_size_t __li_OUTER = 1;
|
|
5991
5991
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
5992
5992
|
|
|
5993
|
-
|
|
5993
|
+
ca_u_cmplx64_t __li_buf_acc[512];
|
|
5994
5994
|
|
|
5995
5995
|
const ca_size_t __li_TILE = 512;
|
|
5996
5996
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5997
5997
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
5998
|
-
const
|
|
5998
|
+
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
|
|
5999
5999
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6000
6000
|
|
|
6001
6001
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -6008,27 +6008,27 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6008
6008
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
6009
6009
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
6010
6010
|
}
|
|
6011
|
-
|
|
6012
|
-
const
|
|
6011
|
+
ca_u_cmplx64_t *__li_op_k = op + __li_obase;
|
|
6012
|
+
const ca_u_cmplx64_t *__li_plane = p_root + __li_pbase;
|
|
6013
6013
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
6014
6014
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
6015
6015
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
6016
6016
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
6017
6017
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6018
|
-
__li_buf_acc[__j] = (
|
|
6018
|
+
__li_buf_acc[__j] = (ca_u_cmplx64_t) (0);
|
|
6019
6019
|
}
|
|
6020
6020
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
6021
|
-
const
|
|
6021
|
+
const ca_u_cmplx64_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
6022
6022
|
_Pragma("omp simd")
|
|
6023
6023
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6024
|
-
|
|
6024
|
+
ca_u_cmplx64_t v = __li_row[__j];
|
|
6025
6025
|
__li_buf_acc[__j] += v;
|
|
6026
6026
|
(void) v;
|
|
6027
6027
|
}
|
|
6028
6028
|
}
|
|
6029
6029
|
|
|
6030
6030
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6031
|
-
__li_op_k[__li_tile + __j] = (
|
|
6031
|
+
__li_op_k[__li_tile + __j] = (ca_u_cmplx64_t) (__li_buf_acc[__j]);
|
|
6032
6032
|
}
|
|
6033
6033
|
}
|
|
6034
6034
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -6047,7 +6047,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6047
6047
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
6048
6048
|
CArray *co;
|
|
6049
6049
|
GetCArray(vout, co);
|
|
6050
|
-
|
|
6050
|
+
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
6051
6051
|
ca_iter_state st;
|
|
6052
6052
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
6053
6053
|
slab_axes, naxes, 0);
|
|
@@ -6060,10 +6060,10 @@ boolean8_t *m;
|
|
|
6060
6060
|
ca_size_t out_i = 0;
|
|
6061
6061
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
6062
6062
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
6063
|
-
|
|
6063
|
+
ca_u_cmplx64_t acc;
|
|
6064
6064
|
ca_size_t masked_cnt = 0;
|
|
6065
|
-
#define __accumulate_cmplx64_EXPR(__x) ((
|
|
6066
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
6065
|
+
#define __accumulate_cmplx64_EXPR(__x) ((ca_u_cmplx64_t)(__x))
|
|
6066
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_cmplx64_t, ca_u_cmplx64_t, st, p, m, acc, 0, __accumulate_cmplx64_EXPR, masked_cnt);
|
|
6067
6067
|
#undef __accumulate_cmplx64_EXPR
|
|
6068
6068
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
6069
6069
|
if ( ! op_mask ) {
|
|
@@ -6071,9 +6071,9 @@ ca_size_t out_i = 0;
|
|
|
6071
6071
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
6072
6072
|
}
|
|
6073
6073
|
op_mask[out_i] = 1;
|
|
6074
|
-
op[out_i++] = (
|
|
6074
|
+
op[out_i++] = (ca_u_cmplx64_t) 0; /* sentinel; mask bit is what counts */
|
|
6075
6075
|
} else {
|
|
6076
|
-
op[out_i++] = (
|
|
6076
|
+
op[out_i++] = (ca_u_cmplx64_t) (acc);
|
|
6077
6077
|
}
|
|
6078
6078
|
}
|
|
6079
6079
|
ca_iter_state_finish(&st);
|
|
@@ -6108,18 +6108,18 @@ accumulate_ki_stream_cmplx128 (VALUE arg)
|
|
|
6108
6108
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
6109
6109
|
ca_size_t __counts[CA_RANK_MAX];
|
|
6110
6110
|
ca_size_t __strides[CA_RANK_MAX];
|
|
6111
|
-
ca_size_t __s = sizeof(
|
|
6111
|
+
ca_size_t __s = sizeof(ca_u_cmplx128_t);
|
|
6112
6112
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
6113
6113
|
__strides[__k] = __s;
|
|
6114
6114
|
__s *= ca->dim[__k];
|
|
6115
6115
|
}
|
|
6116
6116
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
6117
6117
|
ca_size_t __outer_off = 0;
|
|
6118
|
-
|
|
6118
|
+
ca_u_cmplx128_t acc;
|
|
6119
6119
|
acc = (0);
|
|
6120
6120
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
6121
6121
|
(void) masked_cnt;
|
|
6122
|
-
|
|
6122
|
+
ca_u_cmplx128_t *__chunk = (ca_u_cmplx128_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_cmplx128_t));
|
|
6123
6123
|
__ctx->chunk = __chunk;
|
|
6124
6124
|
while ( __outer_off < __outer ) {
|
|
6125
6125
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -6129,12 +6129,12 @@ accumulate_ki_stream_cmplx128 (VALUE arg)
|
|
|
6129
6129
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
6130
6130
|
ca_size_t __i;
|
|
6131
6131
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
6132
|
-
|
|
6132
|
+
ca_u_cmplx128_t v = __chunk[__i];
|
|
6133
6133
|
acc += v;
|
|
6134
6134
|
}
|
|
6135
6135
|
__outer_off += __r;
|
|
6136
6136
|
}
|
|
6137
|
-
return CC2NUM((
|
|
6137
|
+
return CC2NUM((ca_u_cmplx128_t) (acc));
|
|
6138
6138
|
}
|
|
6139
6139
|
|
|
6140
6140
|
static VALUE
|
|
@@ -6178,36 +6178,36 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6178
6178
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
6179
6179
|
CArray *co;
|
|
6180
6180
|
GetCArray(vout, co);
|
|
6181
|
-
|
|
6181
|
+
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
6182
6182
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
6183
|
-
const
|
|
6183
|
+
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) ca->ptr;
|
|
6184
6184
|
|
|
6185
6185
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
6186
6186
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
6187
|
-
|
|
6187
|
+
ca_u_cmplx128_t __li_buf_acc[512];
|
|
6188
6188
|
|
|
6189
6189
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
6190
6190
|
const ca_size_t __li_TILE = 512;
|
|
6191
6191
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
6192
|
-
const
|
|
6192
|
+
const ca_u_cmplx128_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
6193
6193
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
6194
6194
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
6195
6195
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
6196
6196
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6197
|
-
__li_buf_acc[__j] = (
|
|
6197
|
+
__li_buf_acc[__j] = (ca_u_cmplx128_t) (0);
|
|
6198
6198
|
}
|
|
6199
6199
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
6200
|
-
const
|
|
6200
|
+
const ca_u_cmplx128_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
6201
6201
|
_Pragma("omp simd")
|
|
6202
6202
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6203
|
-
|
|
6203
|
+
ca_u_cmplx128_t v = __li_row[__j];
|
|
6204
6204
|
__li_buf_acc[__j] += v;
|
|
6205
6205
|
(void) v;
|
|
6206
6206
|
}
|
|
6207
6207
|
}
|
|
6208
6208
|
|
|
6209
6209
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6210
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
6210
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_cmplx128_t) (__li_buf_acc[__j]);
|
|
6211
6211
|
}
|
|
6212
6212
|
}
|
|
6213
6213
|
}
|
|
@@ -6258,37 +6258,37 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6258
6258
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
6259
6259
|
CArray *co;
|
|
6260
6260
|
GetCArray(vout, co);
|
|
6261
|
-
|
|
6261
|
+
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
6262
6262
|
|
|
6263
|
-
|
|
6263
|
+
ca_u_cmplx128_t __li_buf_acc[512];
|
|
6264
6264
|
|
|
6265
6265
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
6266
6266
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
6267
6267
|
const ca_size_t __li_TILE = 512;
|
|
6268
6268
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6269
6269
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
6270
|
-
const
|
|
6271
|
-
|
|
6270
|
+
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
|
|
6271
|
+
ca_u_cmplx128_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
6272
6272
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
6273
|
-
const
|
|
6273
|
+
const ca_u_cmplx128_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
6274
6274
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
6275
6275
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
6276
6276
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
6277
6277
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6278
|
-
__li_buf_acc[__j] = (
|
|
6278
|
+
__li_buf_acc[__j] = (ca_u_cmplx128_t) (0);
|
|
6279
6279
|
}
|
|
6280
6280
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
6281
|
-
const
|
|
6281
|
+
const ca_u_cmplx128_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
6282
6282
|
_Pragma("omp simd")
|
|
6283
6283
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6284
|
-
|
|
6284
|
+
ca_u_cmplx128_t v = __li_row[__j];
|
|
6285
6285
|
__li_buf_acc[__j] += v;
|
|
6286
6286
|
(void) v;
|
|
6287
6287
|
}
|
|
6288
6288
|
}
|
|
6289
6289
|
|
|
6290
6290
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6291
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
6291
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_cmplx128_t) (__li_buf_acc[__j]);
|
|
6292
6292
|
}
|
|
6293
6293
|
}
|
|
6294
6294
|
}
|
|
@@ -6349,7 +6349,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6349
6349
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
6350
6350
|
CArray *co;
|
|
6351
6351
|
GetCArray(vout, co);
|
|
6352
|
-
|
|
6352
|
+
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
6353
6353
|
|
|
6354
6354
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
6355
6355
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -6378,13 +6378,13 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6378
6378
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
6379
6379
|
}
|
|
6380
6380
|
|
|
6381
|
-
|
|
6381
|
+
ca_u_cmplx128_t __li_buf_acc[512];
|
|
6382
6382
|
|
|
6383
6383
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
6384
6384
|
const ca_size_t __li_TILE = 512;
|
|
6385
6385
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6386
6386
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
6387
|
-
const
|
|
6387
|
+
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
|
|
6388
6388
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6389
6389
|
|
|
6390
6390
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -6395,27 +6395,27 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6395
6395
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
6396
6396
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
6397
6397
|
}
|
|
6398
|
-
|
|
6399
|
-
const
|
|
6398
|
+
ca_u_cmplx128_t *__li_op_k = op + __li_obase;
|
|
6399
|
+
const ca_u_cmplx128_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
6400
6400
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
6401
6401
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
6402
6402
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
6403
6403
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
6404
6404
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6405
|
-
__li_buf_acc[__j] = (
|
|
6405
|
+
__li_buf_acc[__j] = (ca_u_cmplx128_t) (0);
|
|
6406
6406
|
}
|
|
6407
6407
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
6408
|
-
const
|
|
6408
|
+
const ca_u_cmplx128_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
6409
6409
|
_Pragma("omp simd")
|
|
6410
6410
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6411
|
-
|
|
6411
|
+
ca_u_cmplx128_t v = __li_row[__j];
|
|
6412
6412
|
__li_buf_acc[__j] += v;
|
|
6413
6413
|
(void) v;
|
|
6414
6414
|
}
|
|
6415
6415
|
}
|
|
6416
6416
|
|
|
6417
6417
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6418
|
-
__li_op_k[__li_tile + __j] = (
|
|
6418
|
+
__li_op_k[__li_tile + __j] = (ca_u_cmplx128_t) (__li_buf_acc[__j]);
|
|
6419
6419
|
}
|
|
6420
6420
|
}
|
|
6421
6421
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -6496,7 +6496,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6496
6496
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
6497
6497
|
CArray *co;
|
|
6498
6498
|
GetCArray(vout, co);
|
|
6499
|
-
|
|
6499
|
+
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
6500
6500
|
|
|
6501
6501
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
6502
6502
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -6539,12 +6539,12 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6539
6539
|
ca_size_t __li_OUTER = 1;
|
|
6540
6540
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
6541
6541
|
|
|
6542
|
-
|
|
6542
|
+
ca_u_cmplx128_t __li_buf_acc[512];
|
|
6543
6543
|
|
|
6544
6544
|
const ca_size_t __li_TILE = 512;
|
|
6545
6545
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6546
6546
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
6547
|
-
const
|
|
6547
|
+
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
|
|
6548
6548
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6549
6549
|
|
|
6550
6550
|
ca_size_t __li_idx[CA_RANK_MAX];
|
|
@@ -6557,27 +6557,27 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6557
6557
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
6558
6558
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
6559
6559
|
}
|
|
6560
|
-
|
|
6561
|
-
const
|
|
6560
|
+
ca_u_cmplx128_t *__li_op_k = op + __li_obase;
|
|
6561
|
+
const ca_u_cmplx128_t *__li_plane = p_root + __li_pbase;
|
|
6562
6562
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
6563
6563
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
6564
6564
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
6565
6565
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
6566
6566
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6567
|
-
__li_buf_acc[__j] = (
|
|
6567
|
+
__li_buf_acc[__j] = (ca_u_cmplx128_t) (0);
|
|
6568
6568
|
}
|
|
6569
6569
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
6570
|
-
const
|
|
6570
|
+
const ca_u_cmplx128_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
6571
6571
|
_Pragma("omp simd")
|
|
6572
6572
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6573
|
-
|
|
6573
|
+
ca_u_cmplx128_t v = __li_row[__j];
|
|
6574
6574
|
__li_buf_acc[__j] += v;
|
|
6575
6575
|
(void) v;
|
|
6576
6576
|
}
|
|
6577
6577
|
}
|
|
6578
6578
|
|
|
6579
6579
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6580
|
-
__li_op_k[__li_tile + __j] = (
|
|
6580
|
+
__li_op_k[__li_tile + __j] = (ca_u_cmplx128_t) (__li_buf_acc[__j]);
|
|
6581
6581
|
}
|
|
6582
6582
|
}
|
|
6583
6583
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -6596,7 +6596,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6596
6596
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
6597
6597
|
CArray *co;
|
|
6598
6598
|
GetCArray(vout, co);
|
|
6599
|
-
|
|
6599
|
+
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
6600
6600
|
ca_iter_state st;
|
|
6601
6601
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
6602
6602
|
slab_axes, naxes, 0);
|
|
@@ -6609,10 +6609,10 @@ boolean8_t *m;
|
|
|
6609
6609
|
ca_size_t out_i = 0;
|
|
6610
6610
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
6611
6611
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
6612
|
-
|
|
6612
|
+
ca_u_cmplx128_t acc;
|
|
6613
6613
|
ca_size_t masked_cnt = 0;
|
|
6614
|
-
#define __accumulate_cmplx128_EXPR(__x) ((
|
|
6615
|
-
CA_SLAB_REDUCE_SUM8_EX(
|
|
6614
|
+
#define __accumulate_cmplx128_EXPR(__x) ((ca_u_cmplx128_t)(__x))
|
|
6615
|
+
CA_SLAB_REDUCE_SUM8_EX(ca_u_cmplx128_t, ca_u_cmplx128_t, st, p, m, acc, 0, __accumulate_cmplx128_EXPR, masked_cnt);
|
|
6616
6616
|
#undef __accumulate_cmplx128_EXPR
|
|
6617
6617
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
6618
6618
|
if ( ! op_mask ) {
|
|
@@ -6620,9 +6620,9 @@ ca_size_t out_i = 0;
|
|
|
6620
6620
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
6621
6621
|
}
|
|
6622
6622
|
op_mask[out_i] = 1;
|
|
6623
|
-
op[out_i++] = (
|
|
6623
|
+
op[out_i++] = (ca_u_cmplx128_t) 0; /* sentinel; mask bit is what counts */
|
|
6624
6624
|
} else {
|
|
6625
|
-
op[out_i++] = (
|
|
6625
|
+
op[out_i++] = (ca_u_cmplx128_t) (acc);
|
|
6626
6626
|
}
|
|
6627
6627
|
}
|
|
6628
6628
|
ca_iter_state_finish(&st);
|
|
@@ -7470,7 +7470,7 @@ count_true_ki_stream_bool (VALUE arg)
|
|
|
7470
7470
|
}
|
|
7471
7471
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
7472
7472
|
ca_size_t __outer_off = 0;
|
|
7473
|
-
|
|
7473
|
+
ca_u_int64_t acc;
|
|
7474
7474
|
acc = (0);
|
|
7475
7475
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
7476
7476
|
(void) masked_cnt;
|
|
@@ -7489,7 +7489,7 @@ count_true_ki_stream_bool (VALUE arg)
|
|
|
7489
7489
|
}
|
|
7490
7490
|
__outer_off += __r;
|
|
7491
7491
|
}
|
|
7492
|
-
return LL2NUM((
|
|
7492
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
7493
7493
|
}
|
|
7494
7494
|
|
|
7495
7495
|
static VALUE
|
|
@@ -7533,13 +7533,13 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7533
7533
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
7534
7534
|
CArray *co;
|
|
7535
7535
|
GetCArray(vout, co);
|
|
7536
|
-
|
|
7536
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
7537
7537
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
7538
7538
|
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
7539
7539
|
|
|
7540
7540
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
7541
7541
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
7542
|
-
|
|
7542
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
7543
7543
|
|
|
7544
7544
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
7545
7545
|
const ca_size_t __li_TILE = 512;
|
|
@@ -7549,7 +7549,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7549
7549
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
7550
7550
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
7551
7551
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7552
|
-
__li_buf_acc[__j] = (
|
|
7552
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
7553
7553
|
}
|
|
7554
7554
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
7555
7555
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
@@ -7562,7 +7562,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7562
7562
|
}
|
|
7563
7563
|
|
|
7564
7564
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7565
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
7565
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
7566
7566
|
}
|
|
7567
7567
|
}
|
|
7568
7568
|
}
|
|
@@ -7613,9 +7613,9 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7613
7613
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
7614
7614
|
CArray *co;
|
|
7615
7615
|
GetCArray(vout, co);
|
|
7616
|
-
|
|
7616
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
7617
7617
|
|
|
7618
|
-
|
|
7618
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
7619
7619
|
|
|
7620
7620
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
7621
7621
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
@@ -7623,14 +7623,14 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7623
7623
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
7624
7624
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
7625
7625
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
7626
|
-
|
|
7626
|
+
ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
7627
7627
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
7628
7628
|
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
7629
7629
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
7630
7630
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
7631
7631
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
7632
7632
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7633
|
-
__li_buf_acc[__j] = (
|
|
7633
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
7634
7634
|
}
|
|
7635
7635
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
7636
7636
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
@@ -7643,7 +7643,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7643
7643
|
}
|
|
7644
7644
|
|
|
7645
7645
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7646
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
7646
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
7647
7647
|
}
|
|
7648
7648
|
}
|
|
7649
7649
|
}
|
|
@@ -7704,7 +7704,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7704
7704
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
7705
7705
|
CArray *co;
|
|
7706
7706
|
GetCArray(vout, co);
|
|
7707
|
-
|
|
7707
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
7708
7708
|
|
|
7709
7709
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
7710
7710
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -7733,7 +7733,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7733
7733
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
7734
7734
|
}
|
|
7735
7735
|
|
|
7736
|
-
|
|
7736
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
7737
7737
|
|
|
7738
7738
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
7739
7739
|
const ca_size_t __li_TILE = 512;
|
|
@@ -7750,14 +7750,14 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7750
7750
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
7751
7751
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
7752
7752
|
}
|
|
7753
|
-
|
|
7753
|
+
ca_u_int64_t *__li_op_k = op + __li_obase;
|
|
7754
7754
|
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
7755
7755
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
7756
7756
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
7757
7757
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
7758
7758
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
7759
7759
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7760
|
-
__li_buf_acc[__j] = (
|
|
7760
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
7761
7761
|
}
|
|
7762
7762
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
7763
7763
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
@@ -7770,7 +7770,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7770
7770
|
}
|
|
7771
7771
|
|
|
7772
7772
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7773
|
-
__li_op_k[__li_tile + __j] = (
|
|
7773
|
+
__li_op_k[__li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
7774
7774
|
}
|
|
7775
7775
|
}
|
|
7776
7776
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -7851,7 +7851,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7851
7851
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
7852
7852
|
CArray *co;
|
|
7853
7853
|
GetCArray(vout, co);
|
|
7854
|
-
|
|
7854
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
7855
7855
|
|
|
7856
7856
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
7857
7857
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -7894,7 +7894,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7894
7894
|
ca_size_t __li_OUTER = 1;
|
|
7895
7895
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
7896
7896
|
|
|
7897
|
-
|
|
7897
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
7898
7898
|
|
|
7899
7899
|
const ca_size_t __li_TILE = 512;
|
|
7900
7900
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
@@ -7912,14 +7912,14 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7912
7912
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
7913
7913
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
7914
7914
|
}
|
|
7915
|
-
|
|
7915
|
+
ca_u_int64_t *__li_op_k = op + __li_obase;
|
|
7916
7916
|
const boolean8_t *__li_plane = p_root + __li_pbase;
|
|
7917
7917
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
7918
7918
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
7919
7919
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
7920
7920
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
7921
7921
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7922
|
-
__li_buf_acc[__j] = (
|
|
7922
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
7923
7923
|
}
|
|
7924
7924
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
7925
7925
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
@@ -7932,7 +7932,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7932
7932
|
}
|
|
7933
7933
|
|
|
7934
7934
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7935
|
-
__li_op_k[__li_tile + __j] = (
|
|
7935
|
+
__li_op_k[__li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
7936
7936
|
}
|
|
7937
7937
|
}
|
|
7938
7938
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -7951,7 +7951,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7951
7951
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
7952
7952
|
CArray *co;
|
|
7953
7953
|
GetCArray(vout, co);
|
|
7954
|
-
|
|
7954
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
7955
7955
|
ca_iter_state st;
|
|
7956
7956
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
7957
7957
|
slab_axes, naxes, 0);
|
|
@@ -7964,10 +7964,10 @@ boolean8_t *m;
|
|
|
7964
7964
|
ca_size_t out_i = 0;
|
|
7965
7965
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
7966
7966
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
7967
|
-
|
|
7967
|
+
ca_u_int64_t acc;
|
|
7968
7968
|
ca_size_t masked_cnt = 0;
|
|
7969
|
-
#define __count_true_bool_EXPR(__x) ((
|
|
7970
|
-
CA_SLAB_REDUCE_SUM8_EX(boolean8_t,
|
|
7969
|
+
#define __count_true_bool_EXPR(__x) ((ca_u_int64_t)(__x))
|
|
7970
|
+
CA_SLAB_REDUCE_SUM8_EX(boolean8_t, ca_u_int64_t, st, p, m, acc, 0, __count_true_bool_EXPR, masked_cnt);
|
|
7971
7971
|
#undef __count_true_bool_EXPR
|
|
7972
7972
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
7973
7973
|
if ( ! op_mask ) {
|
|
@@ -7975,9 +7975,9 @@ ca_size_t out_i = 0;
|
|
|
7975
7975
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
7976
7976
|
}
|
|
7977
7977
|
op_mask[out_i] = 1;
|
|
7978
|
-
op[out_i++] = (
|
|
7978
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
7979
7979
|
} else {
|
|
7980
|
-
op[out_i++] = (
|
|
7980
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
7981
7981
|
}
|
|
7982
7982
|
}
|
|
7983
7983
|
ca_iter_state_finish(&st);
|
|
@@ -8066,7 +8066,7 @@ count_false_ki_stream_bool (VALUE arg)
|
|
|
8066
8066
|
}
|
|
8067
8067
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
8068
8068
|
ca_size_t __outer_off = 0;
|
|
8069
|
-
|
|
8069
|
+
ca_u_int64_t acc;
|
|
8070
8070
|
acc = (0);
|
|
8071
8071
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
8072
8072
|
(void) masked_cnt;
|
|
@@ -8085,7 +8085,7 @@ count_false_ki_stream_bool (VALUE arg)
|
|
|
8085
8085
|
}
|
|
8086
8086
|
__outer_off += __r;
|
|
8087
8087
|
}
|
|
8088
|
-
return LL2NUM((
|
|
8088
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
8089
8089
|
}
|
|
8090
8090
|
|
|
8091
8091
|
static VALUE
|
|
@@ -8129,13 +8129,13 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8129
8129
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8130
8130
|
CArray *co;
|
|
8131
8131
|
GetCArray(vout, co);
|
|
8132
|
-
|
|
8132
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8133
8133
|
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
8134
8134
|
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
8135
8135
|
|
|
8136
8136
|
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
8137
8137
|
to L7 TILE = 512 cells regardless of INNER). */
|
|
8138
|
-
|
|
8138
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
8139
8139
|
|
|
8140
8140
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
8141
8141
|
const ca_size_t __li_TILE = 512;
|
|
@@ -8145,7 +8145,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8145
8145
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
8146
8146
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
8147
8147
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8148
|
-
__li_buf_acc[__j] = (
|
|
8148
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
8149
8149
|
}
|
|
8150
8150
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
8151
8151
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
@@ -8158,7 +8158,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8158
8158
|
}
|
|
8159
8159
|
|
|
8160
8160
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8161
|
-
op[__li_o * __li_INNER + __li_tile + __j] = (
|
|
8161
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
8162
8162
|
}
|
|
8163
8163
|
}
|
|
8164
8164
|
}
|
|
@@ -8209,9 +8209,9 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8209
8209
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8210
8210
|
CArray *co;
|
|
8211
8211
|
GetCArray(vout, co);
|
|
8212
|
-
|
|
8212
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8213
8213
|
|
|
8214
|
-
|
|
8214
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
8215
8215
|
|
|
8216
8216
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
8217
8217
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
@@ -8219,14 +8219,14 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8219
8219
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
8220
8220
|
ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
|
|
8221
8221
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
8222
|
-
|
|
8222
|
+
ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
8223
8223
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
8224
8224
|
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
8225
8225
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
8226
8226
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
8227
8227
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
8228
8228
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8229
|
-
__li_buf_acc[__j] = (
|
|
8229
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
8230
8230
|
}
|
|
8231
8231
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
8232
8232
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
@@ -8239,7 +8239,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8239
8239
|
}
|
|
8240
8240
|
|
|
8241
8241
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8242
|
-
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (
|
|
8242
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
8243
8243
|
}
|
|
8244
8244
|
}
|
|
8245
8245
|
}
|
|
@@ -8300,7 +8300,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8300
8300
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8301
8301
|
CArray *co;
|
|
8302
8302
|
GetCArray(vout, co);
|
|
8303
|
-
|
|
8303
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8304
8304
|
|
|
8305
8305
|
/* Output row-major stride table (length = ca->ndim - 1).
|
|
8306
8306
|
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
@@ -8329,7 +8329,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8329
8329
|
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
8330
8330
|
}
|
|
8331
8331
|
|
|
8332
|
-
|
|
8332
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
8333
8333
|
|
|
8334
8334
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
8335
8335
|
const ca_size_t __li_TILE = 512;
|
|
@@ -8346,14 +8346,14 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8346
8346
|
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
8347
8347
|
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
8348
8348
|
}
|
|
8349
|
-
|
|
8349
|
+
ca_u_int64_t *__li_op_k = op + __li_obase;
|
|
8350
8350
|
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
8351
8351
|
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
8352
8352
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
8353
8353
|
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
8354
8354
|
? (__li_INNER - __li_tile) : __li_TILE;
|
|
8355
8355
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8356
|
-
__li_buf_acc[__j] = (
|
|
8356
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
8357
8357
|
}
|
|
8358
8358
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
8359
8359
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
@@ -8366,7 +8366,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8366
8366
|
}
|
|
8367
8367
|
|
|
8368
8368
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8369
|
-
__li_op_k[__li_tile + __j] = (
|
|
8369
|
+
__li_op_k[__li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
8370
8370
|
}
|
|
8371
8371
|
}
|
|
8372
8372
|
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
@@ -8447,7 +8447,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8447
8447
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8448
8448
|
CArray *co;
|
|
8449
8449
|
GetCArray(vout, co);
|
|
8450
|
-
|
|
8450
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8451
8451
|
|
|
8452
8452
|
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
8453
8453
|
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
@@ -8490,7 +8490,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8490
8490
|
ca_size_t __li_OUTER = 1;
|
|
8491
8491
|
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
8492
8492
|
|
|
8493
|
-
|
|
8493
|
+
ca_u_int64_t __li_buf_acc[512];
|
|
8494
8494
|
|
|
8495
8495
|
const ca_size_t __li_TILE = 512;
|
|
8496
8496
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
@@ -8508,14 +8508,14 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8508
8508
|
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
8509
8509
|
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
8510
8510
|
}
|
|
8511
|
-
|
|
8511
|
+
ca_u_int64_t *__li_op_k = op + __li_obase;
|
|
8512
8512
|
const boolean8_t *__li_plane = p_root + __li_pbase;
|
|
8513
8513
|
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
8514
8514
|
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
8515
8515
|
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
8516
8516
|
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
8517
8517
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8518
|
-
__li_buf_acc[__j] = (
|
|
8518
|
+
__li_buf_acc[__j] = (ca_u_int64_t) (0);
|
|
8519
8519
|
}
|
|
8520
8520
|
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
8521
8521
|
const boolean8_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
@@ -8528,7 +8528,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8528
8528
|
}
|
|
8529
8529
|
|
|
8530
8530
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8531
|
-
__li_op_k[__li_tile + __j] = (
|
|
8531
|
+
__li_op_k[__li_tile + __j] = (ca_u_int64_t) (__li_buf_acc[__j]);
|
|
8532
8532
|
}
|
|
8533
8533
|
}
|
|
8534
8534
|
/* Carry-increment effective OUTER multi-index. */
|
|
@@ -8547,7 +8547,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8547
8547
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8548
8548
|
CArray *co;
|
|
8549
8549
|
GetCArray(vout, co);
|
|
8550
|
-
|
|
8550
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8551
8551
|
ca_iter_state st;
|
|
8552
8552
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8553
8553
|
slab_axes, naxes, 0);
|
|
@@ -8560,7 +8560,7 @@ boolean8_t *m;
|
|
|
8560
8560
|
ca_size_t out_i = 0;
|
|
8561
8561
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
8562
8562
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
8563
|
-
|
|
8563
|
+
ca_u_int64_t acc;
|
|
8564
8564
|
ca_size_t masked_cnt = 0;
|
|
8565
8565
|
CA_SLAB_REDUCE_T_PLUS_EX(boolean8_t, st, p, m, acc, 0, acc += !v, masked_cnt);
|
|
8566
8566
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
@@ -8569,9 +8569,9 @@ ca_size_t out_i = 0;
|
|
|
8569
8569
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
8570
8570
|
}
|
|
8571
8571
|
op_mask[out_i] = 1;
|
|
8572
|
-
op[out_i++] = (
|
|
8572
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
8573
8573
|
} else {
|
|
8574
|
-
op[out_i++] = (
|
|
8574
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
8575
8575
|
}
|
|
8576
8576
|
}
|
|
8577
8577
|
ca_iter_state_finish(&st);
|
|
@@ -8663,7 +8663,7 @@ count_equal_ki_stream_i8 (VALUE arg)
|
|
|
8663
8663
|
}
|
|
8664
8664
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
8665
8665
|
ca_size_t __outer_off = 0;
|
|
8666
|
-
|
|
8666
|
+
ca_u_int64_t acc;
|
|
8667
8667
|
acc = (0);
|
|
8668
8668
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
8669
8669
|
(void) masked_cnt;
|
|
@@ -8682,7 +8682,7 @@ count_equal_ki_stream_i8 (VALUE arg)
|
|
|
8682
8682
|
}
|
|
8683
8683
|
__outer_off += __r;
|
|
8684
8684
|
}
|
|
8685
|
-
return LL2NUM((
|
|
8685
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
8686
8686
|
}
|
|
8687
8687
|
|
|
8688
8688
|
static VALUE
|
|
@@ -8704,7 +8704,7 @@ count_equal_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
8704
8704
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8705
8705
|
CArray *co;
|
|
8706
8706
|
GetCArray(vout, co);
|
|
8707
|
-
|
|
8707
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8708
8708
|
ca_iter_state st;
|
|
8709
8709
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8710
8710
|
slab_axes, naxes, 0);
|
|
@@ -8717,7 +8717,7 @@ boolean8_t *m;
|
|
|
8717
8717
|
ca_size_t out_i = 0;
|
|
8718
8718
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
8719
8719
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
8720
|
-
|
|
8720
|
+
ca_u_int64_t acc;
|
|
8721
8721
|
ca_size_t masked_cnt = 0;
|
|
8722
8722
|
CA_SLAB_REDUCE_T_PLUS_EX(int8_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
8723
8723
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
@@ -8726,9 +8726,9 @@ ca_size_t out_i = 0;
|
|
|
8726
8726
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
8727
8727
|
}
|
|
8728
8728
|
op_mask[out_i] = 1;
|
|
8729
|
-
op[out_i++] = (
|
|
8729
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
8730
8730
|
} else {
|
|
8731
|
-
op[out_i++] = (
|
|
8731
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
8732
8732
|
}
|
|
8733
8733
|
}
|
|
8734
8734
|
ca_iter_state_finish(&st);
|
|
@@ -8773,7 +8773,7 @@ count_equal_ki_stream_u8 (VALUE arg)
|
|
|
8773
8773
|
}
|
|
8774
8774
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
8775
8775
|
ca_size_t __outer_off = 0;
|
|
8776
|
-
|
|
8776
|
+
ca_u_int64_t acc;
|
|
8777
8777
|
acc = (0);
|
|
8778
8778
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
8779
8779
|
(void) masked_cnt;
|
|
@@ -8792,7 +8792,7 @@ count_equal_ki_stream_u8 (VALUE arg)
|
|
|
8792
8792
|
}
|
|
8793
8793
|
__outer_off += __r;
|
|
8794
8794
|
}
|
|
8795
|
-
return LL2NUM((
|
|
8795
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
8796
8796
|
}
|
|
8797
8797
|
|
|
8798
8798
|
static VALUE
|
|
@@ -8814,7 +8814,7 @@ count_equal_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
8814
8814
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8815
8815
|
CArray *co;
|
|
8816
8816
|
GetCArray(vout, co);
|
|
8817
|
-
|
|
8817
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8818
8818
|
ca_iter_state st;
|
|
8819
8819
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8820
8820
|
slab_axes, naxes, 0);
|
|
@@ -8827,7 +8827,7 @@ boolean8_t *m;
|
|
|
8827
8827
|
ca_size_t out_i = 0;
|
|
8828
8828
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
8829
8829
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
8830
|
-
|
|
8830
|
+
ca_u_int64_t acc;
|
|
8831
8831
|
ca_size_t masked_cnt = 0;
|
|
8832
8832
|
CA_SLAB_REDUCE_T_PLUS_EX(uint8_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
8833
8833
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
@@ -8836,9 +8836,9 @@ ca_size_t out_i = 0;
|
|
|
8836
8836
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
8837
8837
|
}
|
|
8838
8838
|
op_mask[out_i] = 1;
|
|
8839
|
-
op[out_i++] = (
|
|
8839
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
8840
8840
|
} else {
|
|
8841
|
-
op[out_i++] = (
|
|
8841
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
8842
8842
|
}
|
|
8843
8843
|
}
|
|
8844
8844
|
ca_iter_state_finish(&st);
|
|
@@ -8852,7 +8852,7 @@ ca_size_t out_i = 0;
|
|
|
8852
8852
|
typedef struct {
|
|
8853
8853
|
void *chunk;
|
|
8854
8854
|
CArray *ca;
|
|
8855
|
-
|
|
8855
|
+
ca_u_int16_t value_arg;
|
|
8856
8856
|
ca_size_t min_count;
|
|
8857
8857
|
} count_equal_ki_stream_i16_t;
|
|
8858
8858
|
|
|
@@ -8861,7 +8861,7 @@ count_equal_ki_stream_i16 (VALUE arg)
|
|
|
8861
8861
|
{
|
|
8862
8862
|
count_equal_ki_stream_i16_t *__ctx = (count_equal_ki_stream_i16_t *) arg;
|
|
8863
8863
|
CArray *ca = __ctx->ca;
|
|
8864
|
-
|
|
8864
|
+
ca_u_int16_t value_arg = __ctx->value_arg;
|
|
8865
8865
|
(void) value_arg;
|
|
8866
8866
|
ca_size_t min_count = __ctx->min_count;
|
|
8867
8867
|
(void) min_count;
|
|
@@ -8876,18 +8876,18 @@ count_equal_ki_stream_i16 (VALUE arg)
|
|
|
8876
8876
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
8877
8877
|
ca_size_t __counts[CA_RANK_MAX];
|
|
8878
8878
|
ca_size_t __strides[CA_RANK_MAX];
|
|
8879
|
-
ca_size_t __s = sizeof(
|
|
8879
|
+
ca_size_t __s = sizeof(ca_u_int16_t);
|
|
8880
8880
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
8881
8881
|
__strides[__k] = __s;
|
|
8882
8882
|
__s *= ca->dim[__k];
|
|
8883
8883
|
}
|
|
8884
8884
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
8885
8885
|
ca_size_t __outer_off = 0;
|
|
8886
|
-
|
|
8886
|
+
ca_u_int64_t acc;
|
|
8887
8887
|
acc = (0);
|
|
8888
8888
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
8889
8889
|
(void) masked_cnt;
|
|
8890
|
-
|
|
8890
|
+
ca_u_int16_t *__chunk = (ca_u_int16_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_int16_t));
|
|
8891
8891
|
__ctx->chunk = __chunk;
|
|
8892
8892
|
while ( __outer_off < __outer ) {
|
|
8893
8893
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -8897,16 +8897,16 @@ count_equal_ki_stream_i16 (VALUE arg)
|
|
|
8897
8897
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
8898
8898
|
ca_size_t __i;
|
|
8899
8899
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
8900
|
-
|
|
8900
|
+
ca_u_int16_t v = __chunk[__i];
|
|
8901
8901
|
if (v == value_arg) acc += 1;
|
|
8902
8902
|
}
|
|
8903
8903
|
__outer_off += __r;
|
|
8904
8904
|
}
|
|
8905
|
-
return LL2NUM((
|
|
8905
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
8906
8906
|
}
|
|
8907
8907
|
|
|
8908
8908
|
static VALUE
|
|
8909
|
-
count_equal_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
8909
|
+
count_equal_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int16_t value_arg, ca_size_t min_count)
|
|
8910
8910
|
{
|
|
8911
8911
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
8912
8912
|
ca->elements > 0 &&
|
|
@@ -8924,7 +8924,7 @@ count_equal_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
8924
8924
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8925
8925
|
CArray *co;
|
|
8926
8926
|
GetCArray(vout, co);
|
|
8927
|
-
|
|
8927
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8928
8928
|
ca_iter_state st;
|
|
8929
8929
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8930
8930
|
slab_axes, naxes, 0);
|
|
@@ -8937,18 +8937,18 @@ boolean8_t *m;
|
|
|
8937
8937
|
ca_size_t out_i = 0;
|
|
8938
8938
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
8939
8939
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
8940
|
-
|
|
8940
|
+
ca_u_int64_t acc;
|
|
8941
8941
|
ca_size_t masked_cnt = 0;
|
|
8942
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
8942
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_int16_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
8943
8943
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
8944
8944
|
if ( ! op_mask ) {
|
|
8945
8945
|
ca_create_mask(co);
|
|
8946
8946
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
8947
8947
|
}
|
|
8948
8948
|
op_mask[out_i] = 1;
|
|
8949
|
-
op[out_i++] = (
|
|
8949
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
8950
8950
|
} else {
|
|
8951
|
-
op[out_i++] = (
|
|
8951
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
8952
8952
|
}
|
|
8953
8953
|
}
|
|
8954
8954
|
ca_iter_state_finish(&st);
|
|
@@ -8962,7 +8962,7 @@ ca_size_t out_i = 0;
|
|
|
8962
8962
|
typedef struct {
|
|
8963
8963
|
void *chunk;
|
|
8964
8964
|
CArray *ca;
|
|
8965
|
-
|
|
8965
|
+
ca_u_uint16_t value_arg;
|
|
8966
8966
|
ca_size_t min_count;
|
|
8967
8967
|
} count_equal_ki_stream_u16_t;
|
|
8968
8968
|
|
|
@@ -8971,7 +8971,7 @@ count_equal_ki_stream_u16 (VALUE arg)
|
|
|
8971
8971
|
{
|
|
8972
8972
|
count_equal_ki_stream_u16_t *__ctx = (count_equal_ki_stream_u16_t *) arg;
|
|
8973
8973
|
CArray *ca = __ctx->ca;
|
|
8974
|
-
|
|
8974
|
+
ca_u_uint16_t value_arg = __ctx->value_arg;
|
|
8975
8975
|
(void) value_arg;
|
|
8976
8976
|
ca_size_t min_count = __ctx->min_count;
|
|
8977
8977
|
(void) min_count;
|
|
@@ -8986,18 +8986,18 @@ count_equal_ki_stream_u16 (VALUE arg)
|
|
|
8986
8986
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
8987
8987
|
ca_size_t __counts[CA_RANK_MAX];
|
|
8988
8988
|
ca_size_t __strides[CA_RANK_MAX];
|
|
8989
|
-
ca_size_t __s = sizeof(
|
|
8989
|
+
ca_size_t __s = sizeof(ca_u_uint16_t);
|
|
8990
8990
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
8991
8991
|
__strides[__k] = __s;
|
|
8992
8992
|
__s *= ca->dim[__k];
|
|
8993
8993
|
}
|
|
8994
8994
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
8995
8995
|
ca_size_t __outer_off = 0;
|
|
8996
|
-
|
|
8996
|
+
ca_u_int64_t acc;
|
|
8997
8997
|
acc = (0);
|
|
8998
8998
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
8999
8999
|
(void) masked_cnt;
|
|
9000
|
-
|
|
9000
|
+
ca_u_uint16_t *__chunk = (ca_u_uint16_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_uint16_t));
|
|
9001
9001
|
__ctx->chunk = __chunk;
|
|
9002
9002
|
while ( __outer_off < __outer ) {
|
|
9003
9003
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9007,16 +9007,16 @@ count_equal_ki_stream_u16 (VALUE arg)
|
|
|
9007
9007
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9008
9008
|
ca_size_t __i;
|
|
9009
9009
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9010
|
-
|
|
9010
|
+
ca_u_uint16_t v = __chunk[__i];
|
|
9011
9011
|
if (v == value_arg) acc += 1;
|
|
9012
9012
|
}
|
|
9013
9013
|
__outer_off += __r;
|
|
9014
9014
|
}
|
|
9015
|
-
return LL2NUM((
|
|
9015
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9016
9016
|
}
|
|
9017
9017
|
|
|
9018
9018
|
static VALUE
|
|
9019
|
-
count_equal_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9019
|
+
count_equal_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint16_t value_arg, ca_size_t min_count)
|
|
9020
9020
|
{
|
|
9021
9021
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9022
9022
|
ca->elements > 0 &&
|
|
@@ -9034,7 +9034,7 @@ count_equal_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9034
9034
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9035
9035
|
CArray *co;
|
|
9036
9036
|
GetCArray(vout, co);
|
|
9037
|
-
|
|
9037
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9038
9038
|
ca_iter_state st;
|
|
9039
9039
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9040
9040
|
slab_axes, naxes, 0);
|
|
@@ -9047,18 +9047,18 @@ boolean8_t *m;
|
|
|
9047
9047
|
ca_size_t out_i = 0;
|
|
9048
9048
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9049
9049
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9050
|
-
|
|
9050
|
+
ca_u_int64_t acc;
|
|
9051
9051
|
ca_size_t masked_cnt = 0;
|
|
9052
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9052
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_uint16_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9053
9053
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9054
9054
|
if ( ! op_mask ) {
|
|
9055
9055
|
ca_create_mask(co);
|
|
9056
9056
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9057
9057
|
}
|
|
9058
9058
|
op_mask[out_i] = 1;
|
|
9059
|
-
op[out_i++] = (
|
|
9059
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9060
9060
|
} else {
|
|
9061
|
-
op[out_i++] = (
|
|
9061
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9062
9062
|
}
|
|
9063
9063
|
}
|
|
9064
9064
|
ca_iter_state_finish(&st);
|
|
@@ -9072,7 +9072,7 @@ ca_size_t out_i = 0;
|
|
|
9072
9072
|
typedef struct {
|
|
9073
9073
|
void *chunk;
|
|
9074
9074
|
CArray *ca;
|
|
9075
|
-
|
|
9075
|
+
ca_u_int32_t value_arg;
|
|
9076
9076
|
ca_size_t min_count;
|
|
9077
9077
|
} count_equal_ki_stream_i32_t;
|
|
9078
9078
|
|
|
@@ -9081,7 +9081,7 @@ count_equal_ki_stream_i32 (VALUE arg)
|
|
|
9081
9081
|
{
|
|
9082
9082
|
count_equal_ki_stream_i32_t *__ctx = (count_equal_ki_stream_i32_t *) arg;
|
|
9083
9083
|
CArray *ca = __ctx->ca;
|
|
9084
|
-
|
|
9084
|
+
ca_u_int32_t value_arg = __ctx->value_arg;
|
|
9085
9085
|
(void) value_arg;
|
|
9086
9086
|
ca_size_t min_count = __ctx->min_count;
|
|
9087
9087
|
(void) min_count;
|
|
@@ -9096,18 +9096,18 @@ count_equal_ki_stream_i32 (VALUE arg)
|
|
|
9096
9096
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
9097
9097
|
ca_size_t __counts[CA_RANK_MAX];
|
|
9098
9098
|
ca_size_t __strides[CA_RANK_MAX];
|
|
9099
|
-
ca_size_t __s = sizeof(
|
|
9099
|
+
ca_size_t __s = sizeof(ca_u_int32_t);
|
|
9100
9100
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
9101
9101
|
__strides[__k] = __s;
|
|
9102
9102
|
__s *= ca->dim[__k];
|
|
9103
9103
|
}
|
|
9104
9104
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9105
9105
|
ca_size_t __outer_off = 0;
|
|
9106
|
-
|
|
9106
|
+
ca_u_int64_t acc;
|
|
9107
9107
|
acc = (0);
|
|
9108
9108
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9109
9109
|
(void) masked_cnt;
|
|
9110
|
-
|
|
9110
|
+
ca_u_int32_t *__chunk = (ca_u_int32_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_int32_t));
|
|
9111
9111
|
__ctx->chunk = __chunk;
|
|
9112
9112
|
while ( __outer_off < __outer ) {
|
|
9113
9113
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9117,16 +9117,16 @@ count_equal_ki_stream_i32 (VALUE arg)
|
|
|
9117
9117
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9118
9118
|
ca_size_t __i;
|
|
9119
9119
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9120
|
-
|
|
9120
|
+
ca_u_int32_t v = __chunk[__i];
|
|
9121
9121
|
if (v == value_arg) acc += 1;
|
|
9122
9122
|
}
|
|
9123
9123
|
__outer_off += __r;
|
|
9124
9124
|
}
|
|
9125
|
-
return LL2NUM((
|
|
9125
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9126
9126
|
}
|
|
9127
9127
|
|
|
9128
9128
|
static VALUE
|
|
9129
|
-
count_equal_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9129
|
+
count_equal_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int32_t value_arg, ca_size_t min_count)
|
|
9130
9130
|
{
|
|
9131
9131
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9132
9132
|
ca->elements > 0 &&
|
|
@@ -9144,7 +9144,7 @@ count_equal_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9144
9144
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9145
9145
|
CArray *co;
|
|
9146
9146
|
GetCArray(vout, co);
|
|
9147
|
-
|
|
9147
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9148
9148
|
ca_iter_state st;
|
|
9149
9149
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9150
9150
|
slab_axes, naxes, 0);
|
|
@@ -9157,18 +9157,18 @@ boolean8_t *m;
|
|
|
9157
9157
|
ca_size_t out_i = 0;
|
|
9158
9158
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9159
9159
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9160
|
-
|
|
9160
|
+
ca_u_int64_t acc;
|
|
9161
9161
|
ca_size_t masked_cnt = 0;
|
|
9162
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9162
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_int32_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9163
9163
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9164
9164
|
if ( ! op_mask ) {
|
|
9165
9165
|
ca_create_mask(co);
|
|
9166
9166
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9167
9167
|
}
|
|
9168
9168
|
op_mask[out_i] = 1;
|
|
9169
|
-
op[out_i++] = (
|
|
9169
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9170
9170
|
} else {
|
|
9171
|
-
op[out_i++] = (
|
|
9171
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9172
9172
|
}
|
|
9173
9173
|
}
|
|
9174
9174
|
ca_iter_state_finish(&st);
|
|
@@ -9182,7 +9182,7 @@ ca_size_t out_i = 0;
|
|
|
9182
9182
|
typedef struct {
|
|
9183
9183
|
void *chunk;
|
|
9184
9184
|
CArray *ca;
|
|
9185
|
-
|
|
9185
|
+
ca_u_uint32_t value_arg;
|
|
9186
9186
|
ca_size_t min_count;
|
|
9187
9187
|
} count_equal_ki_stream_u32_t;
|
|
9188
9188
|
|
|
@@ -9191,7 +9191,7 @@ count_equal_ki_stream_u32 (VALUE arg)
|
|
|
9191
9191
|
{
|
|
9192
9192
|
count_equal_ki_stream_u32_t *__ctx = (count_equal_ki_stream_u32_t *) arg;
|
|
9193
9193
|
CArray *ca = __ctx->ca;
|
|
9194
|
-
|
|
9194
|
+
ca_u_uint32_t value_arg = __ctx->value_arg;
|
|
9195
9195
|
(void) value_arg;
|
|
9196
9196
|
ca_size_t min_count = __ctx->min_count;
|
|
9197
9197
|
(void) min_count;
|
|
@@ -9206,18 +9206,18 @@ count_equal_ki_stream_u32 (VALUE arg)
|
|
|
9206
9206
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
9207
9207
|
ca_size_t __counts[CA_RANK_MAX];
|
|
9208
9208
|
ca_size_t __strides[CA_RANK_MAX];
|
|
9209
|
-
ca_size_t __s = sizeof(
|
|
9209
|
+
ca_size_t __s = sizeof(ca_u_uint32_t);
|
|
9210
9210
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
9211
9211
|
__strides[__k] = __s;
|
|
9212
9212
|
__s *= ca->dim[__k];
|
|
9213
9213
|
}
|
|
9214
9214
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9215
9215
|
ca_size_t __outer_off = 0;
|
|
9216
|
-
|
|
9216
|
+
ca_u_int64_t acc;
|
|
9217
9217
|
acc = (0);
|
|
9218
9218
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9219
9219
|
(void) masked_cnt;
|
|
9220
|
-
|
|
9220
|
+
ca_u_uint32_t *__chunk = (ca_u_uint32_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_uint32_t));
|
|
9221
9221
|
__ctx->chunk = __chunk;
|
|
9222
9222
|
while ( __outer_off < __outer ) {
|
|
9223
9223
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9227,16 +9227,16 @@ count_equal_ki_stream_u32 (VALUE arg)
|
|
|
9227
9227
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9228
9228
|
ca_size_t __i;
|
|
9229
9229
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9230
|
-
|
|
9230
|
+
ca_u_uint32_t v = __chunk[__i];
|
|
9231
9231
|
if (v == value_arg) acc += 1;
|
|
9232
9232
|
}
|
|
9233
9233
|
__outer_off += __r;
|
|
9234
9234
|
}
|
|
9235
|
-
return LL2NUM((
|
|
9235
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9236
9236
|
}
|
|
9237
9237
|
|
|
9238
9238
|
static VALUE
|
|
9239
|
-
count_equal_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9239
|
+
count_equal_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint32_t value_arg, ca_size_t min_count)
|
|
9240
9240
|
{
|
|
9241
9241
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9242
9242
|
ca->elements > 0 &&
|
|
@@ -9254,7 +9254,7 @@ count_equal_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9254
9254
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9255
9255
|
CArray *co;
|
|
9256
9256
|
GetCArray(vout, co);
|
|
9257
|
-
|
|
9257
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9258
9258
|
ca_iter_state st;
|
|
9259
9259
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9260
9260
|
slab_axes, naxes, 0);
|
|
@@ -9267,18 +9267,18 @@ boolean8_t *m;
|
|
|
9267
9267
|
ca_size_t out_i = 0;
|
|
9268
9268
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9269
9269
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9270
|
-
|
|
9270
|
+
ca_u_int64_t acc;
|
|
9271
9271
|
ca_size_t masked_cnt = 0;
|
|
9272
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9272
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_uint32_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9273
9273
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9274
9274
|
if ( ! op_mask ) {
|
|
9275
9275
|
ca_create_mask(co);
|
|
9276
9276
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9277
9277
|
}
|
|
9278
9278
|
op_mask[out_i] = 1;
|
|
9279
|
-
op[out_i++] = (
|
|
9279
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9280
9280
|
} else {
|
|
9281
|
-
op[out_i++] = (
|
|
9281
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9282
9282
|
}
|
|
9283
9283
|
}
|
|
9284
9284
|
ca_iter_state_finish(&st);
|
|
@@ -9292,7 +9292,7 @@ ca_size_t out_i = 0;
|
|
|
9292
9292
|
typedef struct {
|
|
9293
9293
|
void *chunk;
|
|
9294
9294
|
CArray *ca;
|
|
9295
|
-
|
|
9295
|
+
ca_u_int64_t value_arg;
|
|
9296
9296
|
ca_size_t min_count;
|
|
9297
9297
|
} count_equal_ki_stream_i64_t;
|
|
9298
9298
|
|
|
@@ -9301,7 +9301,7 @@ count_equal_ki_stream_i64 (VALUE arg)
|
|
|
9301
9301
|
{
|
|
9302
9302
|
count_equal_ki_stream_i64_t *__ctx = (count_equal_ki_stream_i64_t *) arg;
|
|
9303
9303
|
CArray *ca = __ctx->ca;
|
|
9304
|
-
|
|
9304
|
+
ca_u_int64_t value_arg = __ctx->value_arg;
|
|
9305
9305
|
(void) value_arg;
|
|
9306
9306
|
ca_size_t min_count = __ctx->min_count;
|
|
9307
9307
|
(void) min_count;
|
|
@@ -9316,18 +9316,18 @@ count_equal_ki_stream_i64 (VALUE arg)
|
|
|
9316
9316
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
9317
9317
|
ca_size_t __counts[CA_RANK_MAX];
|
|
9318
9318
|
ca_size_t __strides[CA_RANK_MAX];
|
|
9319
|
-
ca_size_t __s = sizeof(
|
|
9319
|
+
ca_size_t __s = sizeof(ca_u_int64_t);
|
|
9320
9320
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
9321
9321
|
__strides[__k] = __s;
|
|
9322
9322
|
__s *= ca->dim[__k];
|
|
9323
9323
|
}
|
|
9324
9324
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9325
9325
|
ca_size_t __outer_off = 0;
|
|
9326
|
-
|
|
9326
|
+
ca_u_int64_t acc;
|
|
9327
9327
|
acc = (0);
|
|
9328
9328
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9329
9329
|
(void) masked_cnt;
|
|
9330
|
-
|
|
9330
|
+
ca_u_int64_t *__chunk = (ca_u_int64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_int64_t));
|
|
9331
9331
|
__ctx->chunk = __chunk;
|
|
9332
9332
|
while ( __outer_off < __outer ) {
|
|
9333
9333
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9337,16 +9337,16 @@ count_equal_ki_stream_i64 (VALUE arg)
|
|
|
9337
9337
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9338
9338
|
ca_size_t __i;
|
|
9339
9339
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9340
|
-
|
|
9340
|
+
ca_u_int64_t v = __chunk[__i];
|
|
9341
9341
|
if (v == value_arg) acc += 1;
|
|
9342
9342
|
}
|
|
9343
9343
|
__outer_off += __r;
|
|
9344
9344
|
}
|
|
9345
|
-
return LL2NUM((
|
|
9345
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9346
9346
|
}
|
|
9347
9347
|
|
|
9348
9348
|
static VALUE
|
|
9349
|
-
count_equal_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9349
|
+
count_equal_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int64_t value_arg, ca_size_t min_count)
|
|
9350
9350
|
{
|
|
9351
9351
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9352
9352
|
ca->elements > 0 &&
|
|
@@ -9364,7 +9364,7 @@ count_equal_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9364
9364
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9365
9365
|
CArray *co;
|
|
9366
9366
|
GetCArray(vout, co);
|
|
9367
|
-
|
|
9367
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9368
9368
|
ca_iter_state st;
|
|
9369
9369
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9370
9370
|
slab_axes, naxes, 0);
|
|
@@ -9377,18 +9377,18 @@ boolean8_t *m;
|
|
|
9377
9377
|
ca_size_t out_i = 0;
|
|
9378
9378
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9379
9379
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9380
|
-
|
|
9380
|
+
ca_u_int64_t acc;
|
|
9381
9381
|
ca_size_t masked_cnt = 0;
|
|
9382
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9382
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_int64_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9383
9383
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9384
9384
|
if ( ! op_mask ) {
|
|
9385
9385
|
ca_create_mask(co);
|
|
9386
9386
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9387
9387
|
}
|
|
9388
9388
|
op_mask[out_i] = 1;
|
|
9389
|
-
op[out_i++] = (
|
|
9389
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9390
9390
|
} else {
|
|
9391
|
-
op[out_i++] = (
|
|
9391
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9392
9392
|
}
|
|
9393
9393
|
}
|
|
9394
9394
|
ca_iter_state_finish(&st);
|
|
@@ -9402,7 +9402,7 @@ ca_size_t out_i = 0;
|
|
|
9402
9402
|
typedef struct {
|
|
9403
9403
|
void *chunk;
|
|
9404
9404
|
CArray *ca;
|
|
9405
|
-
|
|
9405
|
+
ca_u_uint64_t value_arg;
|
|
9406
9406
|
ca_size_t min_count;
|
|
9407
9407
|
} count_equal_ki_stream_u64_t;
|
|
9408
9408
|
|
|
@@ -9411,7 +9411,7 @@ count_equal_ki_stream_u64 (VALUE arg)
|
|
|
9411
9411
|
{
|
|
9412
9412
|
count_equal_ki_stream_u64_t *__ctx = (count_equal_ki_stream_u64_t *) arg;
|
|
9413
9413
|
CArray *ca = __ctx->ca;
|
|
9414
|
-
|
|
9414
|
+
ca_u_uint64_t value_arg = __ctx->value_arg;
|
|
9415
9415
|
(void) value_arg;
|
|
9416
9416
|
ca_size_t min_count = __ctx->min_count;
|
|
9417
9417
|
(void) min_count;
|
|
@@ -9426,18 +9426,18 @@ count_equal_ki_stream_u64 (VALUE arg)
|
|
|
9426
9426
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
9427
9427
|
ca_size_t __counts[CA_RANK_MAX];
|
|
9428
9428
|
ca_size_t __strides[CA_RANK_MAX];
|
|
9429
|
-
ca_size_t __s = sizeof(
|
|
9429
|
+
ca_size_t __s = sizeof(ca_u_uint64_t);
|
|
9430
9430
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
9431
9431
|
__strides[__k] = __s;
|
|
9432
9432
|
__s *= ca->dim[__k];
|
|
9433
9433
|
}
|
|
9434
9434
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9435
9435
|
ca_size_t __outer_off = 0;
|
|
9436
|
-
|
|
9436
|
+
ca_u_int64_t acc;
|
|
9437
9437
|
acc = (0);
|
|
9438
9438
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9439
9439
|
(void) masked_cnt;
|
|
9440
|
-
|
|
9440
|
+
ca_u_uint64_t *__chunk = (ca_u_uint64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_uint64_t));
|
|
9441
9441
|
__ctx->chunk = __chunk;
|
|
9442
9442
|
while ( __outer_off < __outer ) {
|
|
9443
9443
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9447,16 +9447,16 @@ count_equal_ki_stream_u64 (VALUE arg)
|
|
|
9447
9447
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9448
9448
|
ca_size_t __i;
|
|
9449
9449
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9450
|
-
|
|
9450
|
+
ca_u_uint64_t v = __chunk[__i];
|
|
9451
9451
|
if (v == value_arg) acc += 1;
|
|
9452
9452
|
}
|
|
9453
9453
|
__outer_off += __r;
|
|
9454
9454
|
}
|
|
9455
|
-
return LL2NUM((
|
|
9455
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9456
9456
|
}
|
|
9457
9457
|
|
|
9458
9458
|
static VALUE
|
|
9459
|
-
count_equal_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9459
|
+
count_equal_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint64_t value_arg, ca_size_t min_count)
|
|
9460
9460
|
{
|
|
9461
9461
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9462
9462
|
ca->elements > 0 &&
|
|
@@ -9474,7 +9474,7 @@ count_equal_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9474
9474
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9475
9475
|
CArray *co;
|
|
9476
9476
|
GetCArray(vout, co);
|
|
9477
|
-
|
|
9477
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9478
9478
|
ca_iter_state st;
|
|
9479
9479
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9480
9480
|
slab_axes, naxes, 0);
|
|
@@ -9487,18 +9487,18 @@ boolean8_t *m;
|
|
|
9487
9487
|
ca_size_t out_i = 0;
|
|
9488
9488
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9489
9489
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9490
|
-
|
|
9490
|
+
ca_u_int64_t acc;
|
|
9491
9491
|
ca_size_t masked_cnt = 0;
|
|
9492
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9492
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_uint64_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9493
9493
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9494
9494
|
if ( ! op_mask ) {
|
|
9495
9495
|
ca_create_mask(co);
|
|
9496
9496
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9497
9497
|
}
|
|
9498
9498
|
op_mask[out_i] = 1;
|
|
9499
|
-
op[out_i++] = (
|
|
9499
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9500
9500
|
} else {
|
|
9501
|
-
op[out_i++] = (
|
|
9501
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9502
9502
|
}
|
|
9503
9503
|
}
|
|
9504
9504
|
ca_iter_state_finish(&st);
|
|
@@ -9512,7 +9512,7 @@ ca_size_t out_i = 0;
|
|
|
9512
9512
|
typedef struct {
|
|
9513
9513
|
void *chunk;
|
|
9514
9514
|
CArray *ca;
|
|
9515
|
-
|
|
9515
|
+
ca_u_float32_t value_arg;
|
|
9516
9516
|
ca_size_t min_count;
|
|
9517
9517
|
} count_equal_ki_stream_f32_t;
|
|
9518
9518
|
|
|
@@ -9521,7 +9521,7 @@ count_equal_ki_stream_f32 (VALUE arg)
|
|
|
9521
9521
|
{
|
|
9522
9522
|
count_equal_ki_stream_f32_t *__ctx = (count_equal_ki_stream_f32_t *) arg;
|
|
9523
9523
|
CArray *ca = __ctx->ca;
|
|
9524
|
-
|
|
9524
|
+
ca_u_float32_t value_arg = __ctx->value_arg;
|
|
9525
9525
|
(void) value_arg;
|
|
9526
9526
|
ca_size_t min_count = __ctx->min_count;
|
|
9527
9527
|
(void) min_count;
|
|
@@ -9536,18 +9536,18 @@ count_equal_ki_stream_f32 (VALUE arg)
|
|
|
9536
9536
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
9537
9537
|
ca_size_t __counts[CA_RANK_MAX];
|
|
9538
9538
|
ca_size_t __strides[CA_RANK_MAX];
|
|
9539
|
-
ca_size_t __s = sizeof(
|
|
9539
|
+
ca_size_t __s = sizeof(ca_u_float32_t);
|
|
9540
9540
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
9541
9541
|
__strides[__k] = __s;
|
|
9542
9542
|
__s *= ca->dim[__k];
|
|
9543
9543
|
}
|
|
9544
9544
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9545
9545
|
ca_size_t __outer_off = 0;
|
|
9546
|
-
|
|
9546
|
+
ca_u_int64_t acc;
|
|
9547
9547
|
acc = (0);
|
|
9548
9548
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9549
9549
|
(void) masked_cnt;
|
|
9550
|
-
|
|
9550
|
+
ca_u_float32_t *__chunk = (ca_u_float32_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_float32_t));
|
|
9551
9551
|
__ctx->chunk = __chunk;
|
|
9552
9552
|
while ( __outer_off < __outer ) {
|
|
9553
9553
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9557,16 +9557,16 @@ count_equal_ki_stream_f32 (VALUE arg)
|
|
|
9557
9557
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9558
9558
|
ca_size_t __i;
|
|
9559
9559
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9560
|
-
|
|
9560
|
+
ca_u_float32_t v = __chunk[__i];
|
|
9561
9561
|
if (v == value_arg) acc += 1;
|
|
9562
9562
|
}
|
|
9563
9563
|
__outer_off += __r;
|
|
9564
9564
|
}
|
|
9565
|
-
return LL2NUM((
|
|
9565
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9566
9566
|
}
|
|
9567
9567
|
|
|
9568
9568
|
static VALUE
|
|
9569
|
-
count_equal_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9569
|
+
count_equal_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float32_t value_arg, ca_size_t min_count)
|
|
9570
9570
|
{
|
|
9571
9571
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9572
9572
|
ca->elements > 0 &&
|
|
@@ -9584,7 +9584,7 @@ count_equal_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9584
9584
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9585
9585
|
CArray *co;
|
|
9586
9586
|
GetCArray(vout, co);
|
|
9587
|
-
|
|
9587
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9588
9588
|
ca_iter_state st;
|
|
9589
9589
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9590
9590
|
slab_axes, naxes, 0);
|
|
@@ -9597,18 +9597,18 @@ boolean8_t *m;
|
|
|
9597
9597
|
ca_size_t out_i = 0;
|
|
9598
9598
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9599
9599
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9600
|
-
|
|
9600
|
+
ca_u_int64_t acc;
|
|
9601
9601
|
ca_size_t masked_cnt = 0;
|
|
9602
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9602
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_float32_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9603
9603
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9604
9604
|
if ( ! op_mask ) {
|
|
9605
9605
|
ca_create_mask(co);
|
|
9606
9606
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9607
9607
|
}
|
|
9608
9608
|
op_mask[out_i] = 1;
|
|
9609
|
-
op[out_i++] = (
|
|
9609
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9610
9610
|
} else {
|
|
9611
|
-
op[out_i++] = (
|
|
9611
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9612
9612
|
}
|
|
9613
9613
|
}
|
|
9614
9614
|
ca_iter_state_finish(&st);
|
|
@@ -9622,7 +9622,7 @@ ca_size_t out_i = 0;
|
|
|
9622
9622
|
typedef struct {
|
|
9623
9623
|
void *chunk;
|
|
9624
9624
|
CArray *ca;
|
|
9625
|
-
|
|
9625
|
+
ca_u_float64_t value_arg;
|
|
9626
9626
|
ca_size_t min_count;
|
|
9627
9627
|
} count_equal_ki_stream_f64_t;
|
|
9628
9628
|
|
|
@@ -9631,7 +9631,7 @@ count_equal_ki_stream_f64 (VALUE arg)
|
|
|
9631
9631
|
{
|
|
9632
9632
|
count_equal_ki_stream_f64_t *__ctx = (count_equal_ki_stream_f64_t *) arg;
|
|
9633
9633
|
CArray *ca = __ctx->ca;
|
|
9634
|
-
|
|
9634
|
+
ca_u_float64_t value_arg = __ctx->value_arg;
|
|
9635
9635
|
(void) value_arg;
|
|
9636
9636
|
ca_size_t min_count = __ctx->min_count;
|
|
9637
9637
|
(void) min_count;
|
|
@@ -9646,18 +9646,18 @@ count_equal_ki_stream_f64 (VALUE arg)
|
|
|
9646
9646
|
ca_size_t __starts[CA_RANK_MAX] = {0};
|
|
9647
9647
|
ca_size_t __counts[CA_RANK_MAX];
|
|
9648
9648
|
ca_size_t __strides[CA_RANK_MAX];
|
|
9649
|
-
ca_size_t __s = sizeof(
|
|
9649
|
+
ca_size_t __s = sizeof(ca_u_float64_t);
|
|
9650
9650
|
for ( __k = ca->ndim - 1; __k >= 0; __k-- ) {
|
|
9651
9651
|
__strides[__k] = __s;
|
|
9652
9652
|
__s *= ca->dim[__k];
|
|
9653
9653
|
}
|
|
9654
9654
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9655
9655
|
ca_size_t __outer_off = 0;
|
|
9656
|
-
|
|
9656
|
+
ca_u_int64_t acc;
|
|
9657
9657
|
acc = (0);
|
|
9658
9658
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9659
9659
|
(void) masked_cnt;
|
|
9660
|
-
|
|
9660
|
+
ca_u_float64_t *__chunk = (ca_u_float64_t *) ca_lazy_arena_acquire(__chunk_elems * sizeof(ca_u_float64_t));
|
|
9661
9661
|
__ctx->chunk = __chunk;
|
|
9662
9662
|
while ( __outer_off < __outer ) {
|
|
9663
9663
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
@@ -9667,16 +9667,16 @@ count_equal_ki_stream_f64 (VALUE arg)
|
|
|
9667
9667
|
ca_xfer_stride(ca, __starts, __counts, __strides, __chunk, CA_XFER_GET);
|
|
9668
9668
|
ca_size_t __i;
|
|
9669
9669
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
9670
|
-
|
|
9670
|
+
ca_u_float64_t v = __chunk[__i];
|
|
9671
9671
|
if (v == value_arg) acc += 1;
|
|
9672
9672
|
}
|
|
9673
9673
|
__outer_off += __r;
|
|
9674
9674
|
}
|
|
9675
|
-
return LL2NUM((
|
|
9675
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9676
9676
|
}
|
|
9677
9677
|
|
|
9678
9678
|
static VALUE
|
|
9679
|
-
count_equal_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis,
|
|
9679
|
+
count_equal_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float64_t value_arg, ca_size_t min_count)
|
|
9680
9680
|
{
|
|
9681
9681
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9682
9682
|
ca->elements > 0 &&
|
|
@@ -9694,7 +9694,7 @@ count_equal_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
9694
9694
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9695
9695
|
CArray *co;
|
|
9696
9696
|
GetCArray(vout, co);
|
|
9697
|
-
|
|
9697
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9698
9698
|
ca_iter_state st;
|
|
9699
9699
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9700
9700
|
slab_axes, naxes, 0);
|
|
@@ -9707,18 +9707,18 @@ boolean8_t *m;
|
|
|
9707
9707
|
ca_size_t out_i = 0;
|
|
9708
9708
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9709
9709
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9710
|
-
|
|
9710
|
+
ca_u_int64_t acc;
|
|
9711
9711
|
ca_size_t masked_cnt = 0;
|
|
9712
|
-
CA_SLAB_REDUCE_T_PLUS_EX(
|
|
9712
|
+
CA_SLAB_REDUCE_T_PLUS_EX(ca_u_float64_t, st, p, m, acc, 0, if (v == value_arg) acc += 1, masked_cnt);
|
|
9713
9713
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
9714
9714
|
if ( ! op_mask ) {
|
|
9715
9715
|
ca_create_mask(co);
|
|
9716
9716
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9717
9717
|
}
|
|
9718
9718
|
op_mask[out_i] = 1;
|
|
9719
|
-
op[out_i++] = (
|
|
9719
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9720
9720
|
} else {
|
|
9721
|
-
op[out_i++] = (
|
|
9721
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9722
9722
|
}
|
|
9723
9723
|
}
|
|
9724
9724
|
ca_iter_state_finish(&st);
|
|
@@ -9763,7 +9763,7 @@ count_equal_ki_stream_object (VALUE arg)
|
|
|
9763
9763
|
}
|
|
9764
9764
|
for ( __k = 1; __k < ca->ndim; __k++ ) __counts[__k] = ca->dim[__k];
|
|
9765
9765
|
ca_size_t __outer_off = 0;
|
|
9766
|
-
|
|
9766
|
+
ca_u_int64_t acc;
|
|
9767
9767
|
acc = (0);
|
|
9768
9768
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
9769
9769
|
(void) masked_cnt;
|
|
@@ -9782,7 +9782,7 @@ count_equal_ki_stream_object (VALUE arg)
|
|
|
9782
9782
|
}
|
|
9783
9783
|
__outer_off += __r;
|
|
9784
9784
|
}
|
|
9785
|
-
return LL2NUM((
|
|
9785
|
+
return LL2NUM((ca_u_int64_t) (acc));
|
|
9786
9786
|
}
|
|
9787
9787
|
|
|
9788
9788
|
/* The object lane calls Ruby for every cell, so a walk can raise part
|
|
@@ -9834,7 +9834,7 @@ count_equal_ki_native_object_body (VALUE arg)
|
|
|
9834
9834
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9835
9835
|
CArray *co;
|
|
9836
9836
|
GetCArray(vout, co);
|
|
9837
|
-
|
|
9837
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9838
9838
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9839
9839
|
slab_axes, naxes, 0);
|
|
9840
9840
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9846,7 +9846,7 @@ boolean8_t *m;
|
|
|
9846
9846
|
ca_size_t out_i = 0;
|
|
9847
9847
|
boolean8_t *op_mask = NULL; /* lazily allocated on first UNDEF */
|
|
9848
9848
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
9849
|
-
|
|
9849
|
+
ca_u_int64_t acc;
|
|
9850
9850
|
ca_size_t masked_cnt = 0;
|
|
9851
9851
|
CA_SLAB_REDUCE_T_EX(VALUE, st, p, m, acc, 0, if (ca_obj_equal(v, value_arg)) acc += 1, masked_cnt);
|
|
9852
9852
|
if ( ca_too_few_present(st.slab_elements - masked_cnt, min_count, 0) ) {
|
|
@@ -9855,9 +9855,9 @@ ca_size_t out_i = 0;
|
|
|
9855
9855
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
9856
9856
|
}
|
|
9857
9857
|
op_mask[out_i] = 1;
|
|
9858
|
-
op[out_i++] = (
|
|
9858
|
+
op[out_i++] = (ca_u_int64_t) 0; /* sentinel; mask bit is what counts */
|
|
9859
9859
|
} else {
|
|
9860
|
-
op[out_i++] = (
|
|
9860
|
+
op[out_i++] = (ca_u_int64_t) (acc);
|
|
9861
9861
|
}
|
|
9862
9862
|
}
|
|
9863
9863
|
ca_iter_state_finish(&st);
|
|
@@ -10002,54 +10002,54 @@ rb_ca_count_equal_ki (int argc, VALUE *argv, VALUE self)
|
|
|
10002
10002
|
}
|
|
10003
10003
|
case CA_INT16: {
|
|
10004
10004
|
ca_check_integer_fits(rval, CA_INT16);
|
|
10005
|
-
|
|
10005
|
+
ca_u_int16_t value_arg = (ca_u_int16_t) NUM2LL(rval);
|
|
10006
10006
|
result = count_equal_ki_native_i16(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10007
10007
|
RB_GC_GUARD(self);
|
|
10008
10008
|
break;
|
|
10009
10009
|
}
|
|
10010
10010
|
case CA_UINT16: {
|
|
10011
10011
|
ca_check_integer_fits(rval, CA_UINT16);
|
|
10012
|
-
|
|
10012
|
+
ca_u_uint16_t value_arg = (ca_u_uint16_t) NUM2ULL(rval);
|
|
10013
10013
|
result = count_equal_ki_native_u16(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10014
10014
|
RB_GC_GUARD(self);
|
|
10015
10015
|
break;
|
|
10016
10016
|
}
|
|
10017
10017
|
case CA_INT32: {
|
|
10018
10018
|
ca_check_integer_fits(rval, CA_INT32);
|
|
10019
|
-
|
|
10019
|
+
ca_u_int32_t value_arg = (ca_u_int32_t) NUM2LL(rval);
|
|
10020
10020
|
result = count_equal_ki_native_i32(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10021
10021
|
RB_GC_GUARD(self);
|
|
10022
10022
|
break;
|
|
10023
10023
|
}
|
|
10024
10024
|
case CA_UINT32: {
|
|
10025
10025
|
ca_check_integer_fits(rval, CA_UINT32);
|
|
10026
|
-
|
|
10026
|
+
ca_u_uint32_t value_arg = (ca_u_uint32_t) NUM2ULL(rval);
|
|
10027
10027
|
result = count_equal_ki_native_u32(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10028
10028
|
RB_GC_GUARD(self);
|
|
10029
10029
|
break;
|
|
10030
10030
|
}
|
|
10031
10031
|
case CA_INT64: {
|
|
10032
10032
|
ca_check_integer_fits(rval, CA_INT64);
|
|
10033
|
-
|
|
10033
|
+
ca_u_int64_t value_arg = (ca_u_int64_t) NUM2LL(rval);
|
|
10034
10034
|
result = count_equal_ki_native_i64(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10035
10035
|
RB_GC_GUARD(self);
|
|
10036
10036
|
break;
|
|
10037
10037
|
}
|
|
10038
10038
|
case CA_UINT64: {
|
|
10039
10039
|
ca_check_integer_fits(rval, CA_UINT64);
|
|
10040
|
-
|
|
10040
|
+
ca_u_uint64_t value_arg = (ca_u_uint64_t) NUM2ULL(rval);
|
|
10041
10041
|
result = count_equal_ki_native_u64(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10042
10042
|
RB_GC_GUARD(self);
|
|
10043
10043
|
break;
|
|
10044
10044
|
}
|
|
10045
10045
|
case CA_FLOAT32: {
|
|
10046
|
-
|
|
10046
|
+
ca_u_float32_t value_arg = (ca_u_float32_t) NUM2DBL(rval);
|
|
10047
10047
|
result = count_equal_ki_native_f32(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10048
10048
|
RB_GC_GUARD(self);
|
|
10049
10049
|
break;
|
|
10050
10050
|
}
|
|
10051
10051
|
case CA_FLOAT64: {
|
|
10052
|
-
|
|
10052
|
+
ca_u_float64_t value_arg = (ca_u_float64_t) NUM2DBL(rval);
|
|
10053
10053
|
result = count_equal_ki_native_f64(self, src, slab_axes, naxes, keep_axis, value_arg, min_count);
|
|
10054
10054
|
RB_GC_GUARD(self);
|
|
10055
10055
|
break;
|