carray 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +2 -1
- data/CHANGELOG.md +845 -3
- data/{NEWS.md → CHANGELOG.v1.md} +1 -1
- data/README.md +33 -33
- data/carray.gemspec +12 -8
- data/ext/ca_axis_dispatch.c +33 -4
- data/ext/ca_axis_group.c +256 -114
- data/ext/ca_binop_dispatch.c +5 -6
- data/ext/ca_binop_dispatch.h +0 -7
- data/ext/ca_categorical_iterator.c +121 -67
- data/ext/ca_for_each_element.h +6 -8
- data/ext/ca_group_iter.c +1 -1
- data/ext/ca_kernel_iterator.c +386 -74
- data/ext/ca_kernel_iterator.h +145 -45
- data/ext/ca_obj_array.c +62 -20
- data/ext/ca_obj_bincmp.c +57 -27
- data/ext/ca_obj_binop.c +86 -30
- data/ext/ca_obj_block.c +4 -4
- data/ext/ca_obj_const_string.c +88 -27
- data/ext/ca_obj_face.c +104 -0
- data/ext/ca_obj_face.h +48 -1
- data/ext/ca_obj_fake.c +11 -0
- data/ext/ca_obj_fixlen_string.c +21 -6
- data/ext/ca_obj_grid.c +11 -3
- data/ext/ca_obj_meld.c +125 -22
- data/ext/ca_obj_moncmp.c +33 -14
- data/ext/ca_obj_monop.c +19 -7
- data/ext/ca_obj_object.c +107 -4
- data/ext/ca_obj_record.c +3 -1
- data/ext/ca_obj_refer.c +30 -6
- data/ext/ca_obj_roll.c +13 -5
- data/ext/ca_obj_select.c +49 -34
- data/ext/ca_obj_select_axis.c +16 -6
- data/ext/ca_obj_shift.c +3 -3
- data/ext/ca_obj_stack.c +3 -8
- data/ext/ca_obj_stride.c +178 -24
- data/ext/ca_obj_string.c +11 -5
- data/ext/ca_obj_tile.c +13 -5
- data/ext/ca_obj_time.c +3 -1
- data/ext/ca_obj_timedelta.c +3 -1
- data/ext/ca_obj_transpose.c +1 -1
- data/ext/ca_obj_triop.c +62 -23
- data/ext/ca_obj_window.c +94 -19
- data/ext/ca_op_cmplx64.h +123 -0
- data/ext/ca_op_ipower.c +1 -5
- data/ext/ca_rng_normal.h +42 -0
- data/ext/ca_rng_xoshiro256pp.h +105 -0
- data/ext/ca_sort_kernels.h +5 -5
- data/ext/ca_sweep_engine.c +362 -155
- data/ext/ca_sweep_engine.h +39 -9
- data/ext/ca_transform_common.c +7 -0
- data/ext/carray.h +148 -132
- data/ext/carray_access.c +83 -49
- data/ext/carray_address_basis.c +590 -0
- data/ext/carray_bincount.c +8 -8
- data/ext/carray_broadcast.c +102 -9
- data/ext/carray_build_flags.h +3 -0
- data/ext/carray_call_cfunc.c +2010 -335
- data/ext/carray_call_cfunc.h +153 -0
- data/ext/carray_cast.c +184 -72
- data/ext/carray_conversion.c +22 -22
- data/ext/carray_copy.c +67 -32
- data/ext/carray_core.c +140 -13
- data/ext/carray_count.c +10 -11
- data/ext/carray_element.c +7 -7
- data/ext/carray_factorize.c +72 -51
- data/ext/carray_hold.c +1 -1
- data/ext/carray_index_classifier.c +7 -21
- data/ext/carray_internal.h +36 -3
- data/ext/carray_kernels_bincmp.c +1 -0
- data/ext/carray_kernels_binop.c +320 -298
- data/ext/carray_kernels_init.c +1095 -0
- data/ext/carray_kernels_map.c +1 -0
- data/ext/carray_kernels_moncmp.c +1 -0
- data/ext/carray_kernels_monop.c +412 -411
- data/ext/carray_kernels_reduce_aggregate.c +256 -87
- data/ext/carray_kernels_reduce_boolean.c +1 -0
- data/ext/carray_kernels_reduce_cumulative.c +324 -54
- data/ext/carray_kernels_reduce_extreme.c +605 -58
- data/ext/carray_kernels_reduce_variance.c +1 -0
- data/ext/carray_kernels_scan.c +5 -4
- data/ext/carray_kernels_search.c +187 -106
- data/ext/carray_kernels_sort.c +1 -0
- data/ext/carray_kernels_triop.c +1 -0
- data/ext/carray_lazy.c +175 -5
- data/ext/carray_loop.c +7 -1
- data/ext/carray_mask.c +29 -13
- data/ext/carray_median_percentile.c +56 -1
- data/ext/carray_memory_view.c +46 -21
- data/ext/carray_operator.c +50 -53
- data/ext/carray_order.c +1 -1
- data/ext/carray_partition.c +4 -4
- data/ext/carray_random.c +391 -47
- data/ext/carray_scatter.c +1 -1
- data/ext/carray_slab.c +13 -0
- data/ext/carray_sort.c +22 -24
- data/ext/carray_sort_kernel.c +10 -10
- data/ext/carray_test.c +1 -1
- data/ext/extconf.rb +18 -0
- data/ext/mk_call_cfunc.rb +323 -145
- data/ext/mkkernel.rb +651 -129
- data/ext/ruby_carray.c +28 -17
- data/ext/version.h +4 -4
- data/lib/carray/autoload_carray.rb +24 -3
- data/lib/carray/autoload_method_extension.rb +14 -1
- data/lib/carray/axis_group.rb +84 -7
- data/lib/carray/basics.rb +63 -59
- data/lib/carray/bincount_nd.rb +12 -12
- data/lib/carray/block_iterator.rb +111 -25
- data/lib/carray/categorical.rb +154 -37
- data/lib/carray/categorical_iterator.rb +300 -151
- data/lib/carray/conditional.rb +14 -14
- data/lib/carray/const_string.rb +131 -27
- data/lib/carray/construct.rb +61 -0
- data/lib/carray/core_extensions.rb +62 -5
- data/lib/carray/data_type_extension.rb +21 -1
- data/lib/carray/data_type_limits.rb +91 -0
- data/lib/carray/fixlen_string.rb +1 -1
- data/lib/carray/frame/concat.rb +3 -3
- data/lib/carray/frame/convert.rb +1 -1
- data/lib/carray/frame/csv_parser.rb +11 -4
- data/lib/carray/frame/frame.rb +85 -21
- data/lib/carray/frame/group.rb +36 -3
- data/lib/carray/frame/io.rb +224 -15
- data/lib/carray/frame/records.rb +18 -4
- data/lib/carray/frame/sort.rb +1 -1
- data/lib/carray/frame/verbs.rb +32 -12
- data/lib/carray/fuse_source.rb +123 -0
- data/lib/carray/fusion.rb +218 -0
- data/lib/carray/histogram.rb +16 -16
- data/lib/carray/inspect.rb +43 -16
- data/lib/carray/iterator.rb +147 -3
- data/lib/carray/lazy.rb +88 -73
- data/lib/carray/mask_gap_fill.rb +3 -1
- data/lib/carray/meld_reduce.rb +2 -2
- data/lib/carray/methods/align_addr.rb +1 -1
- data/lib/carray/methods/composition.rb +1 -1
- data/lib/carray/methods/discovery_along.rb +74 -0
- data/lib/carray/methods/factorize.rb +50 -0
- data/lib/carray/methods/is_in.rb +25 -14
- data/lib/carray/methods/locate_addr.rb +81 -3
- data/lib/carray/methods/mask_duplicates.rb +36 -2
- data/lib/carray/methods/meshgrid.rb +4 -5
- data/lib/carray/methods/mode.rb +2 -2
- data/lib/carray/methods/nunique.rb +23 -2
- data/lib/carray/methods/repeat.rb +110 -0
- data/lib/carray/methods/snap.rb +7 -2
- data/lib/carray/methods/unique.rb +44 -4
- data/lib/carray/methods/value_counts.rb +2 -2
- data/lib/carray/rng.rb +86 -0
- data/lib/carray/runtime.rb +0 -19
- data/lib/carray/slab_iterator.rb +77 -19
- data/lib/carray/string_operation_extension.rb +10 -6
- data/lib/carray/time.rb +906 -491
- data/lib/carray/window_iterator.rb +421 -27
- data/lib/carray.rb +7 -5
- data/yard-stubs/ca_obj_array.rb +385 -0
- data/yard-stubs/ca_obj_bitarray.rb +38 -0
- data/yard-stubs/ca_obj_bitfield.rb +43 -0
- data/yard-stubs/ca_obj_block.rb +68 -0
- data/yard-stubs/ca_obj_byte_swap.rb +56 -0
- data/yard-stubs/ca_obj_fake.rb +31 -0
- data/yard-stubs/ca_obj_farray.rb +32 -0
- data/yard-stubs/ca_obj_field.rb +45 -0
- data/yard-stubs/ca_obj_grid.rb +35 -0
- data/yard-stubs/ca_obj_refer.rb +72 -0
- data/yard-stubs/ca_obj_roll.rb +45 -0
- data/yard-stubs/ca_obj_shift.rb +43 -0
- data/yard-stubs/ca_obj_stride.rb +181 -0
- data/yard-stubs/ca_obj_tile.rb +29 -0
- data/yard-stubs/ca_obj_transpose.rb +40 -0
- data/yard-stubs/ca_obj_window.rb +57 -0
- data/yard-stubs/carray_access.rb +131 -0
- data/yard-stubs/carray_attribute.rb +246 -0
- data/yard-stubs/carray_broadcast.rb +37 -0
- data/yard-stubs/carray_cast.rb +489 -0
- data/yard-stubs/carray_class.rb +65 -0
- data/yard-stubs/carray_conversion.rb +76 -0
- data/yard-stubs/carray_copy.rb +79 -0
- data/yard-stubs/carray_core.rb +34 -0
- data/yard-stubs/carray_count.rb +84 -0
- data/yard-stubs/carray_element.rb +108 -0
- data/yard-stubs/carray_generate.rb +66 -0
- data/yard-stubs/carray_lazy.rb +228 -0
- data/yard-stubs/carray_loop.rb +140 -0
- data/yard-stubs/carray_mask.rb +259 -0
- data/yard-stubs/carray_math.rb +1615 -0
- data/yard-stubs/carray_mathfunc.rb +45 -0
- data/yard-stubs/carray_median_percentile.rb +103 -0
- data/yard-stubs/carray_memory_view.rb +163 -0
- data/yard-stubs/carray_order.rb +252 -0
- data/yard-stubs/carray_random.rb +89 -0
- data/yard-stubs/carray_scatter.rb +106 -0
- data/yard-stubs/carray_slab.rb +59 -0
- data/yard-stubs/carray_sort.rb +165 -0
- data/yard-stubs/carray_test.rb +85 -0
- data/yard-stubs/carray_undef.rb +64 -0
- data/yard-stubs/carray_utils.rb +97 -0
- data/yard-stubs/ruby_carray.rb +193 -0
- metadata +67 -10
- data/ext/ca_obj_unbound_repeat.c +0 -496
|
@@ -34,6 +34,7 @@ static inline VALUE BOOL2VAL (boolean8_t x) { return x ? Qtrue : Qfalse; }
|
|
|
34
34
|
#endif
|
|
35
35
|
|
|
36
36
|
#include "ca_op_powi.h"
|
|
37
|
+
#include "ca_op_cmplx64.h"
|
|
37
38
|
|
|
38
39
|
/* ===== min_ki ============================================ */
|
|
39
40
|
|
|
@@ -136,6 +137,7 @@ min_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
136
137
|
(void) v;
|
|
137
138
|
}
|
|
138
139
|
}
|
|
140
|
+
|
|
139
141
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
140
142
|
op[__li_o * __li_INNER + __li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
141
143
|
}
|
|
@@ -215,6 +217,7 @@ min_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
215
217
|
(void) v;
|
|
216
218
|
}
|
|
217
219
|
}
|
|
220
|
+
|
|
218
221
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
219
222
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
220
223
|
}
|
|
@@ -340,6 +343,7 @@ min_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
340
343
|
(void) v;
|
|
341
344
|
}
|
|
342
345
|
}
|
|
346
|
+
|
|
343
347
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
344
348
|
__li_op_k[__li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
345
349
|
}
|
|
@@ -370,8 +374,8 @@ min_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
370
374
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
371
375
|
tile reduce + write maintained.
|
|
372
376
|
|
|
373
|
-
Performance characteristic (deliver-via-view
|
|
374
|
-
|
|
377
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
378
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
375
379
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
376
380
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
377
381
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -500,6 +504,7 @@ min_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
500
504
|
(void) v;
|
|
501
505
|
}
|
|
502
506
|
}
|
|
507
|
+
|
|
503
508
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
504
509
|
__li_op_k[__li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
505
510
|
}
|
|
@@ -655,6 +660,7 @@ min_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
655
660
|
(void) v;
|
|
656
661
|
}
|
|
657
662
|
}
|
|
663
|
+
|
|
658
664
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
659
665
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
660
666
|
}
|
|
@@ -734,6 +740,7 @@ min_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
734
740
|
(void) v;
|
|
735
741
|
}
|
|
736
742
|
}
|
|
743
|
+
|
|
737
744
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
738
745
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
739
746
|
}
|
|
@@ -859,6 +866,7 @@ min_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
859
866
|
(void) v;
|
|
860
867
|
}
|
|
861
868
|
}
|
|
869
|
+
|
|
862
870
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
863
871
|
__li_op_k[__li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
864
872
|
}
|
|
@@ -889,8 +897,8 @@ min_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
889
897
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
890
898
|
tile reduce + write maintained.
|
|
891
899
|
|
|
892
|
-
Performance characteristic (deliver-via-view
|
|
893
|
-
|
|
900
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
901
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
894
902
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
895
903
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
896
904
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -1019,6 +1027,7 @@ min_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
1019
1027
|
(void) v;
|
|
1020
1028
|
}
|
|
1021
1029
|
}
|
|
1030
|
+
|
|
1022
1031
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1023
1032
|
__li_op_k[__li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
1024
1033
|
}
|
|
@@ -1174,6 +1183,7 @@ min_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1174
1183
|
(void) v;
|
|
1175
1184
|
}
|
|
1176
1185
|
}
|
|
1186
|
+
|
|
1177
1187
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1178
1188
|
op[__li_o * __li_INNER + __li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
1179
1189
|
}
|
|
@@ -1253,6 +1263,7 @@ min_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1253
1263
|
(void) v;
|
|
1254
1264
|
}
|
|
1255
1265
|
}
|
|
1266
|
+
|
|
1256
1267
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1257
1268
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
1258
1269
|
}
|
|
@@ -1378,6 +1389,7 @@ min_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1378
1389
|
(void) v;
|
|
1379
1390
|
}
|
|
1380
1391
|
}
|
|
1392
|
+
|
|
1381
1393
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1382
1394
|
__li_op_k[__li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
1383
1395
|
}
|
|
@@ -1408,8 +1420,8 @@ min_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1408
1420
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
1409
1421
|
tile reduce + write maintained.
|
|
1410
1422
|
|
|
1411
|
-
Performance characteristic (deliver-via-view
|
|
1412
|
-
|
|
1423
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
1424
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
1413
1425
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
1414
1426
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
1415
1427
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -1538,6 +1550,7 @@ min_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1538
1550
|
(void) v;
|
|
1539
1551
|
}
|
|
1540
1552
|
}
|
|
1553
|
+
|
|
1541
1554
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1542
1555
|
__li_op_k[__li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
1543
1556
|
}
|
|
@@ -1693,6 +1706,7 @@ min_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1693
1706
|
(void) v;
|
|
1694
1707
|
}
|
|
1695
1708
|
}
|
|
1709
|
+
|
|
1696
1710
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1697
1711
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
1698
1712
|
}
|
|
@@ -1772,6 +1786,7 @@ min_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1772
1786
|
(void) v;
|
|
1773
1787
|
}
|
|
1774
1788
|
}
|
|
1789
|
+
|
|
1775
1790
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1776
1791
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
1777
1792
|
}
|
|
@@ -1897,6 +1912,7 @@ min_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1897
1912
|
(void) v;
|
|
1898
1913
|
}
|
|
1899
1914
|
}
|
|
1915
|
+
|
|
1900
1916
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1901
1917
|
__li_op_k[__li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
1902
1918
|
}
|
|
@@ -1927,8 +1943,8 @@ min_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
1927
1943
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
1928
1944
|
tile reduce + write maintained.
|
|
1929
1945
|
|
|
1930
|
-
Performance characteristic (deliver-via-view
|
|
1931
|
-
|
|
1946
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
1947
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
1932
1948
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
1933
1949
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
1934
1950
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -2057,6 +2073,7 @@ min_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2057
2073
|
(void) v;
|
|
2058
2074
|
}
|
|
2059
2075
|
}
|
|
2076
|
+
|
|
2060
2077
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2061
2078
|
__li_op_k[__li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
2062
2079
|
}
|
|
@@ -2212,6 +2229,7 @@ min_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2212
2229
|
(void) v;
|
|
2213
2230
|
}
|
|
2214
2231
|
}
|
|
2232
|
+
|
|
2215
2233
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2216
2234
|
op[__li_o * __li_INNER + __li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
2217
2235
|
}
|
|
@@ -2291,6 +2309,7 @@ min_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2291
2309
|
(void) v;
|
|
2292
2310
|
}
|
|
2293
2311
|
}
|
|
2312
|
+
|
|
2294
2313
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2295
2314
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
2296
2315
|
}
|
|
@@ -2416,6 +2435,7 @@ min_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2416
2435
|
(void) v;
|
|
2417
2436
|
}
|
|
2418
2437
|
}
|
|
2438
|
+
|
|
2419
2439
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2420
2440
|
__li_op_k[__li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
2421
2441
|
}
|
|
@@ -2446,8 +2466,8 @@ min_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2446
2466
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
2447
2467
|
tile reduce + write maintained.
|
|
2448
2468
|
|
|
2449
|
-
Performance characteristic (deliver-via-view
|
|
2450
|
-
|
|
2469
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
2470
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
2451
2471
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
2452
2472
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
2453
2473
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -2576,6 +2596,7 @@ min_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2576
2596
|
(void) v;
|
|
2577
2597
|
}
|
|
2578
2598
|
}
|
|
2599
|
+
|
|
2579
2600
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2580
2601
|
__li_op_k[__li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
2581
2602
|
}
|
|
@@ -2731,6 +2752,7 @@ min_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2731
2752
|
(void) v;
|
|
2732
2753
|
}
|
|
2733
2754
|
}
|
|
2755
|
+
|
|
2734
2756
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2735
2757
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
2736
2758
|
}
|
|
@@ -2810,6 +2832,7 @@ min_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2810
2832
|
(void) v;
|
|
2811
2833
|
}
|
|
2812
2834
|
}
|
|
2835
|
+
|
|
2813
2836
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2814
2837
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
2815
2838
|
}
|
|
@@ -2935,6 +2958,7 @@ min_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2935
2958
|
(void) v;
|
|
2936
2959
|
}
|
|
2937
2960
|
}
|
|
2961
|
+
|
|
2938
2962
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
2939
2963
|
__li_op_k[__li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
2940
2964
|
}
|
|
@@ -2965,8 +2989,8 @@ min_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
2965
2989
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
2966
2990
|
tile reduce + write maintained.
|
|
2967
2991
|
|
|
2968
|
-
Performance characteristic (deliver-via-view
|
|
2969
|
-
|
|
2992
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
2993
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
2970
2994
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
2971
2995
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
2972
2996
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -3095,6 +3119,7 @@ min_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3095
3119
|
(void) v;
|
|
3096
3120
|
}
|
|
3097
3121
|
}
|
|
3122
|
+
|
|
3098
3123
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3099
3124
|
__li_op_k[__li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
3100
3125
|
}
|
|
@@ -3250,6 +3275,7 @@ min_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3250
3275
|
(void) v;
|
|
3251
3276
|
}
|
|
3252
3277
|
}
|
|
3278
|
+
|
|
3253
3279
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3254
3280
|
op[__li_o * __li_INNER + __li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
3255
3281
|
}
|
|
@@ -3329,6 +3355,7 @@ min_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3329
3355
|
(void) v;
|
|
3330
3356
|
}
|
|
3331
3357
|
}
|
|
3358
|
+
|
|
3332
3359
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3333
3360
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
3334
3361
|
}
|
|
@@ -3454,6 +3481,7 @@ min_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3454
3481
|
(void) v;
|
|
3455
3482
|
}
|
|
3456
3483
|
}
|
|
3484
|
+
|
|
3457
3485
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3458
3486
|
__li_op_k[__li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
3459
3487
|
}
|
|
@@ -3484,8 +3512,8 @@ min_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3484
3512
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
3485
3513
|
tile reduce + write maintained.
|
|
3486
3514
|
|
|
3487
|
-
Performance characteristic (deliver-via-view
|
|
3488
|
-
|
|
3515
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
3516
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
3489
3517
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
3490
3518
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
3491
3519
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -3614,6 +3642,7 @@ min_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3614
3642
|
(void) v;
|
|
3615
3643
|
}
|
|
3616
3644
|
}
|
|
3645
|
+
|
|
3617
3646
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3618
3647
|
__li_op_k[__li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
3619
3648
|
}
|
|
@@ -3769,6 +3798,7 @@ min_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3769
3798
|
(void) v;
|
|
3770
3799
|
}
|
|
3771
3800
|
}
|
|
3801
|
+
|
|
3772
3802
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3773
3803
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
3774
3804
|
}
|
|
@@ -3848,6 +3878,7 @@ min_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3848
3878
|
(void) v;
|
|
3849
3879
|
}
|
|
3850
3880
|
}
|
|
3881
|
+
|
|
3851
3882
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3852
3883
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
3853
3884
|
}
|
|
@@ -3973,6 +4004,7 @@ min_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
3973
4004
|
(void) v;
|
|
3974
4005
|
}
|
|
3975
4006
|
}
|
|
4007
|
+
|
|
3976
4008
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
3977
4009
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
3978
4010
|
}
|
|
@@ -4003,8 +4035,8 @@ min_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4003
4035
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
4004
4036
|
tile reduce + write maintained.
|
|
4005
4037
|
|
|
4006
|
-
Performance characteristic (deliver-via-view
|
|
4007
|
-
|
|
4038
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
4039
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
4008
4040
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
4009
4041
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
4010
4042
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -4133,6 +4165,7 @@ min_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4133
4165
|
(void) v;
|
|
4134
4166
|
}
|
|
4135
4167
|
}
|
|
4168
|
+
|
|
4136
4169
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4137
4170
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
4138
4171
|
}
|
|
@@ -4216,6 +4249,7 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4216
4249
|
ca_size_t __outer_off = 0;
|
|
4217
4250
|
float acc;
|
|
4218
4251
|
acc = (INFINITY);
|
|
4252
|
+
int64_t __anf_seen = 0;
|
|
4219
4253
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
4220
4254
|
(void) masked_cnt;
|
|
4221
4255
|
ca_lazy_arena_enter();
|
|
@@ -4230,11 +4264,13 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4230
4264
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
4231
4265
|
float v = __chunk[__i];
|
|
4232
4266
|
acc = (v < acc) ? v : acc;
|
|
4267
|
+
__anf_seen |= (v == v);
|
|
4233
4268
|
}
|
|
4234
4269
|
__outer_off += __r;
|
|
4235
4270
|
}
|
|
4236
4271
|
ca_lazy_arena_release(__chunk);
|
|
4237
4272
|
ca_lazy_arena_exit();
|
|
4273
|
+
if ( ! __anf_seen && ca->elements > 0 ) acc = (float) NAN;
|
|
4238
4274
|
return rb_float_new((float) (acc));
|
|
4239
4275
|
}
|
|
4240
4276
|
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
@@ -4288,6 +4324,25 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4288
4324
|
(void) v;
|
|
4289
4325
|
}
|
|
4290
4326
|
}
|
|
4327
|
+
{
|
|
4328
|
+
int __li_susp = 0;
|
|
4329
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4330
|
+
if ( __li_buf_acc[__j] == (float) (INFINITY) ) { __li_susp = 1; break; }
|
|
4331
|
+
}
|
|
4332
|
+
if ( __li_susp ) {
|
|
4333
|
+
int8_t __li_buf_seen[512];
|
|
4334
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
4335
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4336
|
+
const float *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4337
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4338
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
4339
|
+
}
|
|
4340
|
+
}
|
|
4341
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4342
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
4343
|
+
}
|
|
4344
|
+
}
|
|
4345
|
+
}
|
|
4291
4346
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4292
4347
|
op[__li_o * __li_INNER + __li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
4293
4348
|
}
|
|
@@ -4367,6 +4422,25 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4367
4422
|
(void) v;
|
|
4368
4423
|
}
|
|
4369
4424
|
}
|
|
4425
|
+
{
|
|
4426
|
+
int __li_susp = 0;
|
|
4427
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4428
|
+
if ( __li_buf_acc[__j] == (float) (INFINITY) ) { __li_susp = 1; break; }
|
|
4429
|
+
}
|
|
4430
|
+
if ( __li_susp ) {
|
|
4431
|
+
int8_t __li_buf_seen[512];
|
|
4432
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
4433
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4434
|
+
const float *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4435
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4436
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
4437
|
+
}
|
|
4438
|
+
}
|
|
4439
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4440
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
4441
|
+
}
|
|
4442
|
+
}
|
|
4443
|
+
}
|
|
4370
4444
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4371
4445
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
4372
4446
|
}
|
|
@@ -4492,6 +4566,25 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4492
4566
|
(void) v;
|
|
4493
4567
|
}
|
|
4494
4568
|
}
|
|
4569
|
+
{
|
|
4570
|
+
int __li_susp = 0;
|
|
4571
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4572
|
+
if ( __li_buf_acc[__j] == (float) (INFINITY) ) { __li_susp = 1; break; }
|
|
4573
|
+
}
|
|
4574
|
+
if ( __li_susp ) {
|
|
4575
|
+
int8_t __li_buf_seen[512];
|
|
4576
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
4577
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4578
|
+
const float *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4579
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4580
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
4581
|
+
}
|
|
4582
|
+
}
|
|
4583
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4584
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
4585
|
+
}
|
|
4586
|
+
}
|
|
4587
|
+
}
|
|
4495
4588
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4496
4589
|
__li_op_k[__li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
4497
4590
|
}
|
|
@@ -4522,8 +4615,8 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4522
4615
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
4523
4616
|
tile reduce + write maintained.
|
|
4524
4617
|
|
|
4525
|
-
Performance characteristic (deliver-via-view
|
|
4526
|
-
|
|
4618
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
4619
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
4527
4620
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
4528
4621
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
4529
4622
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -4652,6 +4745,25 @@ min_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4652
4745
|
(void) v;
|
|
4653
4746
|
}
|
|
4654
4747
|
}
|
|
4748
|
+
{
|
|
4749
|
+
int __li_susp = 0;
|
|
4750
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4751
|
+
if ( __li_buf_acc[__j] == (float) (INFINITY) ) { __li_susp = 1; break; }
|
|
4752
|
+
}
|
|
4753
|
+
if ( __li_susp ) {
|
|
4754
|
+
int8_t __li_buf_seen[512];
|
|
4755
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
4756
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4757
|
+
const float *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
4758
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4759
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
4760
|
+
}
|
|
4761
|
+
}
|
|
4762
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4763
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
4764
|
+
}
|
|
4765
|
+
}
|
|
4766
|
+
}
|
|
4655
4767
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4656
4768
|
__li_op_k[__li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
4657
4769
|
}
|
|
@@ -4689,6 +4801,16 @@ ca_size_t out_i = 0;
|
|
|
4689
4801
|
#define __min_f32_EXPR(__x) ((float)(__x))
|
|
4690
4802
|
CA_SLAB_REDUCE_MIN8_EX(float, float, st, p, m, acc, INFINITY, __min_f32_EXPR, masked_cnt);
|
|
4691
4803
|
#undef __min_f32_EXPR
|
|
4804
|
+
/* all-NaN fix-up (all_nan_result: :nan). */
|
|
4805
|
+
if ( (acc == (INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
4806
|
+
int64_t __anf_seen = 0;
|
|
4807
|
+
ca_size_t __anf_mc = 0;
|
|
4808
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
4809
|
+
(void) __anf_mc;
|
|
4810
|
+
if ( ! __anf_seen ) {
|
|
4811
|
+
acc = (float) NAN;
|
|
4812
|
+
}
|
|
4813
|
+
}
|
|
4692
4814
|
if ( (min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count) ) {
|
|
4693
4815
|
if ( ! op_mask ) {
|
|
4694
4816
|
ca_create_mask(co);
|
|
@@ -4735,6 +4857,7 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4735
4857
|
ca_size_t __outer_off = 0;
|
|
4736
4858
|
double acc;
|
|
4737
4859
|
acc = (INFINITY);
|
|
4860
|
+
int64_t __anf_seen = 0;
|
|
4738
4861
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
4739
4862
|
(void) masked_cnt;
|
|
4740
4863
|
ca_lazy_arena_enter();
|
|
@@ -4749,11 +4872,13 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4749
4872
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
4750
4873
|
double v = __chunk[__i];
|
|
4751
4874
|
acc = (v < acc) ? v : acc;
|
|
4875
|
+
__anf_seen |= (v == v);
|
|
4752
4876
|
}
|
|
4753
4877
|
__outer_off += __r;
|
|
4754
4878
|
}
|
|
4755
4879
|
ca_lazy_arena_release(__chunk);
|
|
4756
4880
|
ca_lazy_arena_exit();
|
|
4881
|
+
if ( ! __anf_seen && ca->elements > 0 ) acc = (double) NAN;
|
|
4757
4882
|
return rb_float_new((double) (acc));
|
|
4758
4883
|
}
|
|
4759
4884
|
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
@@ -4807,6 +4932,25 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4807
4932
|
(void) v;
|
|
4808
4933
|
}
|
|
4809
4934
|
}
|
|
4935
|
+
{
|
|
4936
|
+
int __li_susp = 0;
|
|
4937
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4938
|
+
if ( __li_buf_acc[__j] == (double) (INFINITY) ) { __li_susp = 1; break; }
|
|
4939
|
+
}
|
|
4940
|
+
if ( __li_susp ) {
|
|
4941
|
+
int8_t __li_buf_seen[512];
|
|
4942
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
4943
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
4944
|
+
const double *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
4945
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4946
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
4947
|
+
}
|
|
4948
|
+
}
|
|
4949
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4950
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
4951
|
+
}
|
|
4952
|
+
}
|
|
4953
|
+
}
|
|
4810
4954
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4811
4955
|
op[__li_o * __li_INNER + __li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
4812
4956
|
}
|
|
@@ -4886,6 +5030,25 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
4886
5030
|
(void) v;
|
|
4887
5031
|
}
|
|
4888
5032
|
}
|
|
5033
|
+
{
|
|
5034
|
+
int __li_susp = 0;
|
|
5035
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5036
|
+
if ( __li_buf_acc[__j] == (double) (INFINITY) ) { __li_susp = 1; break; }
|
|
5037
|
+
}
|
|
5038
|
+
if ( __li_susp ) {
|
|
5039
|
+
int8_t __li_buf_seen[512];
|
|
5040
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
5041
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5042
|
+
const double *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5043
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5044
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
5045
|
+
}
|
|
5046
|
+
}
|
|
5047
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5048
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
5049
|
+
}
|
|
5050
|
+
}
|
|
5051
|
+
}
|
|
4889
5052
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
4890
5053
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
4891
5054
|
}
|
|
@@ -5011,6 +5174,25 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5011
5174
|
(void) v;
|
|
5012
5175
|
}
|
|
5013
5176
|
}
|
|
5177
|
+
{
|
|
5178
|
+
int __li_susp = 0;
|
|
5179
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5180
|
+
if ( __li_buf_acc[__j] == (double) (INFINITY) ) { __li_susp = 1; break; }
|
|
5181
|
+
}
|
|
5182
|
+
if ( __li_susp ) {
|
|
5183
|
+
int8_t __li_buf_seen[512];
|
|
5184
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
5185
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5186
|
+
const double *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
5187
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5188
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
5189
|
+
}
|
|
5190
|
+
}
|
|
5191
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5192
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
5193
|
+
}
|
|
5194
|
+
}
|
|
5195
|
+
}
|
|
5014
5196
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5015
5197
|
__li_op_k[__li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
5016
5198
|
}
|
|
@@ -5041,8 +5223,8 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5041
5223
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
5042
5224
|
tile reduce + write maintained.
|
|
5043
5225
|
|
|
5044
|
-
Performance characteristic (deliver-via-view
|
|
5045
|
-
|
|
5226
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
5227
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
5046
5228
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
5047
5229
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
5048
5230
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -5171,6 +5353,25 @@ min_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5171
5353
|
(void) v;
|
|
5172
5354
|
}
|
|
5173
5355
|
}
|
|
5356
|
+
{
|
|
5357
|
+
int __li_susp = 0;
|
|
5358
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5359
|
+
if ( __li_buf_acc[__j] == (double) (INFINITY) ) { __li_susp = 1; break; }
|
|
5360
|
+
}
|
|
5361
|
+
if ( __li_susp ) {
|
|
5362
|
+
int8_t __li_buf_seen[512];
|
|
5363
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
5364
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
5365
|
+
const double *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
5366
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5367
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
5368
|
+
}
|
|
5369
|
+
}
|
|
5370
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5371
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
5372
|
+
}
|
|
5373
|
+
}
|
|
5374
|
+
}
|
|
5174
5375
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5175
5376
|
__li_op_k[__li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
5176
5377
|
}
|
|
@@ -5208,6 +5409,16 @@ ca_size_t out_i = 0;
|
|
|
5208
5409
|
#define __min_f64_EXPR(__x) ((double)(__x))
|
|
5209
5410
|
CA_SLAB_REDUCE_MIN8_EX(double, double, st, p, m, acc, INFINITY, __min_f64_EXPR, masked_cnt);
|
|
5210
5411
|
#undef __min_f64_EXPR
|
|
5412
|
+
/* all-NaN fix-up (all_nan_result: :nan). */
|
|
5413
|
+
if ( (acc == (INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
5414
|
+
int64_t __anf_seen = 0;
|
|
5415
|
+
ca_size_t __anf_mc = 0;
|
|
5416
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
5417
|
+
(void) __anf_mc;
|
|
5418
|
+
if ( ! __anf_seen ) {
|
|
5419
|
+
acc = (double) NAN;
|
|
5420
|
+
}
|
|
5421
|
+
}
|
|
5211
5422
|
if ( (min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count) ) {
|
|
5212
5423
|
if ( ! op_mask ) {
|
|
5213
5424
|
ca_create_mask(co);
|
|
@@ -5326,6 +5537,7 @@ min_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5326
5537
|
(void) v;
|
|
5327
5538
|
}
|
|
5328
5539
|
}
|
|
5540
|
+
|
|
5329
5541
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5330
5542
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
5331
5543
|
}
|
|
@@ -5405,6 +5617,7 @@ min_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5405
5617
|
(void) v;
|
|
5406
5618
|
}
|
|
5407
5619
|
}
|
|
5620
|
+
|
|
5408
5621
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5409
5622
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
5410
5623
|
}
|
|
@@ -5530,6 +5743,7 @@ min_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5530
5743
|
(void) v;
|
|
5531
5744
|
}
|
|
5532
5745
|
}
|
|
5746
|
+
|
|
5533
5747
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5534
5748
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
5535
5749
|
}
|
|
@@ -5560,8 +5774,8 @@ min_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5560
5774
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
5561
5775
|
tile reduce + write maintained.
|
|
5562
5776
|
|
|
5563
|
-
Performance characteristic (deliver-via-view
|
|
5564
|
-
|
|
5777
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
5778
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
5565
5779
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
5566
5780
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
5567
5781
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -5690,6 +5904,7 @@ min_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
5690
5904
|
(void) v;
|
|
5691
5905
|
}
|
|
5692
5906
|
}
|
|
5907
|
+
|
|
5693
5908
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5694
5909
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
5695
5910
|
}
|
|
@@ -5776,7 +5991,7 @@ min_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
5776
5991
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
5777
5992
|
(void) masked_cnt;
|
|
5778
5993
|
ca_lazy_arena_enter();
|
|
5779
|
-
VALUE *__chunk = (VALUE *)
|
|
5994
|
+
VALUE *__chunk = (VALUE *) ca_lazy_arena_acquire_object(__chunk_elems);
|
|
5780
5995
|
while ( __outer_off < __outer ) {
|
|
5781
5996
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
5782
5997
|
ca_size_t __n = __r * __inner;
|
|
@@ -5845,6 +6060,7 @@ min_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
5845
6060
|
(void) v;
|
|
5846
6061
|
}
|
|
5847
6062
|
}
|
|
6063
|
+
|
|
5848
6064
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5849
6065
|
op[__li_o * __li_INNER + __li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
5850
6066
|
}
|
|
@@ -5924,6 +6140,7 @@ min_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
5924
6140
|
(void) v;
|
|
5925
6141
|
}
|
|
5926
6142
|
}
|
|
6143
|
+
|
|
5927
6144
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
5928
6145
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
5929
6146
|
}
|
|
@@ -6049,6 +6266,7 @@ min_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
6049
6266
|
(void) v;
|
|
6050
6267
|
}
|
|
6051
6268
|
}
|
|
6269
|
+
|
|
6052
6270
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6053
6271
|
__li_op_k[__li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
6054
6272
|
}
|
|
@@ -6079,8 +6297,8 @@ min_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
6079
6297
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
6080
6298
|
tile reduce + write maintained.
|
|
6081
6299
|
|
|
6082
|
-
Performance characteristic (deliver-via-view
|
|
6083
|
-
|
|
6300
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
6301
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
6084
6302
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
6085
6303
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
6086
6304
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -6209,6 +6427,7 @@ min_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
6209
6427
|
(void) v;
|
|
6210
6428
|
}
|
|
6211
6429
|
}
|
|
6430
|
+
|
|
6212
6431
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6213
6432
|
__li_op_k[__li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
6214
6433
|
}
|
|
@@ -6572,6 +6791,7 @@ max_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
6572
6791
|
(void) v;
|
|
6573
6792
|
}
|
|
6574
6793
|
}
|
|
6794
|
+
|
|
6575
6795
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6576
6796
|
op[__li_o * __li_INNER + __li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
6577
6797
|
}
|
|
@@ -6651,6 +6871,7 @@ max_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
6651
6871
|
(void) v;
|
|
6652
6872
|
}
|
|
6653
6873
|
}
|
|
6874
|
+
|
|
6654
6875
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6655
6876
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
6656
6877
|
}
|
|
@@ -6776,6 +6997,7 @@ max_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
6776
6997
|
(void) v;
|
|
6777
6998
|
}
|
|
6778
6999
|
}
|
|
7000
|
+
|
|
6779
7001
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6780
7002
|
__li_op_k[__li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
6781
7003
|
}
|
|
@@ -6806,8 +7028,8 @@ max_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
6806
7028
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
6807
7029
|
tile reduce + write maintained.
|
|
6808
7030
|
|
|
6809
|
-
Performance characteristic (deliver-via-view
|
|
6810
|
-
|
|
7031
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
7032
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
6811
7033
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
6812
7034
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
6813
7035
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -6936,6 +7158,7 @@ max_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
6936
7158
|
(void) v;
|
|
6937
7159
|
}
|
|
6938
7160
|
}
|
|
7161
|
+
|
|
6939
7162
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
6940
7163
|
__li_op_k[__li_tile + __j] = (int8_t) (__li_buf_acc[__j]);
|
|
6941
7164
|
}
|
|
@@ -7091,6 +7314,7 @@ max_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
7091
7314
|
(void) v;
|
|
7092
7315
|
}
|
|
7093
7316
|
}
|
|
7317
|
+
|
|
7094
7318
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7095
7319
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
7096
7320
|
}
|
|
@@ -7170,6 +7394,7 @@ max_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
7170
7394
|
(void) v;
|
|
7171
7395
|
}
|
|
7172
7396
|
}
|
|
7397
|
+
|
|
7173
7398
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7174
7399
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
7175
7400
|
}
|
|
@@ -7295,6 +7520,7 @@ max_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
7295
7520
|
(void) v;
|
|
7296
7521
|
}
|
|
7297
7522
|
}
|
|
7523
|
+
|
|
7298
7524
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7299
7525
|
__li_op_k[__li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
7300
7526
|
}
|
|
@@ -7325,8 +7551,8 @@ max_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
7325
7551
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
7326
7552
|
tile reduce + write maintained.
|
|
7327
7553
|
|
|
7328
|
-
Performance characteristic (deliver-via-view
|
|
7329
|
-
|
|
7554
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
7555
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
7330
7556
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
7331
7557
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
7332
7558
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -7455,6 +7681,7 @@ max_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int k
|
|
|
7455
7681
|
(void) v;
|
|
7456
7682
|
}
|
|
7457
7683
|
}
|
|
7684
|
+
|
|
7458
7685
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7459
7686
|
__li_op_k[__li_tile + __j] = (uint8_t) (__li_buf_acc[__j]);
|
|
7460
7687
|
}
|
|
@@ -7610,6 +7837,7 @@ max_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
7610
7837
|
(void) v;
|
|
7611
7838
|
}
|
|
7612
7839
|
}
|
|
7840
|
+
|
|
7613
7841
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7614
7842
|
op[__li_o * __li_INNER + __li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
7615
7843
|
}
|
|
@@ -7689,6 +7917,7 @@ max_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
7689
7917
|
(void) v;
|
|
7690
7918
|
}
|
|
7691
7919
|
}
|
|
7920
|
+
|
|
7692
7921
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7693
7922
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
7694
7923
|
}
|
|
@@ -7814,6 +8043,7 @@ max_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
7814
8043
|
(void) v;
|
|
7815
8044
|
}
|
|
7816
8045
|
}
|
|
8046
|
+
|
|
7817
8047
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7818
8048
|
__li_op_k[__li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
7819
8049
|
}
|
|
@@ -7844,8 +8074,8 @@ max_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
7844
8074
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
7845
8075
|
tile reduce + write maintained.
|
|
7846
8076
|
|
|
7847
|
-
Performance characteristic (deliver-via-view
|
|
7848
|
-
|
|
8077
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
8078
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
7849
8079
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
7850
8080
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
7851
8081
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -7974,6 +8204,7 @@ max_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
7974
8204
|
(void) v;
|
|
7975
8205
|
}
|
|
7976
8206
|
}
|
|
8207
|
+
|
|
7977
8208
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
7978
8209
|
__li_op_k[__li_tile + __j] = (int16_t) (__li_buf_acc[__j]);
|
|
7979
8210
|
}
|
|
@@ -8129,6 +8360,7 @@ max_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8129
8360
|
(void) v;
|
|
8130
8361
|
}
|
|
8131
8362
|
}
|
|
8363
|
+
|
|
8132
8364
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8133
8365
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
8134
8366
|
}
|
|
@@ -8208,6 +8440,7 @@ max_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8208
8440
|
(void) v;
|
|
8209
8441
|
}
|
|
8210
8442
|
}
|
|
8443
|
+
|
|
8211
8444
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8212
8445
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
8213
8446
|
}
|
|
@@ -8333,6 +8566,7 @@ max_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8333
8566
|
(void) v;
|
|
8334
8567
|
}
|
|
8335
8568
|
}
|
|
8569
|
+
|
|
8336
8570
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8337
8571
|
__li_op_k[__li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
8338
8572
|
}
|
|
@@ -8363,8 +8597,8 @@ max_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8363
8597
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
8364
8598
|
tile reduce + write maintained.
|
|
8365
8599
|
|
|
8366
|
-
Performance characteristic (deliver-via-view
|
|
8367
|
-
|
|
8600
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
8601
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
8368
8602
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
8369
8603
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
8370
8604
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -8493,6 +8727,7 @@ max_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8493
8727
|
(void) v;
|
|
8494
8728
|
}
|
|
8495
8729
|
}
|
|
8730
|
+
|
|
8496
8731
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8497
8732
|
__li_op_k[__li_tile + __j] = (uint16_t) (__li_buf_acc[__j]);
|
|
8498
8733
|
}
|
|
@@ -8648,6 +8883,7 @@ max_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8648
8883
|
(void) v;
|
|
8649
8884
|
}
|
|
8650
8885
|
}
|
|
8886
|
+
|
|
8651
8887
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8652
8888
|
op[__li_o * __li_INNER + __li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
8653
8889
|
}
|
|
@@ -8727,6 +8963,7 @@ max_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8727
8963
|
(void) v;
|
|
8728
8964
|
}
|
|
8729
8965
|
}
|
|
8966
|
+
|
|
8730
8967
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8731
8968
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
8732
8969
|
}
|
|
@@ -8852,6 +9089,7 @@ max_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8852
9089
|
(void) v;
|
|
8853
9090
|
}
|
|
8854
9091
|
}
|
|
9092
|
+
|
|
8855
9093
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
8856
9094
|
__li_op_k[__li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
8857
9095
|
}
|
|
@@ -8882,8 +9120,8 @@ max_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
8882
9120
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
8883
9121
|
tile reduce + write maintained.
|
|
8884
9122
|
|
|
8885
|
-
Performance characteristic (deliver-via-view
|
|
8886
|
-
|
|
9123
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
9124
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
8887
9125
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
8888
9126
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
8889
9127
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -9012,6 +9250,7 @@ max_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9012
9250
|
(void) v;
|
|
9013
9251
|
}
|
|
9014
9252
|
}
|
|
9253
|
+
|
|
9015
9254
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9016
9255
|
__li_op_k[__li_tile + __j] = (int32_t) (__li_buf_acc[__j]);
|
|
9017
9256
|
}
|
|
@@ -9167,6 +9406,7 @@ max_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9167
9406
|
(void) v;
|
|
9168
9407
|
}
|
|
9169
9408
|
}
|
|
9409
|
+
|
|
9170
9410
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9171
9411
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
9172
9412
|
}
|
|
@@ -9246,6 +9486,7 @@ max_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9246
9486
|
(void) v;
|
|
9247
9487
|
}
|
|
9248
9488
|
}
|
|
9489
|
+
|
|
9249
9490
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9250
9491
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
9251
9492
|
}
|
|
@@ -9371,6 +9612,7 @@ max_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9371
9612
|
(void) v;
|
|
9372
9613
|
}
|
|
9373
9614
|
}
|
|
9615
|
+
|
|
9374
9616
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9375
9617
|
__li_op_k[__li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
9376
9618
|
}
|
|
@@ -9401,8 +9643,8 @@ max_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9401
9643
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
9402
9644
|
tile reduce + write maintained.
|
|
9403
9645
|
|
|
9404
|
-
Performance characteristic (deliver-via-view
|
|
9405
|
-
|
|
9646
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
9647
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
9406
9648
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
9407
9649
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
9408
9650
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -9531,6 +9773,7 @@ max_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9531
9773
|
(void) v;
|
|
9532
9774
|
}
|
|
9533
9775
|
}
|
|
9776
|
+
|
|
9534
9777
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9535
9778
|
__li_op_k[__li_tile + __j] = (uint32_t) (__li_buf_acc[__j]);
|
|
9536
9779
|
}
|
|
@@ -9686,6 +9929,7 @@ max_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9686
9929
|
(void) v;
|
|
9687
9930
|
}
|
|
9688
9931
|
}
|
|
9932
|
+
|
|
9689
9933
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9690
9934
|
op[__li_o * __li_INNER + __li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
9691
9935
|
}
|
|
@@ -9765,6 +10009,7 @@ max_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9765
10009
|
(void) v;
|
|
9766
10010
|
}
|
|
9767
10011
|
}
|
|
10012
|
+
|
|
9768
10013
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9769
10014
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
9770
10015
|
}
|
|
@@ -9890,6 +10135,7 @@ max_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9890
10135
|
(void) v;
|
|
9891
10136
|
}
|
|
9892
10137
|
}
|
|
10138
|
+
|
|
9893
10139
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
9894
10140
|
__li_op_k[__li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
9895
10141
|
}
|
|
@@ -9920,8 +10166,8 @@ max_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
9920
10166
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
9921
10167
|
tile reduce + write maintained.
|
|
9922
10168
|
|
|
9923
|
-
Performance characteristic (deliver-via-view
|
|
9924
|
-
|
|
10169
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
10170
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
9925
10171
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
9926
10172
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
9927
10173
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -10050,6 +10296,7 @@ max_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10050
10296
|
(void) v;
|
|
10051
10297
|
}
|
|
10052
10298
|
}
|
|
10299
|
+
|
|
10053
10300
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10054
10301
|
__li_op_k[__li_tile + __j] = (int64_t) (__li_buf_acc[__j]);
|
|
10055
10302
|
}
|
|
@@ -10205,6 +10452,7 @@ max_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10205
10452
|
(void) v;
|
|
10206
10453
|
}
|
|
10207
10454
|
}
|
|
10455
|
+
|
|
10208
10456
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10209
10457
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
10210
10458
|
}
|
|
@@ -10284,6 +10532,7 @@ max_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10284
10532
|
(void) v;
|
|
10285
10533
|
}
|
|
10286
10534
|
}
|
|
10535
|
+
|
|
10287
10536
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10288
10537
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
10289
10538
|
}
|
|
@@ -10409,6 +10658,7 @@ max_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10409
10658
|
(void) v;
|
|
10410
10659
|
}
|
|
10411
10660
|
}
|
|
10661
|
+
|
|
10412
10662
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10413
10663
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
10414
10664
|
}
|
|
@@ -10439,8 +10689,8 @@ max_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10439
10689
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
10440
10690
|
tile reduce + write maintained.
|
|
10441
10691
|
|
|
10442
|
-
Performance characteristic (deliver-via-view
|
|
10443
|
-
|
|
10692
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
10693
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
10444
10694
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
10445
10695
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
10446
10696
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -10569,6 +10819,7 @@ max_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10569
10819
|
(void) v;
|
|
10570
10820
|
}
|
|
10571
10821
|
}
|
|
10822
|
+
|
|
10572
10823
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10573
10824
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
10574
10825
|
}
|
|
@@ -10652,6 +10903,7 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10652
10903
|
ca_size_t __outer_off = 0;
|
|
10653
10904
|
float acc;
|
|
10654
10905
|
acc = (-INFINITY);
|
|
10906
|
+
int64_t __anf_seen = 0;
|
|
10655
10907
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
10656
10908
|
(void) masked_cnt;
|
|
10657
10909
|
ca_lazy_arena_enter();
|
|
@@ -10666,11 +10918,13 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10666
10918
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
10667
10919
|
float v = __chunk[__i];
|
|
10668
10920
|
acc = (v > acc) ? v : acc;
|
|
10921
|
+
__anf_seen |= (v == v);
|
|
10669
10922
|
}
|
|
10670
10923
|
__outer_off += __r;
|
|
10671
10924
|
}
|
|
10672
10925
|
ca_lazy_arena_release(__chunk);
|
|
10673
10926
|
ca_lazy_arena_exit();
|
|
10927
|
+
if ( ! __anf_seen && ca->elements > 0 ) acc = (float) NAN;
|
|
10674
10928
|
return rb_float_new((float) (acc));
|
|
10675
10929
|
}
|
|
10676
10930
|
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
@@ -10724,6 +10978,25 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10724
10978
|
(void) v;
|
|
10725
10979
|
}
|
|
10726
10980
|
}
|
|
10981
|
+
{
|
|
10982
|
+
int __li_susp = 0;
|
|
10983
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10984
|
+
if ( __li_buf_acc[__j] == (float) (-INFINITY) ) { __li_susp = 1; break; }
|
|
10985
|
+
}
|
|
10986
|
+
if ( __li_susp ) {
|
|
10987
|
+
int8_t __li_buf_seen[512];
|
|
10988
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
10989
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
10990
|
+
const float *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
10991
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10992
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
10993
|
+
}
|
|
10994
|
+
}
|
|
10995
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10996
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
10997
|
+
}
|
|
10998
|
+
}
|
|
10999
|
+
}
|
|
10727
11000
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10728
11001
|
op[__li_o * __li_INNER + __li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
10729
11002
|
}
|
|
@@ -10803,6 +11076,25 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10803
11076
|
(void) v;
|
|
10804
11077
|
}
|
|
10805
11078
|
}
|
|
11079
|
+
{
|
|
11080
|
+
int __li_susp = 0;
|
|
11081
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11082
|
+
if ( __li_buf_acc[__j] == (float) (-INFINITY) ) { __li_susp = 1; break; }
|
|
11083
|
+
}
|
|
11084
|
+
if ( __li_susp ) {
|
|
11085
|
+
int8_t __li_buf_seen[512];
|
|
11086
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
11087
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
11088
|
+
const float *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
11089
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11090
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
11091
|
+
}
|
|
11092
|
+
}
|
|
11093
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11094
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
11095
|
+
}
|
|
11096
|
+
}
|
|
11097
|
+
}
|
|
10806
11098
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10807
11099
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
10808
11100
|
}
|
|
@@ -10928,6 +11220,25 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10928
11220
|
(void) v;
|
|
10929
11221
|
}
|
|
10930
11222
|
}
|
|
11223
|
+
{
|
|
11224
|
+
int __li_susp = 0;
|
|
11225
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11226
|
+
if ( __li_buf_acc[__j] == (float) (-INFINITY) ) { __li_susp = 1; break; }
|
|
11227
|
+
}
|
|
11228
|
+
if ( __li_susp ) {
|
|
11229
|
+
int8_t __li_buf_seen[512];
|
|
11230
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
11231
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
11232
|
+
const float *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
11233
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11234
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
11235
|
+
}
|
|
11236
|
+
}
|
|
11237
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11238
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
11239
|
+
}
|
|
11240
|
+
}
|
|
11241
|
+
}
|
|
10931
11242
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
10932
11243
|
__li_op_k[__li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
10933
11244
|
}
|
|
@@ -10958,8 +11269,8 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
10958
11269
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
10959
11270
|
tile reduce + write maintained.
|
|
10960
11271
|
|
|
10961
|
-
Performance characteristic (deliver-via-view
|
|
10962
|
-
|
|
11272
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
11273
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
10963
11274
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
10964
11275
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
10965
11276
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -11088,6 +11399,25 @@ max_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11088
11399
|
(void) v;
|
|
11089
11400
|
}
|
|
11090
11401
|
}
|
|
11402
|
+
{
|
|
11403
|
+
int __li_susp = 0;
|
|
11404
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11405
|
+
if ( __li_buf_acc[__j] == (float) (-INFINITY) ) { __li_susp = 1; break; }
|
|
11406
|
+
}
|
|
11407
|
+
if ( __li_susp ) {
|
|
11408
|
+
int8_t __li_buf_seen[512];
|
|
11409
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
11410
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
11411
|
+
const float *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
11412
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11413
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
11414
|
+
}
|
|
11415
|
+
}
|
|
11416
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11417
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (float) NAN;
|
|
11418
|
+
}
|
|
11419
|
+
}
|
|
11420
|
+
}
|
|
11091
11421
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11092
11422
|
__li_op_k[__li_tile + __j] = (float) (__li_buf_acc[__j]);
|
|
11093
11423
|
}
|
|
@@ -11125,6 +11455,16 @@ ca_size_t out_i = 0;
|
|
|
11125
11455
|
#define __max_f32_EXPR(__x) ((float)(__x))
|
|
11126
11456
|
CA_SLAB_REDUCE_MAX8_EX(float, float, st, p, m, acc, -INFINITY, __max_f32_EXPR, masked_cnt);
|
|
11127
11457
|
#undef __max_f32_EXPR
|
|
11458
|
+
/* all-NaN fix-up (all_nan_result: :nan). */
|
|
11459
|
+
if ( (acc == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
11460
|
+
int64_t __anf_seen = 0;
|
|
11461
|
+
ca_size_t __anf_mc = 0;
|
|
11462
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
11463
|
+
(void) __anf_mc;
|
|
11464
|
+
if ( ! __anf_seen ) {
|
|
11465
|
+
acc = (float) NAN;
|
|
11466
|
+
}
|
|
11467
|
+
}
|
|
11128
11468
|
if ( (min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count) ) {
|
|
11129
11469
|
if ( ! op_mask ) {
|
|
11130
11470
|
ca_create_mask(co);
|
|
@@ -11171,6 +11511,7 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11171
11511
|
ca_size_t __outer_off = 0;
|
|
11172
11512
|
double acc;
|
|
11173
11513
|
acc = (-INFINITY);
|
|
11514
|
+
int64_t __anf_seen = 0;
|
|
11174
11515
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
11175
11516
|
(void) masked_cnt;
|
|
11176
11517
|
ca_lazy_arena_enter();
|
|
@@ -11185,11 +11526,13 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11185
11526
|
for ( __i = 0; __i < __n; __i++ ) {
|
|
11186
11527
|
double v = __chunk[__i];
|
|
11187
11528
|
acc = (v > acc) ? v : acc;
|
|
11529
|
+
__anf_seen |= (v == v);
|
|
11188
11530
|
}
|
|
11189
11531
|
__outer_off += __r;
|
|
11190
11532
|
}
|
|
11191
11533
|
ca_lazy_arena_release(__chunk);
|
|
11192
11534
|
ca_lazy_arena_exit();
|
|
11535
|
+
if ( ! __anf_seen && ca->elements > 0 ) acc = (double) NAN;
|
|
11193
11536
|
return rb_float_new((double) (acc));
|
|
11194
11537
|
}
|
|
11195
11538
|
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
@@ -11243,6 +11586,25 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11243
11586
|
(void) v;
|
|
11244
11587
|
}
|
|
11245
11588
|
}
|
|
11589
|
+
{
|
|
11590
|
+
int __li_susp = 0;
|
|
11591
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11592
|
+
if ( __li_buf_acc[__j] == (double) (-INFINITY) ) { __li_susp = 1; break; }
|
|
11593
|
+
}
|
|
11594
|
+
if ( __li_susp ) {
|
|
11595
|
+
int8_t __li_buf_seen[512];
|
|
11596
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
11597
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
11598
|
+
const double *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
11599
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11600
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
11601
|
+
}
|
|
11602
|
+
}
|
|
11603
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11604
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
11605
|
+
}
|
|
11606
|
+
}
|
|
11607
|
+
}
|
|
11246
11608
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11247
11609
|
op[__li_o * __li_INNER + __li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
11248
11610
|
}
|
|
@@ -11322,6 +11684,25 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11322
11684
|
(void) v;
|
|
11323
11685
|
}
|
|
11324
11686
|
}
|
|
11687
|
+
{
|
|
11688
|
+
int __li_susp = 0;
|
|
11689
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11690
|
+
if ( __li_buf_acc[__j] == (double) (-INFINITY) ) { __li_susp = 1; break; }
|
|
11691
|
+
}
|
|
11692
|
+
if ( __li_susp ) {
|
|
11693
|
+
int8_t __li_buf_seen[512];
|
|
11694
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
11695
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
11696
|
+
const double *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
11697
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11698
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
11699
|
+
}
|
|
11700
|
+
}
|
|
11701
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11702
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
11703
|
+
}
|
|
11704
|
+
}
|
|
11705
|
+
}
|
|
11325
11706
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11326
11707
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
11327
11708
|
}
|
|
@@ -11447,6 +11828,25 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11447
11828
|
(void) v;
|
|
11448
11829
|
}
|
|
11449
11830
|
}
|
|
11831
|
+
{
|
|
11832
|
+
int __li_susp = 0;
|
|
11833
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11834
|
+
if ( __li_buf_acc[__j] == (double) (-INFINITY) ) { __li_susp = 1; break; }
|
|
11835
|
+
}
|
|
11836
|
+
if ( __li_susp ) {
|
|
11837
|
+
int8_t __li_buf_seen[512];
|
|
11838
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
11839
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
11840
|
+
const double *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
11841
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11842
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
11843
|
+
}
|
|
11844
|
+
}
|
|
11845
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11846
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
11847
|
+
}
|
|
11848
|
+
}
|
|
11849
|
+
}
|
|
11450
11850
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11451
11851
|
__li_op_k[__li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
11452
11852
|
}
|
|
@@ -11477,8 +11877,8 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11477
11877
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
11478
11878
|
tile reduce + write maintained.
|
|
11479
11879
|
|
|
11480
|
-
Performance characteristic (deliver-via-view
|
|
11481
|
-
|
|
11880
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
11881
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
11482
11882
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
11483
11883
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
11484
11884
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -11607,6 +12007,25 @@ max_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11607
12007
|
(void) v;
|
|
11608
12008
|
}
|
|
11609
12009
|
}
|
|
12010
|
+
{
|
|
12011
|
+
int __li_susp = 0;
|
|
12012
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12013
|
+
if ( __li_buf_acc[__j] == (double) (-INFINITY) ) { __li_susp = 1; break; }
|
|
12014
|
+
}
|
|
12015
|
+
if ( __li_susp ) {
|
|
12016
|
+
int8_t __li_buf_seen[512];
|
|
12017
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) __li_buf_seen[__j] = 0;
|
|
12018
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
12019
|
+
const double *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
12020
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12021
|
+
__li_buf_seen[__j] |= (__li_row[__j] == __li_row[__j]);
|
|
12022
|
+
}
|
|
12023
|
+
}
|
|
12024
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12025
|
+
if ( ! __li_buf_seen[__j] ) __li_buf_acc[__j] = (double) NAN;
|
|
12026
|
+
}
|
|
12027
|
+
}
|
|
12028
|
+
}
|
|
11610
12029
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11611
12030
|
__li_op_k[__li_tile + __j] = (double) (__li_buf_acc[__j]);
|
|
11612
12031
|
}
|
|
@@ -11644,6 +12063,16 @@ ca_size_t out_i = 0;
|
|
|
11644
12063
|
#define __max_f64_EXPR(__x) ((double)(__x))
|
|
11645
12064
|
CA_SLAB_REDUCE_MAX8_EX(double, double, st, p, m, acc, -INFINITY, __max_f64_EXPR, masked_cnt);
|
|
11646
12065
|
#undef __max_f64_EXPR
|
|
12066
|
+
/* all-NaN fix-up (all_nan_result: :nan). */
|
|
12067
|
+
if ( (acc == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
12068
|
+
int64_t __anf_seen = 0;
|
|
12069
|
+
ca_size_t __anf_mc = 0;
|
|
12070
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
12071
|
+
(void) __anf_mc;
|
|
12072
|
+
if ( ! __anf_seen ) {
|
|
12073
|
+
acc = (double) NAN;
|
|
12074
|
+
}
|
|
12075
|
+
}
|
|
11647
12076
|
if ( (min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count) ) {
|
|
11648
12077
|
if ( ! op_mask ) {
|
|
11649
12078
|
ca_create_mask(co);
|
|
@@ -11762,6 +12191,7 @@ max_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11762
12191
|
(void) v;
|
|
11763
12192
|
}
|
|
11764
12193
|
}
|
|
12194
|
+
|
|
11765
12195
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11766
12196
|
op[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
11767
12197
|
}
|
|
@@ -11841,6 +12271,7 @@ max_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11841
12271
|
(void) v;
|
|
11842
12272
|
}
|
|
11843
12273
|
}
|
|
12274
|
+
|
|
11844
12275
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11845
12276
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
11846
12277
|
}
|
|
@@ -11966,6 +12397,7 @@ max_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11966
12397
|
(void) v;
|
|
11967
12398
|
}
|
|
11968
12399
|
}
|
|
12400
|
+
|
|
11969
12401
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
11970
12402
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
11971
12403
|
}
|
|
@@ -11996,8 +12428,8 @@ max_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
11996
12428
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
11997
12429
|
tile reduce + write maintained.
|
|
11998
12430
|
|
|
11999
|
-
Performance characteristic (deliver-via-view
|
|
12000
|
-
|
|
12431
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
12432
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
12001
12433
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
12002
12434
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
12003
12435
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -12126,6 +12558,7 @@ max_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int
|
|
|
12126
12558
|
(void) v;
|
|
12127
12559
|
}
|
|
12128
12560
|
}
|
|
12561
|
+
|
|
12129
12562
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12130
12563
|
__li_op_k[__li_tile + __j] = (uint64_t) (__li_buf_acc[__j]);
|
|
12131
12564
|
}
|
|
@@ -12212,7 +12645,7 @@ max_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
12212
12645
|
ca_size_t masked_cnt = 0; /* unused on streaming (no mask) */
|
|
12213
12646
|
(void) masked_cnt;
|
|
12214
12647
|
ca_lazy_arena_enter();
|
|
12215
|
-
VALUE *__chunk = (VALUE *)
|
|
12648
|
+
VALUE *__chunk = (VALUE *) ca_lazy_arena_acquire_object(__chunk_elems);
|
|
12216
12649
|
while ( __outer_off < __outer ) {
|
|
12217
12650
|
ca_size_t __r = (__outer - __outer_off < __rows) ? (__outer - __outer_off) : __rows;
|
|
12218
12651
|
ca_size_t __n = __r * __inner;
|
|
@@ -12281,6 +12714,7 @@ max_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
12281
12714
|
(void) v;
|
|
12282
12715
|
}
|
|
12283
12716
|
}
|
|
12717
|
+
|
|
12284
12718
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12285
12719
|
op[__li_o * __li_INNER + __li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
12286
12720
|
}
|
|
@@ -12360,6 +12794,7 @@ max_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
12360
12794
|
(void) v;
|
|
12361
12795
|
}
|
|
12362
12796
|
}
|
|
12797
|
+
|
|
12363
12798
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12364
12799
|
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
12365
12800
|
}
|
|
@@ -12485,6 +12920,7 @@ max_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
12485
12920
|
(void) v;
|
|
12486
12921
|
}
|
|
12487
12922
|
}
|
|
12923
|
+
|
|
12488
12924
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12489
12925
|
__li_op_k[__li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
12490
12926
|
}
|
|
@@ -12515,8 +12951,8 @@ max_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
12515
12951
|
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
12516
12952
|
tile reduce + write maintained.
|
|
12517
12953
|
|
|
12518
|
-
Performance characteristic (deliver-via-view
|
|
12519
|
-
|
|
12954
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
12955
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
12520
12956
|
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
12521
12957
|
= 2.38x slow. Root cause is the multi-parent data layout
|
|
12522
12958
|
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
@@ -12645,6 +13081,7 @@ max_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
12645
13081
|
(void) v;
|
|
12646
13082
|
}
|
|
12647
13083
|
}
|
|
13084
|
+
|
|
12648
13085
|
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
12649
13086
|
__li_op_k[__li_tile + __j] = (VALUE) (__li_buf_acc[__j]);
|
|
12650
13087
|
}
|
|
@@ -13376,6 +13813,17 @@ minmax_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
13376
13813
|
float hi = -INFINITY;
|
|
13377
13814
|
ca_size_t masked_cnt = 0;
|
|
13378
13815
|
CA_SLAB_REDUCE_T_EX(float, st, p, m, lo, INFINITY, if (v < lo) lo = v; if (v > hi) hi = v;, masked_cnt);
|
|
13816
|
+
/* all-NaN fix-up (all_nan_result: :nan). */
|
|
13817
|
+
if ( (lo == (INFINITY) && hi == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
13818
|
+
int64_t __anf_seen = 0;
|
|
13819
|
+
ca_size_t __anf_mc = 0;
|
|
13820
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
13821
|
+
(void) __anf_mc;
|
|
13822
|
+
if ( ! __anf_seen ) {
|
|
13823
|
+
lo = (float) NAN;
|
|
13824
|
+
hi = (float) NAN;
|
|
13825
|
+
}
|
|
13826
|
+
}
|
|
13379
13827
|
if ( (min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count) ) {
|
|
13380
13828
|
if ( ! op_mask_a ) {
|
|
13381
13829
|
ca_create_mask(co_a);
|
|
@@ -13431,6 +13879,17 @@ minmax_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, i
|
|
|
13431
13879
|
double hi = -INFINITY;
|
|
13432
13880
|
ca_size_t masked_cnt = 0;
|
|
13433
13881
|
CA_SLAB_REDUCE_T_EX(double, st, p, m, lo, INFINITY, if (v < lo) lo = v; if (v > hi) hi = v;, masked_cnt);
|
|
13882
|
+
/* all-NaN fix-up (all_nan_result: :nan). */
|
|
13883
|
+
if ( (lo == (INFINITY) && hi == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
13884
|
+
int64_t __anf_seen = 0;
|
|
13885
|
+
ca_size_t __anf_mc = 0;
|
|
13886
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
13887
|
+
(void) __anf_mc;
|
|
13888
|
+
if ( ! __anf_seen ) {
|
|
13889
|
+
lo = (double) NAN;
|
|
13890
|
+
hi = (double) NAN;
|
|
13891
|
+
}
|
|
13892
|
+
}
|
|
13434
13893
|
if ( (min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count) ) {
|
|
13435
13894
|
if ( ! op_mask_a ) {
|
|
13436
13895
|
ca_create_mask(co_a);
|
|
@@ -14020,7 +14479,18 @@ ca_size_t out_i = 0;
|
|
|
14020
14479
|
int64_t best_i = 0;
|
|
14021
14480
|
ca_size_t masked_cnt = 0;
|
|
14022
14481
|
CA_SLAB_REDUCE_T_EX(float, st, p, m, best_v, INFINITY, if (v < best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
14023
|
-
|
|
14482
|
+
int __anf_all_nan = 0;
|
|
14483
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
14484
|
+
if ( (best_v == (INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
14485
|
+
int64_t __anf_seen = 0;
|
|
14486
|
+
ca_size_t __anf_mc = 0;
|
|
14487
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
14488
|
+
(void) __anf_mc;
|
|
14489
|
+
if ( ! __anf_seen ) {
|
|
14490
|
+
__anf_all_nan = 1;
|
|
14491
|
+
}
|
|
14492
|
+
}
|
|
14493
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
14024
14494
|
if ( ! op_mask ) {
|
|
14025
14495
|
ca_create_mask(co);
|
|
14026
14496
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -14062,7 +14532,18 @@ ca_size_t out_i = 0;
|
|
|
14062
14532
|
int64_t best_i = 0;
|
|
14063
14533
|
ca_size_t masked_cnt = 0;
|
|
14064
14534
|
CA_SLAB_REDUCE_T_EX(double, st, p, m, best_v, INFINITY, if (v < best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
14065
|
-
|
|
14535
|
+
int __anf_all_nan = 0;
|
|
14536
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
14537
|
+
if ( (best_v == (INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
14538
|
+
int64_t __anf_seen = 0;
|
|
14539
|
+
ca_size_t __anf_mc = 0;
|
|
14540
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
14541
|
+
(void) __anf_mc;
|
|
14542
|
+
if ( ! __anf_seen ) {
|
|
14543
|
+
__anf_all_nan = 1;
|
|
14544
|
+
}
|
|
14545
|
+
}
|
|
14546
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
14066
14547
|
if ( ! op_mask ) {
|
|
14067
14548
|
ca_create_mask(co);
|
|
14068
14549
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -14707,7 +15188,18 @@ ca_size_t out_i = 0;
|
|
|
14707
15188
|
int64_t best_i = 0;
|
|
14708
15189
|
ca_size_t masked_cnt = 0;
|
|
14709
15190
|
CA_SLAB_REDUCE_T_EX(float, st, p, m, best_v, -INFINITY, if (v > best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
14710
|
-
|
|
15191
|
+
int __anf_all_nan = 0;
|
|
15192
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
15193
|
+
if ( (best_v == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
15194
|
+
int64_t __anf_seen = 0;
|
|
15195
|
+
ca_size_t __anf_mc = 0;
|
|
15196
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
15197
|
+
(void) __anf_mc;
|
|
15198
|
+
if ( ! __anf_seen ) {
|
|
15199
|
+
__anf_all_nan = 1;
|
|
15200
|
+
}
|
|
15201
|
+
}
|
|
15202
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
14711
15203
|
if ( ! op_mask ) {
|
|
14712
15204
|
ca_create_mask(co);
|
|
14713
15205
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -14749,7 +15241,18 @@ ca_size_t out_i = 0;
|
|
|
14749
15241
|
int64_t best_i = 0;
|
|
14750
15242
|
ca_size_t masked_cnt = 0;
|
|
14751
15243
|
CA_SLAB_REDUCE_T_EX(double, st, p, m, best_v, -INFINITY, if (v > best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
14752
|
-
|
|
15244
|
+
int __anf_all_nan = 0;
|
|
15245
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
15246
|
+
if ( (best_v == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
15247
|
+
int64_t __anf_seen = 0;
|
|
15248
|
+
ca_size_t __anf_mc = 0;
|
|
15249
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
15250
|
+
(void) __anf_mc;
|
|
15251
|
+
if ( ! __anf_seen ) {
|
|
15252
|
+
__anf_all_nan = 1;
|
|
15253
|
+
}
|
|
15254
|
+
}
|
|
15255
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
14753
15256
|
if ( ! op_mask ) {
|
|
14754
15257
|
ca_create_mask(co);
|
|
14755
15258
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -15640,7 +16143,18 @@ ca_size_t out_i = 0;
|
|
|
15640
16143
|
int64_t best_i = 0;
|
|
15641
16144
|
ca_size_t masked_cnt = 0;
|
|
15642
16145
|
CA_SLAB_REDUCE_T_EX(float, st, p, m, best_v, INFINITY, if (v < best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
15643
|
-
|
|
16146
|
+
int __anf_all_nan = 0;
|
|
16147
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
16148
|
+
if ( (best_v == (INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
16149
|
+
int64_t __anf_seen = 0;
|
|
16150
|
+
ca_size_t __anf_mc = 0;
|
|
16151
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
16152
|
+
(void) __anf_mc;
|
|
16153
|
+
if ( ! __anf_seen ) {
|
|
16154
|
+
__anf_all_nan = 1;
|
|
16155
|
+
}
|
|
16156
|
+
}
|
|
16157
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
15644
16158
|
if ( ! op_mask ) {
|
|
15645
16159
|
ca_create_mask(co);
|
|
15646
16160
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -15710,7 +16224,18 @@ ca_size_t out_i = 0;
|
|
|
15710
16224
|
int64_t best_i = 0;
|
|
15711
16225
|
ca_size_t masked_cnt = 0;
|
|
15712
16226
|
CA_SLAB_REDUCE_T_EX(double, st, p, m, best_v, INFINITY, if (v < best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
15713
|
-
|
|
16227
|
+
int __anf_all_nan = 0;
|
|
16228
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
16229
|
+
if ( (best_v == (INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
16230
|
+
int64_t __anf_seen = 0;
|
|
16231
|
+
ca_size_t __anf_mc = 0;
|
|
16232
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
16233
|
+
(void) __anf_mc;
|
|
16234
|
+
if ( ! __anf_seen ) {
|
|
16235
|
+
__anf_all_nan = 1;
|
|
16236
|
+
}
|
|
16237
|
+
}
|
|
16238
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
15714
16239
|
if ( ! op_mask ) {
|
|
15715
16240
|
ca_create_mask(co);
|
|
15716
16241
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -16587,7 +17112,18 @@ ca_size_t out_i = 0;
|
|
|
16587
17112
|
int64_t best_i = 0;
|
|
16588
17113
|
ca_size_t masked_cnt = 0;
|
|
16589
17114
|
CA_SLAB_REDUCE_T_EX(float, st, p, m, best_v, -INFINITY, if (v > best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
16590
|
-
|
|
17115
|
+
int __anf_all_nan = 0;
|
|
17116
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
17117
|
+
if ( (best_v == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
17118
|
+
int64_t __anf_seen = 0;
|
|
17119
|
+
ca_size_t __anf_mc = 0;
|
|
17120
|
+
CA_SLAB_REDUCE_T_EX(float, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
17121
|
+
(void) __anf_mc;
|
|
17122
|
+
if ( ! __anf_seen ) {
|
|
17123
|
+
__anf_all_nan = 1;
|
|
17124
|
+
}
|
|
17125
|
+
}
|
|
17126
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
16591
17127
|
if ( ! op_mask ) {
|
|
16592
17128
|
ca_create_mask(co);
|
|
16593
17129
|
op_mask = (boolean8_t *) co->mask->ptr;
|
|
@@ -16657,7 +17193,18 @@ ca_size_t out_i = 0;
|
|
|
16657
17193
|
int64_t best_i = 0;
|
|
16658
17194
|
ca_size_t masked_cnt = 0;
|
|
16659
17195
|
CA_SLAB_REDUCE_T_EX(double, st, p, m, best_v, -INFINITY, if (v > best_v) { best_v = v; best_i = idx; }, masked_cnt);
|
|
16660
|
-
|
|
17196
|
+
int __anf_all_nan = 0;
|
|
17197
|
+
/* all-NaN fix-up (all_nan_result: :undef). */
|
|
17198
|
+
if ( (best_v == (-INFINITY)) && masked_cnt < st.slab_elements ) {
|
|
17199
|
+
int64_t __anf_seen = 0;
|
|
17200
|
+
ca_size_t __anf_mc = 0;
|
|
17201
|
+
CA_SLAB_REDUCE_T_EX(double, st, p, m, __anf_seen, 0, __anf_seen |= (v == v), __anf_mc);
|
|
17202
|
+
(void) __anf_mc;
|
|
17203
|
+
if ( ! __anf_seen ) {
|
|
17204
|
+
__anf_all_nan = 1;
|
|
17205
|
+
}
|
|
17206
|
+
}
|
|
17207
|
+
if ( ((min_count < 0 ? masked_cnt == st.slab_elements : st.slab_elements - masked_cnt < min_count)) || __anf_all_nan ) {
|
|
16661
17208
|
if ( ! op_mask ) {
|
|
16662
17209
|
ca_create_mask(co);
|
|
16663
17210
|
op_mask = (boolean8_t *) co->mask->ptr;
|