carray 3.0.5 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +233 -0
- data/README.md +1 -1
- data/carray.gemspec +3 -2
- data/ext/ca_axis_dispatch.c +6 -0
- data/ext/ca_axis_group.c +1 -1
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.c +116 -231
- data/ext/ca_kernel_iterator.h +410 -254
- data/ext/ca_obj_bincmp.c +9 -1
- data/ext/ca_obj_binop.c +10 -1
- data/ext/ca_obj_bitfield.c +2 -0
- data/ext/ca_obj_byte_swap.c +2 -0
- data/ext/ca_obj_face.c +2 -0
- data/ext/ca_obj_fake.c +3 -1
- data/ext/ca_obj_grid.c +167 -12
- data/ext/ca_obj_meld.c +69 -113
- data/ext/ca_obj_moncmp.c +9 -1
- data/ext/ca_obj_monop.c +40 -2
- data/ext/ca_obj_object.c +54 -20
- data/ext/ca_obj_reduce.c +15 -1
- data/ext/ca_obj_refer.c +65 -20
- data/ext/ca_obj_remap.c +69 -10
- data/ext/ca_obj_roll.c +1 -0
- data/ext/ca_obj_select.c +4 -1
- data/ext/ca_obj_select_axis.c +1 -0
- data/ext/ca_obj_shift.c +1 -0
- data/ext/ca_obj_stack.c +310 -111
- data/ext/ca_obj_stride.c +131 -45
- data/ext/ca_obj_string.c +18 -2
- data/ext/ca_obj_tile.c +1 -0
- data/ext/ca_obj_triop.c +9 -1
- data/ext/ca_obj_window.c +1 -0
- data/ext/ca_op_ipower.c +8 -8
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +2 -1
- data/ext/carray_access.c +27 -64
- data/ext/carray_address_basis.c +134 -0
- data/ext/carray_attribute.c +8 -0
- data/ext/carray_broadcast.c +3 -4
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_cast.c +8 -11
- data/ext/carray_conversion.c +9 -5
- data/ext/carray_core.c +811 -72
- data/ext/carray_element.c +13 -9
- data/ext/carray_histogram.c +25 -3
- data/ext/carray_internal.h +147 -12
- data/ext/carray_kernels_bincmp.c +2007 -3
- data/ext/carray_kernels_binop.c +25365 -5199
- data/ext/carray_kernels_init.c +8 -8
- data/ext/carray_kernels_monop.c +10696 -3678
- data/ext/carray_kernels_reduce_aggregate.c +4429 -702
- data/ext/carray_kernels_reduce_boolean.c +1533 -15
- data/ext/carray_kernels_reduce_cumulative.c +1808 -248
- data/ext/carray_kernels_reduce_extreme.c +4508 -511
- data/ext/carray_kernels_reduce_variance.c +2284 -152
- data/ext/carray_kernels_scan.c +240 -240
- data/ext/carray_kernels_search.c +4211 -264
- data/ext/carray_kernels_sort.c +367 -450
- data/ext/carray_lazy.c +47 -0
- data/ext/carray_loop.c +1 -1
- data/ext/carray_mask.c +130 -139
- data/ext/carray_median_percentile.c +4 -4
- data/ext/carray_memory_view.c +13 -7
- data/ext/carray_operator.c +147 -44
- data/ext/carray_order.c +3 -2
- data/ext/carray_random.c +4 -1
- data/ext/carray_slab.c +97 -21
- data/ext/carray_sort.c +1 -1
- data/ext/carray_test.c +74 -19
- data/ext/carray_time_text.c +2 -2
- data/ext/extconf.rb +11 -0
- data/ext/mkkernel.rb +346 -94
- data/ext/ruby_carray.c +4 -1
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +25 -1
- data/lib/carray/autoload_carray.rb +3 -3
- data/lib/carray/autoload_method_extension.rb +5 -3
- data/lib/carray/axis_group.rb +13 -7
- data/lib/carray/basics.rb +1 -1
- data/lib/carray/block_iterator.rb +17 -3
- data/lib/carray/categorical.rb +6 -8
- data/lib/carray/categorical_iterator.rb +2 -2
- data/lib/carray/complex.rb +2 -2
- data/lib/carray/conditional.rb +3 -2
- data/lib/carray/frame/column_verbs.rb +45 -14
- data/lib/carray/frame/convert.rb +19 -0
- data/lib/carray/frame/csv_parser.rb +32 -32
- data/lib/carray/frame/describe.rb +4 -2
- data/lib/carray/frame/frame.rb +399 -78
- data/lib/carray/frame/group.rb +679 -53
- data/lib/carray/frame/io.rb +209 -71
- data/lib/carray/frame/join.rb +146 -37
- data/lib/carray/frame/records.rb +3 -3
- data/lib/carray/frame/reshape.rb +89 -11
- data/lib/carray/frame/sort.rb +57 -9
- data/lib/carray/frame/verbs.rb +384 -49
- data/lib/carray/frame/windows.rb +44 -6
- data/lib/carray/inspect.rb +7 -2
- data/lib/carray/iterator.rb +32 -0
- data/lib/carray/lazy.rb +17 -8
- data/lib/carray/mask_gap_fill.rb +37 -17
- data/lib/carray/meld_reduce.rb +1 -1
- data/lib/carray/methods/choose.rb +3 -2
- data/lib/carray/methods/meshgrid.rb +3 -1
- data/lib/carray/runtime.rb +47 -8
- data/lib/carray/segment_iterator.rb +23 -8
- data/lib/carray/serialize.rb +34 -17
- data/lib/carray/stack.rb +18 -6
- data/lib/carray/time.rb +5 -0
- data/lib/carray/window_iterator.rb +16 -4
- data/yard-stubs/ca_obj_array.rb +26 -1
- data/yard-stubs/carray_cast.rb +8 -5
- data/yard-stubs/carray_conversion.rb +8 -2
- data/yard-stubs/carray_mask.rb +20 -1
- data/yard-stubs/carray_math.rb +31 -16
- data/yard-stubs/carray_memory_view.rb +7 -0
- data/yard-stubs/carray_test.rb +7 -4
- metadata +18 -4
|
@@ -98,14 +98,41 @@ accumulate_ki_stream_i8 (VALUE arg)
|
|
|
98
98
|
return LL2NUM((int8_t) (acc));
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
102
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
103
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
104
|
+
typedef struct {
|
|
105
|
+
ca_iter_state states[1];
|
|
106
|
+
VALUE self;
|
|
107
|
+
CArray *ca;
|
|
108
|
+
int8_t *slab_axes;
|
|
109
|
+
int8_t naxes;
|
|
110
|
+
int keep_axis;
|
|
111
|
+
ca_size_t min_count;
|
|
112
|
+
} accumulate_ki_native_i8_frame;
|
|
113
|
+
|
|
101
114
|
static VALUE
|
|
102
|
-
|
|
115
|
+
accumulate_ki_native_i8_body (VALUE arg)
|
|
103
116
|
{
|
|
117
|
+
accumulate_ki_native_i8_frame *fr = (accumulate_ki_native_i8_frame *) arg;
|
|
118
|
+
VALUE self = fr->self;
|
|
119
|
+
(void) self;
|
|
120
|
+
CArray *ca = fr->ca;
|
|
121
|
+
(void) ca;
|
|
122
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
123
|
+
(void) slab_axes;
|
|
124
|
+
int8_t naxes = fr->naxes;
|
|
125
|
+
(void) naxes;
|
|
126
|
+
int keep_axis = fr->keep_axis;
|
|
127
|
+
(void) keep_axis;
|
|
128
|
+
ca_size_t min_count = fr->min_count;
|
|
129
|
+
(void) min_count;
|
|
130
|
+
#define st (fr->states[0])
|
|
104
131
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
105
132
|
ca->elements > 0 &&
|
|
106
133
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
107
134
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
108
|
-
|
|
135
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
109
136
|
accumulate_ki_stream_i8_t __ctx;
|
|
110
137
|
__ctx.chunk = NULL;
|
|
111
138
|
__ctx.ca = ca;
|
|
@@ -126,7 +153,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
126
153
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
127
154
|
&& ! ca_has_mask(ca)
|
|
128
155
|
&& ca_attach_is_alias(ca)
|
|
129
|
-
&&
|
|
156
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
130
157
|
int8_t __li_ax = slab_axes[0];
|
|
131
158
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
132
159
|
ca_size_t __li_INNER = 1;
|
|
@@ -199,7 +226,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
199
226
|
&& ! ca_has_mask(ca)
|
|
200
227
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
201
228
|
&& ((CAStack *) ca)->k_axis == 0
|
|
202
|
-
&&
|
|
229
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
203
230
|
CAStack *__li_st = (CAStack *) ca;
|
|
204
231
|
int8_t __li_ax = slab_axes[0];
|
|
205
232
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -227,7 +254,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
227
254
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
228
255
|
const ca_size_t __li_TILE = 512;
|
|
229
256
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
230
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
257
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
231
258
|
const int8_t *p_root = (const int8_t *) __li_st->parents[__kk]->ptr;
|
|
232
259
|
int8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
233
260
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -287,7 +314,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
287
314
|
&& ((CAStack *) ca)->k_axis > 0
|
|
288
315
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
289
316
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
290
|
-
&&
|
|
317
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
291
318
|
CAStack *__li_st = (CAStack *) ca;
|
|
292
319
|
int8_t __li_ax = slab_axes[0];
|
|
293
320
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -344,7 +371,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
344
371
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
345
372
|
const ca_size_t __li_TILE = 512;
|
|
346
373
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
347
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
374
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
348
375
|
const int8_t *p_root = (const int8_t *) __li_st->parents[__kk]->ptr;
|
|
349
376
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
350
377
|
|
|
@@ -432,7 +459,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
432
459
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
433
460
|
&& ((CAStack *) ca)->k_axis > 0
|
|
434
461
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
435
|
-
&&
|
|
462
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
436
463
|
CAStack *__li_st = (CAStack *) ca;
|
|
437
464
|
int8_t __li_ax = slab_axes[0];
|
|
438
465
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -504,7 +531,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
504
531
|
|
|
505
532
|
const ca_size_t __li_TILE = 512;
|
|
506
533
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
507
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
534
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
508
535
|
const int8_t *p_root = (const int8_t *) __li_st->parents[__kk]->ptr;
|
|
509
536
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
510
537
|
|
|
@@ -553,12 +580,41 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
553
580
|
return vout;
|
|
554
581
|
}
|
|
555
582
|
}
|
|
583
|
+
if ( naxes == 1
|
|
584
|
+
&& ca->ndim >= 2
|
|
585
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
586
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
587
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
588
|
+
&& ! ca_has_mask(ca)
|
|
589
|
+
&& ca_attach_is_alias(ca)
|
|
590
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
591
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
592
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
593
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT8, keep_axis);
|
|
594
|
+
CArray *co;
|
|
595
|
+
GetCArray(vout, co);
|
|
596
|
+
int8_t *op = (int8_t *) co->ptr;
|
|
597
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
598
|
+
const int8_t *p_root = (const int8_t *) ca->ptr;
|
|
599
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
600
|
+
const int8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
601
|
+
int8_t acc = (int8_t) (0);
|
|
602
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
603
|
+
int8_t v = __sf_row[__j];
|
|
604
|
+
acc += v;
|
|
605
|
+
(void) v;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
op[__sf_o] = (int8_t) (acc);
|
|
609
|
+
}
|
|
610
|
+
ca_detach(ca);
|
|
611
|
+
return vout;
|
|
612
|
+
}
|
|
556
613
|
}
|
|
557
614
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT8, keep_axis);
|
|
558
615
|
CArray *co;
|
|
559
616
|
GetCArray(vout, co);
|
|
560
617
|
int8_t *op = (int8_t *) co->ptr;
|
|
561
|
-
ca_iter_state st;
|
|
562
618
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
563
619
|
slab_axes, naxes, 0);
|
|
564
620
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -592,7 +648,22 @@ ca_size_t out_i = 0;
|
|
|
592
648
|
return LL2NUM(op[0]);
|
|
593
649
|
}
|
|
594
650
|
return vout;
|
|
595
|
-
|
|
651
|
+
#undef st
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
static VALUE
|
|
655
|
+
accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
656
|
+
{
|
|
657
|
+
accumulate_ki_native_i8_frame fr;
|
|
658
|
+
fr.self = self;
|
|
659
|
+
fr.ca = ca;
|
|
660
|
+
fr.slab_axes = slab_axes;
|
|
661
|
+
fr.naxes = naxes;
|
|
662
|
+
fr.keep_axis = keep_axis;
|
|
663
|
+
fr.min_count = min_count;
|
|
664
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_i8_body, (VALUE) &fr);
|
|
665
|
+
}
|
|
666
|
+
|
|
596
667
|
|
|
597
668
|
typedef struct {
|
|
598
669
|
void *chunk;
|
|
@@ -647,14 +718,41 @@ accumulate_ki_stream_u8 (VALUE arg)
|
|
|
647
718
|
return ULL2NUM((uint8_t) (acc));
|
|
648
719
|
}
|
|
649
720
|
|
|
721
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
722
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
723
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
724
|
+
typedef struct {
|
|
725
|
+
ca_iter_state states[1];
|
|
726
|
+
VALUE self;
|
|
727
|
+
CArray *ca;
|
|
728
|
+
int8_t *slab_axes;
|
|
729
|
+
int8_t naxes;
|
|
730
|
+
int keep_axis;
|
|
731
|
+
ca_size_t min_count;
|
|
732
|
+
} accumulate_ki_native_u8_frame;
|
|
733
|
+
|
|
650
734
|
static VALUE
|
|
651
|
-
|
|
735
|
+
accumulate_ki_native_u8_body (VALUE arg)
|
|
652
736
|
{
|
|
737
|
+
accumulate_ki_native_u8_frame *fr = (accumulate_ki_native_u8_frame *) arg;
|
|
738
|
+
VALUE self = fr->self;
|
|
739
|
+
(void) self;
|
|
740
|
+
CArray *ca = fr->ca;
|
|
741
|
+
(void) ca;
|
|
742
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
743
|
+
(void) slab_axes;
|
|
744
|
+
int8_t naxes = fr->naxes;
|
|
745
|
+
(void) naxes;
|
|
746
|
+
int keep_axis = fr->keep_axis;
|
|
747
|
+
(void) keep_axis;
|
|
748
|
+
ca_size_t min_count = fr->min_count;
|
|
749
|
+
(void) min_count;
|
|
750
|
+
#define st (fr->states[0])
|
|
653
751
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
654
752
|
ca->elements > 0 &&
|
|
655
753
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
656
754
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
657
|
-
|
|
755
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
658
756
|
accumulate_ki_stream_u8_t __ctx;
|
|
659
757
|
__ctx.chunk = NULL;
|
|
660
758
|
__ctx.ca = ca;
|
|
@@ -675,7 +773,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
675
773
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
676
774
|
&& ! ca_has_mask(ca)
|
|
677
775
|
&& ca_attach_is_alias(ca)
|
|
678
|
-
&&
|
|
776
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
679
777
|
int8_t __li_ax = slab_axes[0];
|
|
680
778
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
681
779
|
ca_size_t __li_INNER = 1;
|
|
@@ -748,7 +846,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
748
846
|
&& ! ca_has_mask(ca)
|
|
749
847
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
750
848
|
&& ((CAStack *) ca)->k_axis == 0
|
|
751
|
-
&&
|
|
849
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
752
850
|
CAStack *__li_st = (CAStack *) ca;
|
|
753
851
|
int8_t __li_ax = slab_axes[0];
|
|
754
852
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -776,7 +874,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
776
874
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
777
875
|
const ca_size_t __li_TILE = 512;
|
|
778
876
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
779
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
877
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
780
878
|
const uint8_t *p_root = (const uint8_t *) __li_st->parents[__kk]->ptr;
|
|
781
879
|
uint8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
782
880
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -836,7 +934,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
836
934
|
&& ((CAStack *) ca)->k_axis > 0
|
|
837
935
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
838
936
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
839
|
-
&&
|
|
937
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
840
938
|
CAStack *__li_st = (CAStack *) ca;
|
|
841
939
|
int8_t __li_ax = slab_axes[0];
|
|
842
940
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -893,7 +991,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
893
991
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
894
992
|
const ca_size_t __li_TILE = 512;
|
|
895
993
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
896
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
994
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
897
995
|
const uint8_t *p_root = (const uint8_t *) __li_st->parents[__kk]->ptr;
|
|
898
996
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
899
997
|
|
|
@@ -981,7 +1079,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
981
1079
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
982
1080
|
&& ((CAStack *) ca)->k_axis > 0
|
|
983
1081
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
984
|
-
&&
|
|
1082
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
985
1083
|
CAStack *__li_st = (CAStack *) ca;
|
|
986
1084
|
int8_t __li_ax = slab_axes[0];
|
|
987
1085
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -1053,7 +1151,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
1053
1151
|
|
|
1054
1152
|
const ca_size_t __li_TILE = 512;
|
|
1055
1153
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1056
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
1154
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1057
1155
|
const uint8_t *p_root = (const uint8_t *) __li_st->parents[__kk]->ptr;
|
|
1058
1156
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1059
1157
|
|
|
@@ -1102,12 +1200,41 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
|
|
|
1102
1200
|
return vout;
|
|
1103
1201
|
}
|
|
1104
1202
|
}
|
|
1203
|
+
if ( naxes == 1
|
|
1204
|
+
&& ca->ndim >= 2
|
|
1205
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
1206
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
1207
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
1208
|
+
&& ! ca_has_mask(ca)
|
|
1209
|
+
&& ca_attach_is_alias(ca)
|
|
1210
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
1211
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
1212
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
1213
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT8, keep_axis);
|
|
1214
|
+
CArray *co;
|
|
1215
|
+
GetCArray(vout, co);
|
|
1216
|
+
uint8_t *op = (uint8_t *) co->ptr;
|
|
1217
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1218
|
+
const uint8_t *p_root = (const uint8_t *) ca->ptr;
|
|
1219
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
1220
|
+
const uint8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
1221
|
+
uint8_t acc = (uint8_t) (0);
|
|
1222
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
1223
|
+
uint8_t v = __sf_row[__j];
|
|
1224
|
+
acc += v;
|
|
1225
|
+
(void) v;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
op[__sf_o] = (uint8_t) (acc);
|
|
1229
|
+
}
|
|
1230
|
+
ca_detach(ca);
|
|
1231
|
+
return vout;
|
|
1232
|
+
}
|
|
1105
1233
|
}
|
|
1106
1234
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT8, keep_axis);
|
|
1107
1235
|
CArray *co;
|
|
1108
1236
|
GetCArray(vout, co);
|
|
1109
1237
|
uint8_t *op = (uint8_t *) co->ptr;
|
|
1110
|
-
ca_iter_state st;
|
|
1111
1238
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
1112
1239
|
slab_axes, naxes, 0);
|
|
1113
1240
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -1141,7 +1268,22 @@ ca_size_t out_i = 0;
|
|
|
1141
1268
|
return ULL2NUM(op[0]);
|
|
1142
1269
|
}
|
|
1143
1270
|
return vout;
|
|
1144
|
-
|
|
1271
|
+
#undef st
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
static VALUE
|
|
1275
|
+
accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
1276
|
+
{
|
|
1277
|
+
accumulate_ki_native_u8_frame fr;
|
|
1278
|
+
fr.self = self;
|
|
1279
|
+
fr.ca = ca;
|
|
1280
|
+
fr.slab_axes = slab_axes;
|
|
1281
|
+
fr.naxes = naxes;
|
|
1282
|
+
fr.keep_axis = keep_axis;
|
|
1283
|
+
fr.min_count = min_count;
|
|
1284
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_u8_body, (VALUE) &fr);
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1145
1287
|
|
|
1146
1288
|
typedef struct {
|
|
1147
1289
|
void *chunk;
|
|
@@ -1196,14 +1338,41 @@ accumulate_ki_stream_i16 (VALUE arg)
|
|
|
1196
1338
|
return LL2NUM((ca_u_int16_t) (acc));
|
|
1197
1339
|
}
|
|
1198
1340
|
|
|
1341
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
1342
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
1343
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
1344
|
+
typedef struct {
|
|
1345
|
+
ca_iter_state states[1];
|
|
1346
|
+
VALUE self;
|
|
1347
|
+
CArray *ca;
|
|
1348
|
+
int8_t *slab_axes;
|
|
1349
|
+
int8_t naxes;
|
|
1350
|
+
int keep_axis;
|
|
1351
|
+
ca_size_t min_count;
|
|
1352
|
+
} accumulate_ki_native_i16_frame;
|
|
1353
|
+
|
|
1199
1354
|
static VALUE
|
|
1200
|
-
|
|
1355
|
+
accumulate_ki_native_i16_body (VALUE arg)
|
|
1201
1356
|
{
|
|
1357
|
+
accumulate_ki_native_i16_frame *fr = (accumulate_ki_native_i16_frame *) arg;
|
|
1358
|
+
VALUE self = fr->self;
|
|
1359
|
+
(void) self;
|
|
1360
|
+
CArray *ca = fr->ca;
|
|
1361
|
+
(void) ca;
|
|
1362
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
1363
|
+
(void) slab_axes;
|
|
1364
|
+
int8_t naxes = fr->naxes;
|
|
1365
|
+
(void) naxes;
|
|
1366
|
+
int keep_axis = fr->keep_axis;
|
|
1367
|
+
(void) keep_axis;
|
|
1368
|
+
ca_size_t min_count = fr->min_count;
|
|
1369
|
+
(void) min_count;
|
|
1370
|
+
#define st (fr->states[0])
|
|
1202
1371
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
1203
1372
|
ca->elements > 0 &&
|
|
1204
1373
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
1205
1374
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
1206
|
-
|
|
1375
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
1207
1376
|
accumulate_ki_stream_i16_t __ctx;
|
|
1208
1377
|
__ctx.chunk = NULL;
|
|
1209
1378
|
__ctx.ca = ca;
|
|
@@ -1224,7 +1393,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1224
1393
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1225
1394
|
&& ! ca_has_mask(ca)
|
|
1226
1395
|
&& ca_attach_is_alias(ca)
|
|
1227
|
-
&&
|
|
1396
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
1228
1397
|
int8_t __li_ax = slab_axes[0];
|
|
1229
1398
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
1230
1399
|
ca_size_t __li_INNER = 1;
|
|
@@ -1297,7 +1466,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1297
1466
|
&& ! ca_has_mask(ca)
|
|
1298
1467
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1299
1468
|
&& ((CAStack *) ca)->k_axis == 0
|
|
1300
|
-
&&
|
|
1469
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1301
1470
|
CAStack *__li_st = (CAStack *) ca;
|
|
1302
1471
|
int8_t __li_ax = slab_axes[0];
|
|
1303
1472
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -1325,7 +1494,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1325
1494
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
1326
1495
|
const ca_size_t __li_TILE = 512;
|
|
1327
1496
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1328
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
1497
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1329
1498
|
const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
|
|
1330
1499
|
ca_u_int16_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
1331
1500
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -1385,7 +1554,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1385
1554
|
&& ((CAStack *) ca)->k_axis > 0
|
|
1386
1555
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
1387
1556
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
1388
|
-
&&
|
|
1557
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1389
1558
|
CAStack *__li_st = (CAStack *) ca;
|
|
1390
1559
|
int8_t __li_ax = slab_axes[0];
|
|
1391
1560
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -1442,7 +1611,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1442
1611
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1443
1612
|
const ca_size_t __li_TILE = 512;
|
|
1444
1613
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1445
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
1614
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1446
1615
|
const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
|
|
1447
1616
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1448
1617
|
|
|
@@ -1530,7 +1699,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1530
1699
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1531
1700
|
&& ((CAStack *) ca)->k_axis > 0
|
|
1532
1701
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
1533
|
-
&&
|
|
1702
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1534
1703
|
CAStack *__li_st = (CAStack *) ca;
|
|
1535
1704
|
int8_t __li_ax = slab_axes[0];
|
|
1536
1705
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -1602,7 +1771,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1602
1771
|
|
|
1603
1772
|
const ca_size_t __li_TILE = 512;
|
|
1604
1773
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1605
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
1774
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1606
1775
|
const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
|
|
1607
1776
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1608
1777
|
|
|
@@ -1651,12 +1820,41 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1651
1820
|
return vout;
|
|
1652
1821
|
}
|
|
1653
1822
|
}
|
|
1823
|
+
if ( naxes == 1
|
|
1824
|
+
&& ca->ndim >= 2
|
|
1825
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
1826
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
1827
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
1828
|
+
&& ! ca_has_mask(ca)
|
|
1829
|
+
&& ca_attach_is_alias(ca)
|
|
1830
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
1831
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
1832
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
1833
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1834
|
+
CArray *co;
|
|
1835
|
+
GetCArray(vout, co);
|
|
1836
|
+
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1837
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1838
|
+
const ca_u_int16_t *p_root = (const ca_u_int16_t *) ca->ptr;
|
|
1839
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
1840
|
+
const ca_u_int16_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
1841
|
+
ca_u_int16_t acc = (ca_u_int16_t) (0);
|
|
1842
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
1843
|
+
ca_u_int16_t v = __sf_row[__j];
|
|
1844
|
+
acc += v;
|
|
1845
|
+
(void) v;
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
op[__sf_o] = (ca_u_int16_t) (acc);
|
|
1849
|
+
}
|
|
1850
|
+
ca_detach(ca);
|
|
1851
|
+
return vout;
|
|
1852
|
+
}
|
|
1654
1853
|
}
|
|
1655
1854
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
|
|
1656
1855
|
CArray *co;
|
|
1657
1856
|
GetCArray(vout, co);
|
|
1658
1857
|
ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
|
|
1659
|
-
ca_iter_state st;
|
|
1660
1858
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
1661
1859
|
slab_axes, naxes, 0);
|
|
1662
1860
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -1690,7 +1888,22 @@ ca_size_t out_i = 0;
|
|
|
1690
1888
|
return LL2NUM(op[0]);
|
|
1691
1889
|
}
|
|
1692
1890
|
return vout;
|
|
1693
|
-
|
|
1891
|
+
#undef st
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
static VALUE
|
|
1895
|
+
accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
1896
|
+
{
|
|
1897
|
+
accumulate_ki_native_i16_frame fr;
|
|
1898
|
+
fr.self = self;
|
|
1899
|
+
fr.ca = ca;
|
|
1900
|
+
fr.slab_axes = slab_axes;
|
|
1901
|
+
fr.naxes = naxes;
|
|
1902
|
+
fr.keep_axis = keep_axis;
|
|
1903
|
+
fr.min_count = min_count;
|
|
1904
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_i16_body, (VALUE) &fr);
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1694
1907
|
|
|
1695
1908
|
typedef struct {
|
|
1696
1909
|
void *chunk;
|
|
@@ -1745,14 +1958,41 @@ accumulate_ki_stream_u16 (VALUE arg)
|
|
|
1745
1958
|
return ULL2NUM((ca_u_uint16_t) (acc));
|
|
1746
1959
|
}
|
|
1747
1960
|
|
|
1961
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
1962
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
1963
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
1964
|
+
typedef struct {
|
|
1965
|
+
ca_iter_state states[1];
|
|
1966
|
+
VALUE self;
|
|
1967
|
+
CArray *ca;
|
|
1968
|
+
int8_t *slab_axes;
|
|
1969
|
+
int8_t naxes;
|
|
1970
|
+
int keep_axis;
|
|
1971
|
+
ca_size_t min_count;
|
|
1972
|
+
} accumulate_ki_native_u16_frame;
|
|
1973
|
+
|
|
1748
1974
|
static VALUE
|
|
1749
|
-
|
|
1975
|
+
accumulate_ki_native_u16_body (VALUE arg)
|
|
1750
1976
|
{
|
|
1977
|
+
accumulate_ki_native_u16_frame *fr = (accumulate_ki_native_u16_frame *) arg;
|
|
1978
|
+
VALUE self = fr->self;
|
|
1979
|
+
(void) self;
|
|
1980
|
+
CArray *ca = fr->ca;
|
|
1981
|
+
(void) ca;
|
|
1982
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
1983
|
+
(void) slab_axes;
|
|
1984
|
+
int8_t naxes = fr->naxes;
|
|
1985
|
+
(void) naxes;
|
|
1986
|
+
int keep_axis = fr->keep_axis;
|
|
1987
|
+
(void) keep_axis;
|
|
1988
|
+
ca_size_t min_count = fr->min_count;
|
|
1989
|
+
(void) min_count;
|
|
1990
|
+
#define st (fr->states[0])
|
|
1751
1991
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
1752
1992
|
ca->elements > 0 &&
|
|
1753
1993
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
1754
1994
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
1755
|
-
|
|
1995
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
1756
1996
|
accumulate_ki_stream_u16_t __ctx;
|
|
1757
1997
|
__ctx.chunk = NULL;
|
|
1758
1998
|
__ctx.ca = ca;
|
|
@@ -1773,7 +2013,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1773
2013
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1774
2014
|
&& ! ca_has_mask(ca)
|
|
1775
2015
|
&& ca_attach_is_alias(ca)
|
|
1776
|
-
&&
|
|
2016
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
1777
2017
|
int8_t __li_ax = slab_axes[0];
|
|
1778
2018
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
1779
2019
|
ca_size_t __li_INNER = 1;
|
|
@@ -1846,7 +2086,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1846
2086
|
&& ! ca_has_mask(ca)
|
|
1847
2087
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1848
2088
|
&& ((CAStack *) ca)->k_axis == 0
|
|
1849
|
-
&&
|
|
2089
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1850
2090
|
CAStack *__li_st = (CAStack *) ca;
|
|
1851
2091
|
int8_t __li_ax = slab_axes[0];
|
|
1852
2092
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -1874,7 +2114,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1874
2114
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
1875
2115
|
const ca_size_t __li_TILE = 512;
|
|
1876
2116
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1877
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
2117
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1878
2118
|
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
|
|
1879
2119
|
ca_u_uint16_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
1880
2120
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -1934,7 +2174,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1934
2174
|
&& ((CAStack *) ca)->k_axis > 0
|
|
1935
2175
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
1936
2176
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
1937
|
-
&&
|
|
2177
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1938
2178
|
CAStack *__li_st = (CAStack *) ca;
|
|
1939
2179
|
int8_t __li_ax = slab_axes[0];
|
|
1940
2180
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -1991,7 +2231,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
1991
2231
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1992
2232
|
const ca_size_t __li_TILE = 512;
|
|
1993
2233
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1994
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
2234
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1995
2235
|
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
|
|
1996
2236
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1997
2237
|
|
|
@@ -2079,7 +2319,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2079
2319
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
2080
2320
|
&& ((CAStack *) ca)->k_axis > 0
|
|
2081
2321
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
2082
|
-
&&
|
|
2322
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
2083
2323
|
CAStack *__li_st = (CAStack *) ca;
|
|
2084
2324
|
int8_t __li_ax = slab_axes[0];
|
|
2085
2325
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -2151,7 +2391,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2151
2391
|
|
|
2152
2392
|
const ca_size_t __li_TILE = 512;
|
|
2153
2393
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2154
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
2394
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
2155
2395
|
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
|
|
2156
2396
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
2157
2397
|
|
|
@@ -2200,12 +2440,41 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2200
2440
|
return vout;
|
|
2201
2441
|
}
|
|
2202
2442
|
}
|
|
2443
|
+
if ( naxes == 1
|
|
2444
|
+
&& ca->ndim >= 2
|
|
2445
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
2446
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
2447
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
2448
|
+
&& ! ca_has_mask(ca)
|
|
2449
|
+
&& ca_attach_is_alias(ca)
|
|
2450
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
2451
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
2452
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
2453
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
2454
|
+
CArray *co;
|
|
2455
|
+
GetCArray(vout, co);
|
|
2456
|
+
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
2457
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
2458
|
+
const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) ca->ptr;
|
|
2459
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
2460
|
+
const ca_u_uint16_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
2461
|
+
ca_u_uint16_t acc = (ca_u_uint16_t) (0);
|
|
2462
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
2463
|
+
ca_u_uint16_t v = __sf_row[__j];
|
|
2464
|
+
acc += v;
|
|
2465
|
+
(void) v;
|
|
2466
|
+
}
|
|
2467
|
+
|
|
2468
|
+
op[__sf_o] = (ca_u_uint16_t) (acc);
|
|
2469
|
+
}
|
|
2470
|
+
ca_detach(ca);
|
|
2471
|
+
return vout;
|
|
2472
|
+
}
|
|
2203
2473
|
}
|
|
2204
2474
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
|
|
2205
2475
|
CArray *co;
|
|
2206
2476
|
GetCArray(vout, co);
|
|
2207
2477
|
ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
|
|
2208
|
-
ca_iter_state st;
|
|
2209
2478
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
2210
2479
|
slab_axes, naxes, 0);
|
|
2211
2480
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -2239,7 +2508,22 @@ ca_size_t out_i = 0;
|
|
|
2239
2508
|
return ULL2NUM(op[0]);
|
|
2240
2509
|
}
|
|
2241
2510
|
return vout;
|
|
2242
|
-
|
|
2511
|
+
#undef st
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
static VALUE
|
|
2515
|
+
accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
2516
|
+
{
|
|
2517
|
+
accumulate_ki_native_u16_frame fr;
|
|
2518
|
+
fr.self = self;
|
|
2519
|
+
fr.ca = ca;
|
|
2520
|
+
fr.slab_axes = slab_axes;
|
|
2521
|
+
fr.naxes = naxes;
|
|
2522
|
+
fr.keep_axis = keep_axis;
|
|
2523
|
+
fr.min_count = min_count;
|
|
2524
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_u16_body, (VALUE) &fr);
|
|
2525
|
+
}
|
|
2526
|
+
|
|
2243
2527
|
|
|
2244
2528
|
typedef struct {
|
|
2245
2529
|
void *chunk;
|
|
@@ -2294,14 +2578,41 @@ accumulate_ki_stream_i32 (VALUE arg)
|
|
|
2294
2578
|
return LL2NUM((ca_u_int32_t) (acc));
|
|
2295
2579
|
}
|
|
2296
2580
|
|
|
2581
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
2582
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
2583
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
2584
|
+
typedef struct {
|
|
2585
|
+
ca_iter_state states[1];
|
|
2586
|
+
VALUE self;
|
|
2587
|
+
CArray *ca;
|
|
2588
|
+
int8_t *slab_axes;
|
|
2589
|
+
int8_t naxes;
|
|
2590
|
+
int keep_axis;
|
|
2591
|
+
ca_size_t min_count;
|
|
2592
|
+
} accumulate_ki_native_i32_frame;
|
|
2593
|
+
|
|
2297
2594
|
static VALUE
|
|
2298
|
-
|
|
2595
|
+
accumulate_ki_native_i32_body (VALUE arg)
|
|
2299
2596
|
{
|
|
2597
|
+
accumulate_ki_native_i32_frame *fr = (accumulate_ki_native_i32_frame *) arg;
|
|
2598
|
+
VALUE self = fr->self;
|
|
2599
|
+
(void) self;
|
|
2600
|
+
CArray *ca = fr->ca;
|
|
2601
|
+
(void) ca;
|
|
2602
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
2603
|
+
(void) slab_axes;
|
|
2604
|
+
int8_t naxes = fr->naxes;
|
|
2605
|
+
(void) naxes;
|
|
2606
|
+
int keep_axis = fr->keep_axis;
|
|
2607
|
+
(void) keep_axis;
|
|
2608
|
+
ca_size_t min_count = fr->min_count;
|
|
2609
|
+
(void) min_count;
|
|
2610
|
+
#define st (fr->states[0])
|
|
2300
2611
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
2301
2612
|
ca->elements > 0 &&
|
|
2302
2613
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
2303
2614
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
2304
|
-
|
|
2615
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
2305
2616
|
accumulate_ki_stream_i32_t __ctx;
|
|
2306
2617
|
__ctx.chunk = NULL;
|
|
2307
2618
|
__ctx.ca = ca;
|
|
@@ -2322,7 +2633,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2322
2633
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
2323
2634
|
&& ! ca_has_mask(ca)
|
|
2324
2635
|
&& ca_attach_is_alias(ca)
|
|
2325
|
-
&&
|
|
2636
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
2326
2637
|
int8_t __li_ax = slab_axes[0];
|
|
2327
2638
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
2328
2639
|
ca_size_t __li_INNER = 1;
|
|
@@ -2395,7 +2706,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2395
2706
|
&& ! ca_has_mask(ca)
|
|
2396
2707
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
2397
2708
|
&& ((CAStack *) ca)->k_axis == 0
|
|
2398
|
-
&&
|
|
2709
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
2399
2710
|
CAStack *__li_st = (CAStack *) ca;
|
|
2400
2711
|
int8_t __li_ax = slab_axes[0];
|
|
2401
2712
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -2423,7 +2734,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2423
2734
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
2424
2735
|
const ca_size_t __li_TILE = 512;
|
|
2425
2736
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2426
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
2737
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
2427
2738
|
const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
|
|
2428
2739
|
ca_u_int32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
2429
2740
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -2483,7 +2794,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2483
2794
|
&& ((CAStack *) ca)->k_axis > 0
|
|
2484
2795
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
2485
2796
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
2486
|
-
&&
|
|
2797
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
2487
2798
|
CAStack *__li_st = (CAStack *) ca;
|
|
2488
2799
|
int8_t __li_ax = slab_axes[0];
|
|
2489
2800
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -2540,7 +2851,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2540
2851
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
2541
2852
|
const ca_size_t __li_TILE = 512;
|
|
2542
2853
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2543
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
2854
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
2544
2855
|
const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
|
|
2545
2856
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
2546
2857
|
|
|
@@ -2628,7 +2939,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2628
2939
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
2629
2940
|
&& ((CAStack *) ca)->k_axis > 0
|
|
2630
2941
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
2631
|
-
&&
|
|
2942
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
2632
2943
|
CAStack *__li_st = (CAStack *) ca;
|
|
2633
2944
|
int8_t __li_ax = slab_axes[0];
|
|
2634
2945
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -2700,7 +3011,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2700
3011
|
|
|
2701
3012
|
const ca_size_t __li_TILE = 512;
|
|
2702
3013
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2703
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
3014
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
2704
3015
|
const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
|
|
2705
3016
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
2706
3017
|
|
|
@@ -2749,12 +3060,41 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2749
3060
|
return vout;
|
|
2750
3061
|
}
|
|
2751
3062
|
}
|
|
3063
|
+
if ( naxes == 1
|
|
3064
|
+
&& ca->ndim >= 2
|
|
3065
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
3066
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
3067
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
3068
|
+
&& ! ca_has_mask(ca)
|
|
3069
|
+
&& ca_attach_is_alias(ca)
|
|
3070
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
3071
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
3072
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
3073
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
3074
|
+
CArray *co;
|
|
3075
|
+
GetCArray(vout, co);
|
|
3076
|
+
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
3077
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
3078
|
+
const ca_u_int32_t *p_root = (const ca_u_int32_t *) ca->ptr;
|
|
3079
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
3080
|
+
const ca_u_int32_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
3081
|
+
ca_u_int32_t acc = (ca_u_int32_t) (0);
|
|
3082
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
3083
|
+
ca_u_int32_t v = __sf_row[__j];
|
|
3084
|
+
acc += v;
|
|
3085
|
+
(void) v;
|
|
3086
|
+
}
|
|
3087
|
+
|
|
3088
|
+
op[__sf_o] = (ca_u_int32_t) (acc);
|
|
3089
|
+
}
|
|
3090
|
+
ca_detach(ca);
|
|
3091
|
+
return vout;
|
|
3092
|
+
}
|
|
2752
3093
|
}
|
|
2753
3094
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
|
|
2754
3095
|
CArray *co;
|
|
2755
3096
|
GetCArray(vout, co);
|
|
2756
3097
|
ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
|
|
2757
|
-
ca_iter_state st;
|
|
2758
3098
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
2759
3099
|
slab_axes, naxes, 0);
|
|
2760
3100
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -2788,7 +3128,22 @@ ca_size_t out_i = 0;
|
|
|
2788
3128
|
return LL2NUM(op[0]);
|
|
2789
3129
|
}
|
|
2790
3130
|
return vout;
|
|
2791
|
-
|
|
3131
|
+
#undef st
|
|
3132
|
+
}
|
|
3133
|
+
|
|
3134
|
+
static VALUE
|
|
3135
|
+
accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
3136
|
+
{
|
|
3137
|
+
accumulate_ki_native_i32_frame fr;
|
|
3138
|
+
fr.self = self;
|
|
3139
|
+
fr.ca = ca;
|
|
3140
|
+
fr.slab_axes = slab_axes;
|
|
3141
|
+
fr.naxes = naxes;
|
|
3142
|
+
fr.keep_axis = keep_axis;
|
|
3143
|
+
fr.min_count = min_count;
|
|
3144
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_i32_body, (VALUE) &fr);
|
|
3145
|
+
}
|
|
3146
|
+
|
|
2792
3147
|
|
|
2793
3148
|
typedef struct {
|
|
2794
3149
|
void *chunk;
|
|
@@ -2843,21 +3198,48 @@ accumulate_ki_stream_u32 (VALUE arg)
|
|
|
2843
3198
|
return ULL2NUM((ca_u_uint32_t) (acc));
|
|
2844
3199
|
}
|
|
2845
3200
|
|
|
3201
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
3202
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
3203
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
3204
|
+
typedef struct {
|
|
3205
|
+
ca_iter_state states[1];
|
|
3206
|
+
VALUE self;
|
|
3207
|
+
CArray *ca;
|
|
3208
|
+
int8_t *slab_axes;
|
|
3209
|
+
int8_t naxes;
|
|
3210
|
+
int keep_axis;
|
|
3211
|
+
ca_size_t min_count;
|
|
3212
|
+
} accumulate_ki_native_u32_frame;
|
|
3213
|
+
|
|
2846
3214
|
static VALUE
|
|
2847
|
-
|
|
3215
|
+
accumulate_ki_native_u32_body (VALUE arg)
|
|
2848
3216
|
{
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
3217
|
+
accumulate_ki_native_u32_frame *fr = (accumulate_ki_native_u32_frame *) arg;
|
|
3218
|
+
VALUE self = fr->self;
|
|
3219
|
+
(void) self;
|
|
3220
|
+
CArray *ca = fr->ca;
|
|
3221
|
+
(void) ca;
|
|
3222
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
3223
|
+
(void) slab_axes;
|
|
3224
|
+
int8_t naxes = fr->naxes;
|
|
3225
|
+
(void) naxes;
|
|
3226
|
+
int keep_axis = fr->keep_axis;
|
|
3227
|
+
(void) keep_axis;
|
|
3228
|
+
ca_size_t min_count = fr->min_count;
|
|
3229
|
+
(void) min_count;
|
|
3230
|
+
#define st (fr->states[0])
|
|
3231
|
+
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
3232
|
+
ca->elements > 0 &&
|
|
3233
|
+
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
3234
|
+
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
3235
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
3236
|
+
accumulate_ki_stream_u32_t __ctx;
|
|
3237
|
+
__ctx.chunk = NULL;
|
|
3238
|
+
__ctx.ca = ca;
|
|
3239
|
+
__ctx.min_count = min_count;
|
|
3240
|
+
return ca_lazy_arena_protect(accumulate_ki_stream_u32, ca_lazy_arena_release_held,
|
|
3241
|
+
(VALUE) &__ctx);
|
|
3242
|
+
}
|
|
2861
3243
|
if ( naxes != 1 || ! ca_too_few_present(ca->dim[slab_axes[0]], min_count, 0) ) {
|
|
2862
3244
|
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
2863
3245
|
(PROPOSAL_REDUCTION_LOOP_INTERCHANGE).
|
|
@@ -2871,7 +3253,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2871
3253
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
2872
3254
|
&& ! ca_has_mask(ca)
|
|
2873
3255
|
&& ca_attach_is_alias(ca)
|
|
2874
|
-
&&
|
|
3256
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
2875
3257
|
int8_t __li_ax = slab_axes[0];
|
|
2876
3258
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
2877
3259
|
ca_size_t __li_INNER = 1;
|
|
@@ -2944,7 +3326,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2944
3326
|
&& ! ca_has_mask(ca)
|
|
2945
3327
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
2946
3328
|
&& ((CAStack *) ca)->k_axis == 0
|
|
2947
|
-
&&
|
|
3329
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
2948
3330
|
CAStack *__li_st = (CAStack *) ca;
|
|
2949
3331
|
int8_t __li_ax = slab_axes[0];
|
|
2950
3332
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -2972,7 +3354,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
2972
3354
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
2973
3355
|
const ca_size_t __li_TILE = 512;
|
|
2974
3356
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
2975
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
3357
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
2976
3358
|
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
|
|
2977
3359
|
ca_u_uint32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
2978
3360
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -3032,7 +3414,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3032
3414
|
&& ((CAStack *) ca)->k_axis > 0
|
|
3033
3415
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
3034
3416
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
3035
|
-
&&
|
|
3417
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
3036
3418
|
CAStack *__li_st = (CAStack *) ca;
|
|
3037
3419
|
int8_t __li_ax = slab_axes[0];
|
|
3038
3420
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -3089,7 +3471,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3089
3471
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3090
3472
|
const ca_size_t __li_TILE = 512;
|
|
3091
3473
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3092
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
3474
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
3093
3475
|
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
|
|
3094
3476
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3095
3477
|
|
|
@@ -3177,7 +3559,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3177
3559
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
3178
3560
|
&& ((CAStack *) ca)->k_axis > 0
|
|
3179
3561
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
3180
|
-
&&
|
|
3562
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
3181
3563
|
CAStack *__li_st = (CAStack *) ca;
|
|
3182
3564
|
int8_t __li_ax = slab_axes[0];
|
|
3183
3565
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -3249,7 +3631,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3249
3631
|
|
|
3250
3632
|
const ca_size_t __li_TILE = 512;
|
|
3251
3633
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3252
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
3634
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
3253
3635
|
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
|
|
3254
3636
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3255
3637
|
|
|
@@ -3298,12 +3680,41 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3298
3680
|
return vout;
|
|
3299
3681
|
}
|
|
3300
3682
|
}
|
|
3683
|
+
if ( naxes == 1
|
|
3684
|
+
&& ca->ndim >= 2
|
|
3685
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
3686
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
3687
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
3688
|
+
&& ! ca_has_mask(ca)
|
|
3689
|
+
&& ca_attach_is_alias(ca)
|
|
3690
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
3691
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
3692
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
3693
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
3694
|
+
CArray *co;
|
|
3695
|
+
GetCArray(vout, co);
|
|
3696
|
+
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
3697
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
3698
|
+
const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) ca->ptr;
|
|
3699
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
3700
|
+
const ca_u_uint32_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
3701
|
+
ca_u_uint32_t acc = (ca_u_uint32_t) (0);
|
|
3702
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
3703
|
+
ca_u_uint32_t v = __sf_row[__j];
|
|
3704
|
+
acc += v;
|
|
3705
|
+
(void) v;
|
|
3706
|
+
}
|
|
3707
|
+
|
|
3708
|
+
op[__sf_o] = (ca_u_uint32_t) (acc);
|
|
3709
|
+
}
|
|
3710
|
+
ca_detach(ca);
|
|
3711
|
+
return vout;
|
|
3712
|
+
}
|
|
3301
3713
|
}
|
|
3302
3714
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
|
|
3303
3715
|
CArray *co;
|
|
3304
3716
|
GetCArray(vout, co);
|
|
3305
3717
|
ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
|
|
3306
|
-
ca_iter_state st;
|
|
3307
3718
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
3308
3719
|
slab_axes, naxes, 0);
|
|
3309
3720
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -3337,7 +3748,22 @@ ca_size_t out_i = 0;
|
|
|
3337
3748
|
return ULL2NUM(op[0]);
|
|
3338
3749
|
}
|
|
3339
3750
|
return vout;
|
|
3340
|
-
|
|
3751
|
+
#undef st
|
|
3752
|
+
}
|
|
3753
|
+
|
|
3754
|
+
static VALUE
|
|
3755
|
+
accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
3756
|
+
{
|
|
3757
|
+
accumulate_ki_native_u32_frame fr;
|
|
3758
|
+
fr.self = self;
|
|
3759
|
+
fr.ca = ca;
|
|
3760
|
+
fr.slab_axes = slab_axes;
|
|
3761
|
+
fr.naxes = naxes;
|
|
3762
|
+
fr.keep_axis = keep_axis;
|
|
3763
|
+
fr.min_count = min_count;
|
|
3764
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_u32_body, (VALUE) &fr);
|
|
3765
|
+
}
|
|
3766
|
+
|
|
3341
3767
|
|
|
3342
3768
|
typedef struct {
|
|
3343
3769
|
void *chunk;
|
|
@@ -3392,14 +3818,41 @@ accumulate_ki_stream_i64 (VALUE arg)
|
|
|
3392
3818
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
3393
3819
|
}
|
|
3394
3820
|
|
|
3821
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
3822
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
3823
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
3824
|
+
typedef struct {
|
|
3825
|
+
ca_iter_state states[1];
|
|
3826
|
+
VALUE self;
|
|
3827
|
+
CArray *ca;
|
|
3828
|
+
int8_t *slab_axes;
|
|
3829
|
+
int8_t naxes;
|
|
3830
|
+
int keep_axis;
|
|
3831
|
+
ca_size_t min_count;
|
|
3832
|
+
} accumulate_ki_native_i64_frame;
|
|
3833
|
+
|
|
3395
3834
|
static VALUE
|
|
3396
|
-
|
|
3835
|
+
accumulate_ki_native_i64_body (VALUE arg)
|
|
3397
3836
|
{
|
|
3837
|
+
accumulate_ki_native_i64_frame *fr = (accumulate_ki_native_i64_frame *) arg;
|
|
3838
|
+
VALUE self = fr->self;
|
|
3839
|
+
(void) self;
|
|
3840
|
+
CArray *ca = fr->ca;
|
|
3841
|
+
(void) ca;
|
|
3842
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
3843
|
+
(void) slab_axes;
|
|
3844
|
+
int8_t naxes = fr->naxes;
|
|
3845
|
+
(void) naxes;
|
|
3846
|
+
int keep_axis = fr->keep_axis;
|
|
3847
|
+
(void) keep_axis;
|
|
3848
|
+
ca_size_t min_count = fr->min_count;
|
|
3849
|
+
(void) min_count;
|
|
3850
|
+
#define st (fr->states[0])
|
|
3398
3851
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
3399
3852
|
ca->elements > 0 &&
|
|
3400
3853
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
3401
3854
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
3402
|
-
|
|
3855
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
3403
3856
|
accumulate_ki_stream_i64_t __ctx;
|
|
3404
3857
|
__ctx.chunk = NULL;
|
|
3405
3858
|
__ctx.ca = ca;
|
|
@@ -3420,7 +3873,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3420
3873
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
3421
3874
|
&& ! ca_has_mask(ca)
|
|
3422
3875
|
&& ca_attach_is_alias(ca)
|
|
3423
|
-
&&
|
|
3876
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
3424
3877
|
int8_t __li_ax = slab_axes[0];
|
|
3425
3878
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
3426
3879
|
ca_size_t __li_INNER = 1;
|
|
@@ -3493,7 +3946,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3493
3946
|
&& ! ca_has_mask(ca)
|
|
3494
3947
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
3495
3948
|
&& ((CAStack *) ca)->k_axis == 0
|
|
3496
|
-
&&
|
|
3949
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
3497
3950
|
CAStack *__li_st = (CAStack *) ca;
|
|
3498
3951
|
int8_t __li_ax = slab_axes[0];
|
|
3499
3952
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -3521,7 +3974,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3521
3974
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
3522
3975
|
const ca_size_t __li_TILE = 512;
|
|
3523
3976
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3524
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
3977
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
3525
3978
|
const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
|
|
3526
3979
|
ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
3527
3980
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -3581,7 +4034,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3581
4034
|
&& ((CAStack *) ca)->k_axis > 0
|
|
3582
4035
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
3583
4036
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
3584
|
-
&&
|
|
4037
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
3585
4038
|
CAStack *__li_st = (CAStack *) ca;
|
|
3586
4039
|
int8_t __li_ax = slab_axes[0];
|
|
3587
4040
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -3638,7 +4091,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3638
4091
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
3639
4092
|
const ca_size_t __li_TILE = 512;
|
|
3640
4093
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3641
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
4094
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
3642
4095
|
const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
|
|
3643
4096
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3644
4097
|
|
|
@@ -3726,7 +4179,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3726
4179
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
3727
4180
|
&& ((CAStack *) ca)->k_axis > 0
|
|
3728
4181
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
3729
|
-
&&
|
|
4182
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
3730
4183
|
CAStack *__li_st = (CAStack *) ca;
|
|
3731
4184
|
int8_t __li_ax = slab_axes[0];
|
|
3732
4185
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -3798,7 +4251,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3798
4251
|
|
|
3799
4252
|
const ca_size_t __li_TILE = 512;
|
|
3800
4253
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
3801
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
4254
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
3802
4255
|
const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
|
|
3803
4256
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
3804
4257
|
|
|
@@ -3847,12 +4300,41 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3847
4300
|
return vout;
|
|
3848
4301
|
}
|
|
3849
4302
|
}
|
|
4303
|
+
if ( naxes == 1
|
|
4304
|
+
&& ca->ndim >= 2
|
|
4305
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
4306
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
4307
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
4308
|
+
&& ! ca_has_mask(ca)
|
|
4309
|
+
&& ca_attach_is_alias(ca)
|
|
4310
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
4311
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
4312
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
4313
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
4314
|
+
CArray *co;
|
|
4315
|
+
GetCArray(vout, co);
|
|
4316
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
4317
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
4318
|
+
const ca_u_int64_t *p_root = (const ca_u_int64_t *) ca->ptr;
|
|
4319
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
4320
|
+
const ca_u_int64_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
4321
|
+
ca_u_int64_t acc = (ca_u_int64_t) (0);
|
|
4322
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
4323
|
+
ca_u_int64_t v = __sf_row[__j];
|
|
4324
|
+
acc += v;
|
|
4325
|
+
(void) v;
|
|
4326
|
+
}
|
|
4327
|
+
|
|
4328
|
+
op[__sf_o] = (ca_u_int64_t) (acc);
|
|
4329
|
+
}
|
|
4330
|
+
ca_detach(ca);
|
|
4331
|
+
return vout;
|
|
4332
|
+
}
|
|
3850
4333
|
}
|
|
3851
4334
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
3852
4335
|
CArray *co;
|
|
3853
4336
|
GetCArray(vout, co);
|
|
3854
4337
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
3855
|
-
ca_iter_state st;
|
|
3856
4338
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
3857
4339
|
slab_axes, naxes, 0);
|
|
3858
4340
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -3886,7 +4368,22 @@ ca_size_t out_i = 0;
|
|
|
3886
4368
|
return LL2NUM(op[0]);
|
|
3887
4369
|
}
|
|
3888
4370
|
return vout;
|
|
3889
|
-
|
|
4371
|
+
#undef st
|
|
4372
|
+
}
|
|
4373
|
+
|
|
4374
|
+
static VALUE
|
|
4375
|
+
accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
4376
|
+
{
|
|
4377
|
+
accumulate_ki_native_i64_frame fr;
|
|
4378
|
+
fr.self = self;
|
|
4379
|
+
fr.ca = ca;
|
|
4380
|
+
fr.slab_axes = slab_axes;
|
|
4381
|
+
fr.naxes = naxes;
|
|
4382
|
+
fr.keep_axis = keep_axis;
|
|
4383
|
+
fr.min_count = min_count;
|
|
4384
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_i64_body, (VALUE) &fr);
|
|
4385
|
+
}
|
|
4386
|
+
|
|
3890
4387
|
|
|
3891
4388
|
typedef struct {
|
|
3892
4389
|
void *chunk;
|
|
@@ -3941,14 +4438,41 @@ accumulate_ki_stream_u64 (VALUE arg)
|
|
|
3941
4438
|
return ULL2NUM((ca_u_uint64_t) (acc));
|
|
3942
4439
|
}
|
|
3943
4440
|
|
|
4441
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
4442
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
4443
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
4444
|
+
typedef struct {
|
|
4445
|
+
ca_iter_state states[1];
|
|
4446
|
+
VALUE self;
|
|
4447
|
+
CArray *ca;
|
|
4448
|
+
int8_t *slab_axes;
|
|
4449
|
+
int8_t naxes;
|
|
4450
|
+
int keep_axis;
|
|
4451
|
+
ca_size_t min_count;
|
|
4452
|
+
} accumulate_ki_native_u64_frame;
|
|
4453
|
+
|
|
3944
4454
|
static VALUE
|
|
3945
|
-
|
|
4455
|
+
accumulate_ki_native_u64_body (VALUE arg)
|
|
3946
4456
|
{
|
|
4457
|
+
accumulate_ki_native_u64_frame *fr = (accumulate_ki_native_u64_frame *) arg;
|
|
4458
|
+
VALUE self = fr->self;
|
|
4459
|
+
(void) self;
|
|
4460
|
+
CArray *ca = fr->ca;
|
|
4461
|
+
(void) ca;
|
|
4462
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
4463
|
+
(void) slab_axes;
|
|
4464
|
+
int8_t naxes = fr->naxes;
|
|
4465
|
+
(void) naxes;
|
|
4466
|
+
int keep_axis = fr->keep_axis;
|
|
4467
|
+
(void) keep_axis;
|
|
4468
|
+
ca_size_t min_count = fr->min_count;
|
|
4469
|
+
(void) min_count;
|
|
4470
|
+
#define st (fr->states[0])
|
|
3947
4471
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
3948
4472
|
ca->elements > 0 &&
|
|
3949
4473
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
3950
4474
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
3951
|
-
|
|
4475
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
3952
4476
|
accumulate_ki_stream_u64_t __ctx;
|
|
3953
4477
|
__ctx.chunk = NULL;
|
|
3954
4478
|
__ctx.ca = ca;
|
|
@@ -3969,7 +4493,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
3969
4493
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
3970
4494
|
&& ! ca_has_mask(ca)
|
|
3971
4495
|
&& ca_attach_is_alias(ca)
|
|
3972
|
-
&&
|
|
4496
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
3973
4497
|
int8_t __li_ax = slab_axes[0];
|
|
3974
4498
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
3975
4499
|
ca_size_t __li_INNER = 1;
|
|
@@ -4042,7 +4566,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4042
4566
|
&& ! ca_has_mask(ca)
|
|
4043
4567
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
4044
4568
|
&& ((CAStack *) ca)->k_axis == 0
|
|
4045
|
-
&&
|
|
4569
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
4046
4570
|
CAStack *__li_st = (CAStack *) ca;
|
|
4047
4571
|
int8_t __li_ax = slab_axes[0];
|
|
4048
4572
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -4070,7 +4594,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4070
4594
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
4071
4595
|
const ca_size_t __li_TILE = 512;
|
|
4072
4596
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4073
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
4597
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
4074
4598
|
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
|
|
4075
4599
|
ca_u_uint64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
4076
4600
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -4130,7 +4654,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4130
4654
|
&& ((CAStack *) ca)->k_axis > 0
|
|
4131
4655
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
4132
4656
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
4133
|
-
&&
|
|
4657
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
4134
4658
|
CAStack *__li_st = (CAStack *) ca;
|
|
4135
4659
|
int8_t __li_ax = slab_axes[0];
|
|
4136
4660
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -4187,7 +4711,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4187
4711
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4188
4712
|
const ca_size_t __li_TILE = 512;
|
|
4189
4713
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4190
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
4714
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
4191
4715
|
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
|
|
4192
4716
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4193
4717
|
|
|
@@ -4275,7 +4799,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4275
4799
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
4276
4800
|
&& ((CAStack *) ca)->k_axis > 0
|
|
4277
4801
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
4278
|
-
&&
|
|
4802
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
4279
4803
|
CAStack *__li_st = (CAStack *) ca;
|
|
4280
4804
|
int8_t __li_ax = slab_axes[0];
|
|
4281
4805
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -4347,7 +4871,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4347
4871
|
|
|
4348
4872
|
const ca_size_t __li_TILE = 512;
|
|
4349
4873
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4350
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
4874
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
4351
4875
|
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
|
|
4352
4876
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4353
4877
|
|
|
@@ -4396,12 +4920,41 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4396
4920
|
return vout;
|
|
4397
4921
|
}
|
|
4398
4922
|
}
|
|
4923
|
+
if ( naxes == 1
|
|
4924
|
+
&& ca->ndim >= 2
|
|
4925
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
4926
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
4927
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
4928
|
+
&& ! ca_has_mask(ca)
|
|
4929
|
+
&& ca_attach_is_alias(ca)
|
|
4930
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
4931
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
4932
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
4933
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
4934
|
+
CArray *co;
|
|
4935
|
+
GetCArray(vout, co);
|
|
4936
|
+
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
4937
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
4938
|
+
const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) ca->ptr;
|
|
4939
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
4940
|
+
const ca_u_uint64_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
4941
|
+
ca_u_uint64_t acc = (ca_u_uint64_t) (0);
|
|
4942
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
4943
|
+
ca_u_uint64_t v = __sf_row[__j];
|
|
4944
|
+
acc += v;
|
|
4945
|
+
(void) v;
|
|
4946
|
+
}
|
|
4947
|
+
|
|
4948
|
+
op[__sf_o] = (ca_u_uint64_t) (acc);
|
|
4949
|
+
}
|
|
4950
|
+
ca_detach(ca);
|
|
4951
|
+
return vout;
|
|
4952
|
+
}
|
|
4399
4953
|
}
|
|
4400
4954
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
|
|
4401
4955
|
CArray *co;
|
|
4402
4956
|
GetCArray(vout, co);
|
|
4403
4957
|
ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
|
|
4404
|
-
ca_iter_state st;
|
|
4405
4958
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
4406
4959
|
slab_axes, naxes, 0);
|
|
4407
4960
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -4435,7 +4988,22 @@ ca_size_t out_i = 0;
|
|
|
4435
4988
|
return ULL2NUM(op[0]);
|
|
4436
4989
|
}
|
|
4437
4990
|
return vout;
|
|
4438
|
-
|
|
4991
|
+
#undef st
|
|
4992
|
+
}
|
|
4993
|
+
|
|
4994
|
+
static VALUE
|
|
4995
|
+
accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
4996
|
+
{
|
|
4997
|
+
accumulate_ki_native_u64_frame fr;
|
|
4998
|
+
fr.self = self;
|
|
4999
|
+
fr.ca = ca;
|
|
5000
|
+
fr.slab_axes = slab_axes;
|
|
5001
|
+
fr.naxes = naxes;
|
|
5002
|
+
fr.keep_axis = keep_axis;
|
|
5003
|
+
fr.min_count = min_count;
|
|
5004
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_u64_body, (VALUE) &fr);
|
|
5005
|
+
}
|
|
5006
|
+
|
|
4439
5007
|
|
|
4440
5008
|
typedef struct {
|
|
4441
5009
|
void *chunk;
|
|
@@ -4490,14 +5058,41 @@ accumulate_ki_stream_f32 (VALUE arg)
|
|
|
4490
5058
|
return rb_float_new((ca_u_float32_t) (acc));
|
|
4491
5059
|
}
|
|
4492
5060
|
|
|
5061
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
5062
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
5063
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
5064
|
+
typedef struct {
|
|
5065
|
+
ca_iter_state states[1];
|
|
5066
|
+
VALUE self;
|
|
5067
|
+
CArray *ca;
|
|
5068
|
+
int8_t *slab_axes;
|
|
5069
|
+
int8_t naxes;
|
|
5070
|
+
int keep_axis;
|
|
5071
|
+
ca_size_t min_count;
|
|
5072
|
+
} accumulate_ki_native_f32_frame;
|
|
5073
|
+
|
|
4493
5074
|
static VALUE
|
|
4494
|
-
|
|
5075
|
+
accumulate_ki_native_f32_body (VALUE arg)
|
|
4495
5076
|
{
|
|
5077
|
+
accumulate_ki_native_f32_frame *fr = (accumulate_ki_native_f32_frame *) arg;
|
|
5078
|
+
VALUE self = fr->self;
|
|
5079
|
+
(void) self;
|
|
5080
|
+
CArray *ca = fr->ca;
|
|
5081
|
+
(void) ca;
|
|
5082
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
5083
|
+
(void) slab_axes;
|
|
5084
|
+
int8_t naxes = fr->naxes;
|
|
5085
|
+
(void) naxes;
|
|
5086
|
+
int keep_axis = fr->keep_axis;
|
|
5087
|
+
(void) keep_axis;
|
|
5088
|
+
ca_size_t min_count = fr->min_count;
|
|
5089
|
+
(void) min_count;
|
|
5090
|
+
#define st (fr->states[0])
|
|
4496
5091
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
4497
5092
|
ca->elements > 0 &&
|
|
4498
5093
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
4499
5094
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
4500
|
-
|
|
5095
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
4501
5096
|
accumulate_ki_stream_f32_t __ctx;
|
|
4502
5097
|
__ctx.chunk = NULL;
|
|
4503
5098
|
__ctx.ca = ca;
|
|
@@ -4518,7 +5113,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4518
5113
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
4519
5114
|
&& ! ca_has_mask(ca)
|
|
4520
5115
|
&& ca_attach_is_alias(ca)
|
|
4521
|
-
&&
|
|
5116
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
4522
5117
|
int8_t __li_ax = slab_axes[0];
|
|
4523
5118
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
4524
5119
|
ca_size_t __li_INNER = 1;
|
|
@@ -4591,7 +5186,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4591
5186
|
&& ! ca_has_mask(ca)
|
|
4592
5187
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
4593
5188
|
&& ((CAStack *) ca)->k_axis == 0
|
|
4594
|
-
&&
|
|
5189
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
4595
5190
|
CAStack *__li_st = (CAStack *) ca;
|
|
4596
5191
|
int8_t __li_ax = slab_axes[0];
|
|
4597
5192
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -4619,7 +5214,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4619
5214
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
4620
5215
|
const ca_size_t __li_TILE = 512;
|
|
4621
5216
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4622
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
5217
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
4623
5218
|
const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
|
|
4624
5219
|
ca_u_float32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
4625
5220
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -4679,7 +5274,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4679
5274
|
&& ((CAStack *) ca)->k_axis > 0
|
|
4680
5275
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
4681
5276
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
4682
|
-
&&
|
|
5277
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
4683
5278
|
CAStack *__li_st = (CAStack *) ca;
|
|
4684
5279
|
int8_t __li_ax = slab_axes[0];
|
|
4685
5280
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -4736,7 +5331,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4736
5331
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
4737
5332
|
const ca_size_t __li_TILE = 512;
|
|
4738
5333
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4739
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
5334
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
4740
5335
|
const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
|
|
4741
5336
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4742
5337
|
|
|
@@ -4824,7 +5419,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4824
5419
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
4825
5420
|
&& ((CAStack *) ca)->k_axis > 0
|
|
4826
5421
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
4827
|
-
&&
|
|
5422
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
4828
5423
|
CAStack *__li_st = (CAStack *) ca;
|
|
4829
5424
|
int8_t __li_ax = slab_axes[0];
|
|
4830
5425
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -4896,7 +5491,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4896
5491
|
|
|
4897
5492
|
const ca_size_t __li_TILE = 512;
|
|
4898
5493
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
4899
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
5494
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
4900
5495
|
const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
|
|
4901
5496
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
4902
5497
|
|
|
@@ -4945,12 +5540,41 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
4945
5540
|
return vout;
|
|
4946
5541
|
}
|
|
4947
5542
|
}
|
|
5543
|
+
if ( naxes == 1
|
|
5544
|
+
&& ca->ndim >= 2
|
|
5545
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
5546
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
5547
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
5548
|
+
&& ! ca_has_mask(ca)
|
|
5549
|
+
&& ca_attach_is_alias(ca)
|
|
5550
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
5551
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
5552
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
5553
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
5554
|
+
CArray *co;
|
|
5555
|
+
GetCArray(vout, co);
|
|
5556
|
+
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
5557
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
5558
|
+
const ca_u_float32_t *p_root = (const ca_u_float32_t *) ca->ptr;
|
|
5559
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
5560
|
+
const ca_u_float32_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
5561
|
+
ca_u_float32_t acc = (ca_u_float32_t) (0);
|
|
5562
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
5563
|
+
ca_u_float32_t v = __sf_row[__j];
|
|
5564
|
+
acc += v;
|
|
5565
|
+
(void) v;
|
|
5566
|
+
}
|
|
5567
|
+
|
|
5568
|
+
op[__sf_o] = (ca_u_float32_t) (acc);
|
|
5569
|
+
}
|
|
5570
|
+
ca_detach(ca);
|
|
5571
|
+
return vout;
|
|
5572
|
+
}
|
|
4948
5573
|
}
|
|
4949
5574
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
|
|
4950
5575
|
CArray *co;
|
|
4951
5576
|
GetCArray(vout, co);
|
|
4952
5577
|
ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
|
|
4953
|
-
ca_iter_state st;
|
|
4954
5578
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
4955
5579
|
slab_axes, naxes, 0);
|
|
4956
5580
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -4984,7 +5608,22 @@ ca_size_t out_i = 0;
|
|
|
4984
5608
|
return rb_float_new(op[0]);
|
|
4985
5609
|
}
|
|
4986
5610
|
return vout;
|
|
4987
|
-
|
|
5611
|
+
#undef st
|
|
5612
|
+
}
|
|
5613
|
+
|
|
5614
|
+
static VALUE
|
|
5615
|
+
accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
5616
|
+
{
|
|
5617
|
+
accumulate_ki_native_f32_frame fr;
|
|
5618
|
+
fr.self = self;
|
|
5619
|
+
fr.ca = ca;
|
|
5620
|
+
fr.slab_axes = slab_axes;
|
|
5621
|
+
fr.naxes = naxes;
|
|
5622
|
+
fr.keep_axis = keep_axis;
|
|
5623
|
+
fr.min_count = min_count;
|
|
5624
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_f32_body, (VALUE) &fr);
|
|
5625
|
+
}
|
|
5626
|
+
|
|
4988
5627
|
|
|
4989
5628
|
typedef struct {
|
|
4990
5629
|
void *chunk;
|
|
@@ -5039,14 +5678,41 @@ accumulate_ki_stream_f64 (VALUE arg)
|
|
|
5039
5678
|
return rb_float_new((ca_u_float64_t) (acc));
|
|
5040
5679
|
}
|
|
5041
5680
|
|
|
5681
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
5682
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
5683
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
5684
|
+
typedef struct {
|
|
5685
|
+
ca_iter_state states[1];
|
|
5686
|
+
VALUE self;
|
|
5687
|
+
CArray *ca;
|
|
5688
|
+
int8_t *slab_axes;
|
|
5689
|
+
int8_t naxes;
|
|
5690
|
+
int keep_axis;
|
|
5691
|
+
ca_size_t min_count;
|
|
5692
|
+
} accumulate_ki_native_f64_frame;
|
|
5693
|
+
|
|
5042
5694
|
static VALUE
|
|
5043
|
-
|
|
5695
|
+
accumulate_ki_native_f64_body (VALUE arg)
|
|
5044
5696
|
{
|
|
5697
|
+
accumulate_ki_native_f64_frame *fr = (accumulate_ki_native_f64_frame *) arg;
|
|
5698
|
+
VALUE self = fr->self;
|
|
5699
|
+
(void) self;
|
|
5700
|
+
CArray *ca = fr->ca;
|
|
5701
|
+
(void) ca;
|
|
5702
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
5703
|
+
(void) slab_axes;
|
|
5704
|
+
int8_t naxes = fr->naxes;
|
|
5705
|
+
(void) naxes;
|
|
5706
|
+
int keep_axis = fr->keep_axis;
|
|
5707
|
+
(void) keep_axis;
|
|
5708
|
+
ca_size_t min_count = fr->min_count;
|
|
5709
|
+
(void) min_count;
|
|
5710
|
+
#define st (fr->states[0])
|
|
5045
5711
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
5046
5712
|
ca->elements > 0 &&
|
|
5047
5713
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
5048
5714
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
5049
|
-
|
|
5715
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
5050
5716
|
accumulate_ki_stream_f64_t __ctx;
|
|
5051
5717
|
__ctx.chunk = NULL;
|
|
5052
5718
|
__ctx.ca = ca;
|
|
@@ -5067,7 +5733,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5067
5733
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
5068
5734
|
&& ! ca_has_mask(ca)
|
|
5069
5735
|
&& ca_attach_is_alias(ca)
|
|
5070
|
-
&&
|
|
5736
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
5071
5737
|
int8_t __li_ax = slab_axes[0];
|
|
5072
5738
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
5073
5739
|
ca_size_t __li_INNER = 1;
|
|
@@ -5140,7 +5806,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5140
5806
|
&& ! ca_has_mask(ca)
|
|
5141
5807
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
5142
5808
|
&& ((CAStack *) ca)->k_axis == 0
|
|
5143
|
-
&&
|
|
5809
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
5144
5810
|
CAStack *__li_st = (CAStack *) ca;
|
|
5145
5811
|
int8_t __li_ax = slab_axes[0];
|
|
5146
5812
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -5168,7 +5834,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5168
5834
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
5169
5835
|
const ca_size_t __li_TILE = 512;
|
|
5170
5836
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5171
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
5837
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
5172
5838
|
const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
|
|
5173
5839
|
ca_u_float64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
5174
5840
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -5228,7 +5894,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5228
5894
|
&& ((CAStack *) ca)->k_axis > 0
|
|
5229
5895
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
5230
5896
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
5231
|
-
&&
|
|
5897
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
5232
5898
|
CAStack *__li_st = (CAStack *) ca;
|
|
5233
5899
|
int8_t __li_ax = slab_axes[0];
|
|
5234
5900
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -5285,7 +5951,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5285
5951
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5286
5952
|
const ca_size_t __li_TILE = 512;
|
|
5287
5953
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5288
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
5954
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
5289
5955
|
const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
|
|
5290
5956
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
5291
5957
|
|
|
@@ -5373,7 +6039,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5373
6039
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
5374
6040
|
&& ((CAStack *) ca)->k_axis > 0
|
|
5375
6041
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
5376
|
-
&&
|
|
6042
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
5377
6043
|
CAStack *__li_st = (CAStack *) ca;
|
|
5378
6044
|
int8_t __li_ax = slab_axes[0];
|
|
5379
6045
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -5445,7 +6111,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5445
6111
|
|
|
5446
6112
|
const ca_size_t __li_TILE = 512;
|
|
5447
6113
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5448
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
6114
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
5449
6115
|
const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
|
|
5450
6116
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
5451
6117
|
|
|
@@ -5494,12 +6160,41 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
|
|
|
5494
6160
|
return vout;
|
|
5495
6161
|
}
|
|
5496
6162
|
}
|
|
6163
|
+
if ( naxes == 1
|
|
6164
|
+
&& ca->ndim >= 2
|
|
6165
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
6166
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
6167
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
6168
|
+
&& ! ca_has_mask(ca)
|
|
6169
|
+
&& ca_attach_is_alias(ca)
|
|
6170
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
6171
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
6172
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
6173
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
6174
|
+
CArray *co;
|
|
6175
|
+
GetCArray(vout, co);
|
|
6176
|
+
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
6177
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
6178
|
+
const ca_u_float64_t *p_root = (const ca_u_float64_t *) ca->ptr;
|
|
6179
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
6180
|
+
const ca_u_float64_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
6181
|
+
ca_u_float64_t acc = (ca_u_float64_t) (0);
|
|
6182
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
6183
|
+
ca_u_float64_t v = __sf_row[__j];
|
|
6184
|
+
acc += v;
|
|
6185
|
+
(void) v;
|
|
6186
|
+
}
|
|
6187
|
+
|
|
6188
|
+
op[__sf_o] = (ca_u_float64_t) (acc);
|
|
6189
|
+
}
|
|
6190
|
+
ca_detach(ca);
|
|
6191
|
+
return vout;
|
|
6192
|
+
}
|
|
5497
6193
|
}
|
|
5498
6194
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
|
|
5499
6195
|
CArray *co;
|
|
5500
6196
|
GetCArray(vout, co);
|
|
5501
6197
|
ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
|
|
5502
|
-
ca_iter_state st;
|
|
5503
6198
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
5504
6199
|
slab_axes, naxes, 0);
|
|
5505
6200
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -5533,7 +6228,22 @@ ca_size_t out_i = 0;
|
|
|
5533
6228
|
return rb_float_new(op[0]);
|
|
5534
6229
|
}
|
|
5535
6230
|
return vout;
|
|
5536
|
-
|
|
6231
|
+
#undef st
|
|
6232
|
+
}
|
|
6233
|
+
|
|
6234
|
+
static VALUE
|
|
6235
|
+
accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
6236
|
+
{
|
|
6237
|
+
accumulate_ki_native_f64_frame fr;
|
|
6238
|
+
fr.self = self;
|
|
6239
|
+
fr.ca = ca;
|
|
6240
|
+
fr.slab_axes = slab_axes;
|
|
6241
|
+
fr.naxes = naxes;
|
|
6242
|
+
fr.keep_axis = keep_axis;
|
|
6243
|
+
fr.min_count = min_count;
|
|
6244
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_f64_body, (VALUE) &fr);
|
|
6245
|
+
}
|
|
6246
|
+
|
|
5537
6247
|
|
|
5538
6248
|
typedef struct {
|
|
5539
6249
|
void *chunk;
|
|
@@ -5588,20 +6298,47 @@ accumulate_ki_stream_cmplx64 (VALUE arg)
|
|
|
5588
6298
|
return CC2NUM((ca_u_cmplx64_t) (acc));
|
|
5589
6299
|
}
|
|
5590
6300
|
|
|
6301
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
6302
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
6303
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
6304
|
+
typedef struct {
|
|
6305
|
+
ca_iter_state states[1];
|
|
6306
|
+
VALUE self;
|
|
6307
|
+
CArray *ca;
|
|
6308
|
+
int8_t *slab_axes;
|
|
6309
|
+
int8_t naxes;
|
|
6310
|
+
int keep_axis;
|
|
6311
|
+
ca_size_t min_count;
|
|
6312
|
+
} accumulate_ki_native_cmplx64_frame;
|
|
6313
|
+
|
|
5591
6314
|
static VALUE
|
|
5592
|
-
|
|
6315
|
+
accumulate_ki_native_cmplx64_body (VALUE arg)
|
|
5593
6316
|
{
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
6317
|
+
accumulate_ki_native_cmplx64_frame *fr = (accumulate_ki_native_cmplx64_frame *) arg;
|
|
6318
|
+
VALUE self = fr->self;
|
|
6319
|
+
(void) self;
|
|
6320
|
+
CArray *ca = fr->ca;
|
|
6321
|
+
(void) ca;
|
|
6322
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
6323
|
+
(void) slab_axes;
|
|
6324
|
+
int8_t naxes = fr->naxes;
|
|
6325
|
+
(void) naxes;
|
|
6326
|
+
int keep_axis = fr->keep_axis;
|
|
6327
|
+
(void) keep_axis;
|
|
6328
|
+
ca_size_t min_count = fr->min_count;
|
|
6329
|
+
(void) min_count;
|
|
6330
|
+
#define st (fr->states[0])
|
|
6331
|
+
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
6332
|
+
ca->elements > 0 &&
|
|
6333
|
+
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
6334
|
+
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
6335
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
6336
|
+
accumulate_ki_stream_cmplx64_t __ctx;
|
|
6337
|
+
__ctx.chunk = NULL;
|
|
6338
|
+
__ctx.ca = ca;
|
|
6339
|
+
__ctx.min_count = min_count;
|
|
6340
|
+
return ca_lazy_arena_protect(accumulate_ki_stream_cmplx64, ca_lazy_arena_release_held,
|
|
6341
|
+
(VALUE) &__ctx);
|
|
5605
6342
|
}
|
|
5606
6343
|
if ( naxes != 1 || ! ca_too_few_present(ca->dim[slab_axes[0]], min_count, 0) ) {
|
|
5607
6344
|
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
@@ -5616,7 +6353,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5616
6353
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
5617
6354
|
&& ! ca_has_mask(ca)
|
|
5618
6355
|
&& ca_attach_is_alias(ca)
|
|
5619
|
-
&&
|
|
6356
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
5620
6357
|
int8_t __li_ax = slab_axes[0];
|
|
5621
6358
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
5622
6359
|
ca_size_t __li_INNER = 1;
|
|
@@ -5689,7 +6426,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5689
6426
|
&& ! ca_has_mask(ca)
|
|
5690
6427
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
5691
6428
|
&& ((CAStack *) ca)->k_axis == 0
|
|
5692
|
-
&&
|
|
6429
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
5693
6430
|
CAStack *__li_st = (CAStack *) ca;
|
|
5694
6431
|
int8_t __li_ax = slab_axes[0];
|
|
5695
6432
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -5717,7 +6454,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5717
6454
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
5718
6455
|
const ca_size_t __li_TILE = 512;
|
|
5719
6456
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5720
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
6457
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
5721
6458
|
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
|
|
5722
6459
|
ca_u_cmplx64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
5723
6460
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -5777,7 +6514,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5777
6514
|
&& ((CAStack *) ca)->k_axis > 0
|
|
5778
6515
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
5779
6516
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
5780
|
-
&&
|
|
6517
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
5781
6518
|
CAStack *__li_st = (CAStack *) ca;
|
|
5782
6519
|
int8_t __li_ax = slab_axes[0];
|
|
5783
6520
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -5834,7 +6571,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5834
6571
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
5835
6572
|
const ca_size_t __li_TILE = 512;
|
|
5836
6573
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5837
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
6574
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
5838
6575
|
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
|
|
5839
6576
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
5840
6577
|
|
|
@@ -5922,7 +6659,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5922
6659
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
5923
6660
|
&& ((CAStack *) ca)->k_axis > 0
|
|
5924
6661
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
5925
|
-
&&
|
|
6662
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
5926
6663
|
CAStack *__li_st = (CAStack *) ca;
|
|
5927
6664
|
int8_t __li_ax = slab_axes[0];
|
|
5928
6665
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -5994,7 +6731,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
5994
6731
|
|
|
5995
6732
|
const ca_size_t __li_TILE = 512;
|
|
5996
6733
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
5997
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
6734
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
5998
6735
|
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
|
|
5999
6736
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6000
6737
|
|
|
@@ -6043,12 +6780,41 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6043
6780
|
return vout;
|
|
6044
6781
|
}
|
|
6045
6782
|
}
|
|
6783
|
+
if ( naxes == 1
|
|
6784
|
+
&& ca->ndim >= 2
|
|
6785
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
6786
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
6787
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
6788
|
+
&& ! ca_has_mask(ca)
|
|
6789
|
+
&& ca_attach_is_alias(ca)
|
|
6790
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
6791
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
6792
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
6793
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
6794
|
+
CArray *co;
|
|
6795
|
+
GetCArray(vout, co);
|
|
6796
|
+
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
6797
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
6798
|
+
const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) ca->ptr;
|
|
6799
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
6800
|
+
const ca_u_cmplx64_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
6801
|
+
ca_u_cmplx64_t acc = (ca_u_cmplx64_t) (0);
|
|
6802
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
6803
|
+
ca_u_cmplx64_t v = __sf_row[__j];
|
|
6804
|
+
acc += v;
|
|
6805
|
+
(void) v;
|
|
6806
|
+
}
|
|
6807
|
+
|
|
6808
|
+
op[__sf_o] = (ca_u_cmplx64_t) (acc);
|
|
6809
|
+
}
|
|
6810
|
+
ca_detach(ca);
|
|
6811
|
+
return vout;
|
|
6812
|
+
}
|
|
6046
6813
|
}
|
|
6047
6814
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
|
|
6048
6815
|
CArray *co;
|
|
6049
6816
|
GetCArray(vout, co);
|
|
6050
6817
|
ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
|
|
6051
|
-
ca_iter_state st;
|
|
6052
6818
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
6053
6819
|
slab_axes, naxes, 0);
|
|
6054
6820
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -6082,7 +6848,22 @@ ca_size_t out_i = 0;
|
|
|
6082
6848
|
return CC2NUM(op[0]);
|
|
6083
6849
|
}
|
|
6084
6850
|
return vout;
|
|
6085
|
-
|
|
6851
|
+
#undef st
|
|
6852
|
+
}
|
|
6853
|
+
|
|
6854
|
+
static VALUE
|
|
6855
|
+
accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
6856
|
+
{
|
|
6857
|
+
accumulate_ki_native_cmplx64_frame fr;
|
|
6858
|
+
fr.self = self;
|
|
6859
|
+
fr.ca = ca;
|
|
6860
|
+
fr.slab_axes = slab_axes;
|
|
6861
|
+
fr.naxes = naxes;
|
|
6862
|
+
fr.keep_axis = keep_axis;
|
|
6863
|
+
fr.min_count = min_count;
|
|
6864
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_cmplx64_body, (VALUE) &fr);
|
|
6865
|
+
}
|
|
6866
|
+
|
|
6086
6867
|
|
|
6087
6868
|
typedef struct {
|
|
6088
6869
|
void *chunk;
|
|
@@ -6137,14 +6918,41 @@ accumulate_ki_stream_cmplx128 (VALUE arg)
|
|
|
6137
6918
|
return CC2NUM((ca_u_cmplx128_t) (acc));
|
|
6138
6919
|
}
|
|
6139
6920
|
|
|
6921
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
6922
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
6923
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
6924
|
+
typedef struct {
|
|
6925
|
+
ca_iter_state states[1];
|
|
6926
|
+
VALUE self;
|
|
6927
|
+
CArray *ca;
|
|
6928
|
+
int8_t *slab_axes;
|
|
6929
|
+
int8_t naxes;
|
|
6930
|
+
int keep_axis;
|
|
6931
|
+
ca_size_t min_count;
|
|
6932
|
+
} accumulate_ki_native_cmplx128_frame;
|
|
6933
|
+
|
|
6140
6934
|
static VALUE
|
|
6141
|
-
|
|
6935
|
+
accumulate_ki_native_cmplx128_body (VALUE arg)
|
|
6142
6936
|
{
|
|
6937
|
+
accumulate_ki_native_cmplx128_frame *fr = (accumulate_ki_native_cmplx128_frame *) arg;
|
|
6938
|
+
VALUE self = fr->self;
|
|
6939
|
+
(void) self;
|
|
6940
|
+
CArray *ca = fr->ca;
|
|
6941
|
+
(void) ca;
|
|
6942
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
6943
|
+
(void) slab_axes;
|
|
6944
|
+
int8_t naxes = fr->naxes;
|
|
6945
|
+
(void) naxes;
|
|
6946
|
+
int keep_axis = fr->keep_axis;
|
|
6947
|
+
(void) keep_axis;
|
|
6948
|
+
ca_size_t min_count = fr->min_count;
|
|
6949
|
+
(void) min_count;
|
|
6950
|
+
#define st (fr->states[0])
|
|
6143
6951
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
6144
6952
|
ca->elements > 0 &&
|
|
6145
6953
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
6146
6954
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
6147
|
-
|
|
6955
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
6148
6956
|
accumulate_ki_stream_cmplx128_t __ctx;
|
|
6149
6957
|
__ctx.chunk = NULL;
|
|
6150
6958
|
__ctx.ca = ca;
|
|
@@ -6165,7 +6973,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6165
6973
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
6166
6974
|
&& ! ca_has_mask(ca)
|
|
6167
6975
|
&& ca_attach_is_alias(ca)
|
|
6168
|
-
&&
|
|
6976
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
6169
6977
|
int8_t __li_ax = slab_axes[0];
|
|
6170
6978
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
6171
6979
|
ca_size_t __li_INNER = 1;
|
|
@@ -6238,7 +7046,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6238
7046
|
&& ! ca_has_mask(ca)
|
|
6239
7047
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
6240
7048
|
&& ((CAStack *) ca)->k_axis == 0
|
|
6241
|
-
&&
|
|
7049
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
6242
7050
|
CAStack *__li_st = (CAStack *) ca;
|
|
6243
7051
|
int8_t __li_ax = slab_axes[0];
|
|
6244
7052
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -6266,7 +7074,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6266
7074
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
6267
7075
|
const ca_size_t __li_TILE = 512;
|
|
6268
7076
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6269
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
7077
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
6270
7078
|
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
|
|
6271
7079
|
ca_u_cmplx128_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
6272
7080
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -6326,7 +7134,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6326
7134
|
&& ((CAStack *) ca)->k_axis > 0
|
|
6327
7135
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
6328
7136
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
6329
|
-
&&
|
|
7137
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
6330
7138
|
CAStack *__li_st = (CAStack *) ca;
|
|
6331
7139
|
int8_t __li_ax = slab_axes[0];
|
|
6332
7140
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -6383,7 +7191,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6383
7191
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
6384
7192
|
const ca_size_t __li_TILE = 512;
|
|
6385
7193
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6386
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
7194
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
6387
7195
|
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
|
|
6388
7196
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6389
7197
|
|
|
@@ -6471,7 +7279,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6471
7279
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
6472
7280
|
&& ((CAStack *) ca)->k_axis > 0
|
|
6473
7281
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
6474
|
-
&&
|
|
7282
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
6475
7283
|
CAStack *__li_st = (CAStack *) ca;
|
|
6476
7284
|
int8_t __li_ax = slab_axes[0];
|
|
6477
7285
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -6543,7 +7351,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6543
7351
|
|
|
6544
7352
|
const ca_size_t __li_TILE = 512;
|
|
6545
7353
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6546
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
7354
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
6547
7355
|
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
|
|
6548
7356
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6549
7357
|
|
|
@@ -6592,12 +7400,41 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
6592
7400
|
return vout;
|
|
6593
7401
|
}
|
|
6594
7402
|
}
|
|
7403
|
+
if ( naxes == 1
|
|
7404
|
+
&& ca->ndim >= 2
|
|
7405
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
7406
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
7407
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
7408
|
+
&& ! ca_has_mask(ca)
|
|
7409
|
+
&& ca_attach_is_alias(ca)
|
|
7410
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
7411
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
7412
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
7413
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
7414
|
+
CArray *co;
|
|
7415
|
+
GetCArray(vout, co);
|
|
7416
|
+
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
7417
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
7418
|
+
const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) ca->ptr;
|
|
7419
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
7420
|
+
const ca_u_cmplx128_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
7421
|
+
ca_u_cmplx128_t acc = (ca_u_cmplx128_t) (0);
|
|
7422
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
7423
|
+
ca_u_cmplx128_t v = __sf_row[__j];
|
|
7424
|
+
acc += v;
|
|
7425
|
+
(void) v;
|
|
7426
|
+
}
|
|
7427
|
+
|
|
7428
|
+
op[__sf_o] = (ca_u_cmplx128_t) (acc);
|
|
7429
|
+
}
|
|
7430
|
+
ca_detach(ca);
|
|
7431
|
+
return vout;
|
|
7432
|
+
}
|
|
6595
7433
|
}
|
|
6596
7434
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
|
|
6597
7435
|
CArray *co;
|
|
6598
7436
|
GetCArray(vout, co);
|
|
6599
7437
|
ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
|
|
6600
|
-
ca_iter_state st;
|
|
6601
7438
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
6602
7439
|
slab_axes, naxes, 0);
|
|
6603
7440
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -6631,7 +7468,22 @@ ca_size_t out_i = 0;
|
|
|
6631
7468
|
return CC2NUM(op[0]);
|
|
6632
7469
|
}
|
|
6633
7470
|
return vout;
|
|
6634
|
-
|
|
7471
|
+
#undef st
|
|
7472
|
+
}
|
|
7473
|
+
|
|
7474
|
+
static VALUE
|
|
7475
|
+
accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
7476
|
+
{
|
|
7477
|
+
accumulate_ki_native_cmplx128_frame fr;
|
|
7478
|
+
fr.self = self;
|
|
7479
|
+
fr.ca = ca;
|
|
7480
|
+
fr.slab_axes = slab_axes;
|
|
7481
|
+
fr.naxes = naxes;
|
|
7482
|
+
fr.keep_axis = keep_axis;
|
|
7483
|
+
fr.min_count = min_count;
|
|
7484
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_cmplx128_body, (VALUE) &fr);
|
|
7485
|
+
}
|
|
7486
|
+
|
|
6635
7487
|
|
|
6636
7488
|
typedef struct {
|
|
6637
7489
|
void *chunk;
|
|
@@ -6686,14 +7538,41 @@ accumulate_ki_stream_bool (VALUE arg)
|
|
|
6686
7538
|
return INT2NUM((boolean8_t) (acc));
|
|
6687
7539
|
}
|
|
6688
7540
|
|
|
7541
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
7542
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
7543
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
7544
|
+
typedef struct {
|
|
7545
|
+
ca_iter_state states[1];
|
|
7546
|
+
VALUE self;
|
|
7547
|
+
CArray *ca;
|
|
7548
|
+
int8_t *slab_axes;
|
|
7549
|
+
int8_t naxes;
|
|
7550
|
+
int keep_axis;
|
|
7551
|
+
ca_size_t min_count;
|
|
7552
|
+
} accumulate_ki_native_bool_frame;
|
|
7553
|
+
|
|
6689
7554
|
static VALUE
|
|
6690
|
-
|
|
7555
|
+
accumulate_ki_native_bool_body (VALUE arg)
|
|
6691
7556
|
{
|
|
7557
|
+
accumulate_ki_native_bool_frame *fr = (accumulate_ki_native_bool_frame *) arg;
|
|
7558
|
+
VALUE self = fr->self;
|
|
7559
|
+
(void) self;
|
|
7560
|
+
CArray *ca = fr->ca;
|
|
7561
|
+
(void) ca;
|
|
7562
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
7563
|
+
(void) slab_axes;
|
|
7564
|
+
int8_t naxes = fr->naxes;
|
|
7565
|
+
(void) naxes;
|
|
7566
|
+
int keep_axis = fr->keep_axis;
|
|
7567
|
+
(void) keep_axis;
|
|
7568
|
+
ca_size_t min_count = fr->min_count;
|
|
7569
|
+
(void) min_count;
|
|
7570
|
+
#define st (fr->states[0])
|
|
6692
7571
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
6693
7572
|
ca->elements > 0 &&
|
|
6694
7573
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
6695
7574
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
6696
|
-
|
|
7575
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
6697
7576
|
accumulate_ki_stream_bool_t __ctx;
|
|
6698
7577
|
__ctx.chunk = NULL;
|
|
6699
7578
|
__ctx.ca = ca;
|
|
@@ -6714,7 +7593,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
6714
7593
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
6715
7594
|
&& ! ca_has_mask(ca)
|
|
6716
7595
|
&& ca_attach_is_alias(ca)
|
|
6717
|
-
&&
|
|
7596
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
6718
7597
|
int8_t __li_ax = slab_axes[0];
|
|
6719
7598
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
6720
7599
|
ca_size_t __li_INNER = 1;
|
|
@@ -6787,7 +7666,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
6787
7666
|
&& ! ca_has_mask(ca)
|
|
6788
7667
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
6789
7668
|
&& ((CAStack *) ca)->k_axis == 0
|
|
6790
|
-
&&
|
|
7669
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
6791
7670
|
CAStack *__li_st = (CAStack *) ca;
|
|
6792
7671
|
int8_t __li_ax = slab_axes[0];
|
|
6793
7672
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -6815,7 +7694,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
6815
7694
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
6816
7695
|
const ca_size_t __li_TILE = 512;
|
|
6817
7696
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6818
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
7697
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
6819
7698
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
6820
7699
|
boolean8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
6821
7700
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -6875,7 +7754,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
6875
7754
|
&& ((CAStack *) ca)->k_axis > 0
|
|
6876
7755
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
6877
7756
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
6878
|
-
&&
|
|
7757
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
6879
7758
|
CAStack *__li_st = (CAStack *) ca;
|
|
6880
7759
|
int8_t __li_ax = slab_axes[0];
|
|
6881
7760
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -6932,7 +7811,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
6932
7811
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
6933
7812
|
const ca_size_t __li_TILE = 512;
|
|
6934
7813
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
6935
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
7814
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
6936
7815
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
6937
7816
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
6938
7817
|
|
|
@@ -7020,7 +7899,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7020
7899
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
7021
7900
|
&& ((CAStack *) ca)->k_axis > 0
|
|
7022
7901
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
7023
|
-
&&
|
|
7902
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
7024
7903
|
CAStack *__li_st = (CAStack *) ca;
|
|
7025
7904
|
int8_t __li_ax = slab_axes[0];
|
|
7026
7905
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -7092,7 +7971,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7092
7971
|
|
|
7093
7972
|
const ca_size_t __li_TILE = 512;
|
|
7094
7973
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
7095
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
7974
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
7096
7975
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
7097
7976
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
7098
7977
|
|
|
@@ -7141,12 +8020,41 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7141
8020
|
return vout;
|
|
7142
8021
|
}
|
|
7143
8022
|
}
|
|
8023
|
+
if ( naxes == 1
|
|
8024
|
+
&& ca->ndim >= 2
|
|
8025
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
8026
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
8027
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
8028
|
+
&& ! ca_has_mask(ca)
|
|
8029
|
+
&& ca_attach_is_alias(ca)
|
|
8030
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
8031
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
8032
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
8033
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
8034
|
+
CArray *co;
|
|
8035
|
+
GetCArray(vout, co);
|
|
8036
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
8037
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
8038
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
8039
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
8040
|
+
const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
8041
|
+
boolean8_t acc = (boolean8_t) (0);
|
|
8042
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
8043
|
+
boolean8_t v = __sf_row[__j];
|
|
8044
|
+
acc ^= v;
|
|
8045
|
+
(void) v;
|
|
8046
|
+
}
|
|
8047
|
+
|
|
8048
|
+
op[__sf_o] = (boolean8_t) (acc);
|
|
8049
|
+
}
|
|
8050
|
+
ca_detach(ca);
|
|
8051
|
+
return vout;
|
|
8052
|
+
}
|
|
7144
8053
|
}
|
|
7145
8054
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
7146
8055
|
CArray *co;
|
|
7147
8056
|
GetCArray(vout, co);
|
|
7148
8057
|
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
7149
|
-
ca_iter_state st;
|
|
7150
8058
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
7151
8059
|
slab_axes, naxes, 0);
|
|
7152
8060
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -7178,7 +8086,22 @@ ca_size_t out_i = 0;
|
|
|
7178
8086
|
return INT2NUM(op[0]);
|
|
7179
8087
|
}
|
|
7180
8088
|
return vout;
|
|
7181
|
-
|
|
8089
|
+
#undef st
|
|
8090
|
+
}
|
|
8091
|
+
|
|
8092
|
+
static VALUE
|
|
8093
|
+
accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
8094
|
+
{
|
|
8095
|
+
accumulate_ki_native_bool_frame fr;
|
|
8096
|
+
fr.self = self;
|
|
8097
|
+
fr.ca = ca;
|
|
8098
|
+
fr.slab_axes = slab_axes;
|
|
8099
|
+
fr.naxes = naxes;
|
|
8100
|
+
fr.keep_axis = keep_axis;
|
|
8101
|
+
fr.min_count = min_count;
|
|
8102
|
+
return ca_iter_ensure(1, fr.states, accumulate_ki_native_bool_body, (VALUE) &fr);
|
|
8103
|
+
}
|
|
8104
|
+
|
|
7182
8105
|
|
|
7183
8106
|
typedef struct {
|
|
7184
8107
|
void *chunk;
|
|
@@ -7233,9 +8156,9 @@ accumulate_ki_stream_object (VALUE arg)
|
|
|
7233
8156
|
return (VALUE)((VALUE) (acc));
|
|
7234
8157
|
}
|
|
7235
8158
|
|
|
7236
|
-
/* The
|
|
7237
|
-
|
|
7238
|
-
finishes them however the body leaves. */
|
|
8159
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
8160
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
8161
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
7239
8162
|
typedef struct {
|
|
7240
8163
|
ca_iter_state states[1];
|
|
7241
8164
|
VALUE self;
|
|
@@ -7267,7 +8190,7 @@ accumulate_ki_native_object_body (VALUE arg)
|
|
|
7267
8190
|
ca->elements > 0 &&
|
|
7268
8191
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
7269
8192
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
7270
|
-
|
|
8193
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
7271
8194
|
accumulate_ki_stream_object_t __ctx;
|
|
7272
8195
|
__ctx.chunk = NULL;
|
|
7273
8196
|
__ctx.ca = ca;
|
|
@@ -7323,7 +8246,7 @@ accumulate_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t n
|
|
|
7323
8246
|
fr.naxes = naxes;
|
|
7324
8247
|
fr.keep_axis = keep_axis;
|
|
7325
8248
|
fr.min_count = min_count;
|
|
7326
|
-
return
|
|
8249
|
+
return ca_iter_ensure_computing(ca, 1, fr.states, accumulate_ki_native_object_body, (VALUE) &fr);
|
|
7327
8250
|
}
|
|
7328
8251
|
|
|
7329
8252
|
|
|
@@ -7492,14 +8415,41 @@ count_true_ki_stream_bool (VALUE arg)
|
|
|
7492
8415
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
7493
8416
|
}
|
|
7494
8417
|
|
|
8418
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
8419
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
8420
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
8421
|
+
typedef struct {
|
|
8422
|
+
ca_iter_state states[1];
|
|
8423
|
+
VALUE self;
|
|
8424
|
+
CArray *ca;
|
|
8425
|
+
int8_t *slab_axes;
|
|
8426
|
+
int8_t naxes;
|
|
8427
|
+
int keep_axis;
|
|
8428
|
+
ca_size_t min_count;
|
|
8429
|
+
} count_true_ki_native_bool_frame;
|
|
8430
|
+
|
|
7495
8431
|
static VALUE
|
|
7496
|
-
|
|
8432
|
+
count_true_ki_native_bool_body (VALUE arg)
|
|
7497
8433
|
{
|
|
8434
|
+
count_true_ki_native_bool_frame *fr = (count_true_ki_native_bool_frame *) arg;
|
|
8435
|
+
VALUE self = fr->self;
|
|
8436
|
+
(void) self;
|
|
8437
|
+
CArray *ca = fr->ca;
|
|
8438
|
+
(void) ca;
|
|
8439
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
8440
|
+
(void) slab_axes;
|
|
8441
|
+
int8_t naxes = fr->naxes;
|
|
8442
|
+
(void) naxes;
|
|
8443
|
+
int keep_axis = fr->keep_axis;
|
|
8444
|
+
(void) keep_axis;
|
|
8445
|
+
ca_size_t min_count = fr->min_count;
|
|
8446
|
+
(void) min_count;
|
|
8447
|
+
#define st (fr->states[0])
|
|
7498
8448
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
7499
8449
|
ca->elements > 0 &&
|
|
7500
8450
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
7501
8451
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
7502
|
-
|
|
8452
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
7503
8453
|
count_true_ki_stream_bool_t __ctx;
|
|
7504
8454
|
__ctx.chunk = NULL;
|
|
7505
8455
|
__ctx.ca = ca;
|
|
@@ -7520,7 +8470,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7520
8470
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
7521
8471
|
&& ! ca_has_mask(ca)
|
|
7522
8472
|
&& ca_attach_is_alias(ca)
|
|
7523
|
-
&&
|
|
8473
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
7524
8474
|
int8_t __li_ax = slab_axes[0];
|
|
7525
8475
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
7526
8476
|
ca_size_t __li_INNER = 1;
|
|
@@ -7593,7 +8543,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7593
8543
|
&& ! ca_has_mask(ca)
|
|
7594
8544
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
7595
8545
|
&& ((CAStack *) ca)->k_axis == 0
|
|
7596
|
-
&&
|
|
8546
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
7597
8547
|
CAStack *__li_st = (CAStack *) ca;
|
|
7598
8548
|
int8_t __li_ax = slab_axes[0];
|
|
7599
8549
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -7621,7 +8571,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7621
8571
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
7622
8572
|
const ca_size_t __li_TILE = 512;
|
|
7623
8573
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
7624
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
8574
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
7625
8575
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
7626
8576
|
ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
7627
8577
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -7681,7 +8631,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7681
8631
|
&& ((CAStack *) ca)->k_axis > 0
|
|
7682
8632
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
7683
8633
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
7684
|
-
&&
|
|
8634
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
7685
8635
|
CAStack *__li_st = (CAStack *) ca;
|
|
7686
8636
|
int8_t __li_ax = slab_axes[0];
|
|
7687
8637
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -7738,7 +8688,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7738
8688
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
7739
8689
|
const ca_size_t __li_TILE = 512;
|
|
7740
8690
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
7741
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
8691
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
7742
8692
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
7743
8693
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
7744
8694
|
|
|
@@ -7826,7 +8776,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7826
8776
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
7827
8777
|
&& ((CAStack *) ca)->k_axis > 0
|
|
7828
8778
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
7829
|
-
&&
|
|
8779
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
7830
8780
|
CAStack *__li_st = (CAStack *) ca;
|
|
7831
8781
|
int8_t __li_ax = slab_axes[0];
|
|
7832
8782
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -7898,7 +8848,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7898
8848
|
|
|
7899
8849
|
const ca_size_t __li_TILE = 512;
|
|
7900
8850
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
7901
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
8851
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
7902
8852
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
7903
8853
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
7904
8854
|
|
|
@@ -7947,12 +8897,41 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
|
|
|
7947
8897
|
return vout;
|
|
7948
8898
|
}
|
|
7949
8899
|
}
|
|
8900
|
+
if ( naxes == 1
|
|
8901
|
+
&& ca->ndim >= 2
|
|
8902
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
8903
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
8904
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
8905
|
+
&& ! ca_has_mask(ca)
|
|
8906
|
+
&& ca_attach_is_alias(ca)
|
|
8907
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
8908
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
8909
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
8910
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8911
|
+
CArray *co;
|
|
8912
|
+
GetCArray(vout, co);
|
|
8913
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8914
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
8915
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
8916
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
8917
|
+
const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
8918
|
+
ca_u_int64_t acc = (ca_u_int64_t) (0);
|
|
8919
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
8920
|
+
boolean8_t v = __sf_row[__j];
|
|
8921
|
+
acc += v;
|
|
8922
|
+
(void) v;
|
|
8923
|
+
}
|
|
8924
|
+
|
|
8925
|
+
op[__sf_o] = (ca_u_int64_t) (acc);
|
|
8926
|
+
}
|
|
8927
|
+
ca_detach(ca);
|
|
8928
|
+
return vout;
|
|
8929
|
+
}
|
|
7950
8930
|
}
|
|
7951
8931
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
7952
8932
|
CArray *co;
|
|
7953
8933
|
GetCArray(vout, co);
|
|
7954
8934
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
7955
|
-
ca_iter_state st;
|
|
7956
8935
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
7957
8936
|
slab_axes, naxes, 0);
|
|
7958
8937
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -7986,7 +8965,22 @@ ca_size_t out_i = 0;
|
|
|
7986
8965
|
return LL2NUM(op[0]);
|
|
7987
8966
|
}
|
|
7988
8967
|
return vout;
|
|
7989
|
-
|
|
8968
|
+
#undef st
|
|
8969
|
+
}
|
|
8970
|
+
|
|
8971
|
+
static VALUE
|
|
8972
|
+
count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
8973
|
+
{
|
|
8974
|
+
count_true_ki_native_bool_frame fr;
|
|
8975
|
+
fr.self = self;
|
|
8976
|
+
fr.ca = ca;
|
|
8977
|
+
fr.slab_axes = slab_axes;
|
|
8978
|
+
fr.naxes = naxes;
|
|
8979
|
+
fr.keep_axis = keep_axis;
|
|
8980
|
+
fr.min_count = min_count;
|
|
8981
|
+
return ca_iter_ensure(1, fr.states, count_true_ki_native_bool_body, (VALUE) &fr);
|
|
8982
|
+
}
|
|
8983
|
+
|
|
7990
8984
|
|
|
7991
8985
|
VALUE
|
|
7992
8986
|
rb_ca_count_true_ki (int argc, VALUE *argv, VALUE self)
|
|
@@ -8088,14 +9082,41 @@ count_false_ki_stream_bool (VALUE arg)
|
|
|
8088
9082
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
8089
9083
|
}
|
|
8090
9084
|
|
|
9085
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
9086
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
9087
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
9088
|
+
typedef struct {
|
|
9089
|
+
ca_iter_state states[1];
|
|
9090
|
+
VALUE self;
|
|
9091
|
+
CArray *ca;
|
|
9092
|
+
int8_t *slab_axes;
|
|
9093
|
+
int8_t naxes;
|
|
9094
|
+
int keep_axis;
|
|
9095
|
+
ca_size_t min_count;
|
|
9096
|
+
} count_false_ki_native_bool_frame;
|
|
9097
|
+
|
|
8091
9098
|
static VALUE
|
|
8092
|
-
|
|
9099
|
+
count_false_ki_native_bool_body (VALUE arg)
|
|
8093
9100
|
{
|
|
9101
|
+
count_false_ki_native_bool_frame *fr = (count_false_ki_native_bool_frame *) arg;
|
|
9102
|
+
VALUE self = fr->self;
|
|
9103
|
+
(void) self;
|
|
9104
|
+
CArray *ca = fr->ca;
|
|
9105
|
+
(void) ca;
|
|
9106
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
9107
|
+
(void) slab_axes;
|
|
9108
|
+
int8_t naxes = fr->naxes;
|
|
9109
|
+
(void) naxes;
|
|
9110
|
+
int keep_axis = fr->keep_axis;
|
|
9111
|
+
(void) keep_axis;
|
|
9112
|
+
ca_size_t min_count = fr->min_count;
|
|
9113
|
+
(void) min_count;
|
|
9114
|
+
#define st (fr->states[0])
|
|
8094
9115
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
8095
9116
|
ca->elements > 0 &&
|
|
8096
9117
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
8097
9118
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
8098
|
-
|
|
9119
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
8099
9120
|
count_false_ki_stream_bool_t __ctx;
|
|
8100
9121
|
__ctx.chunk = NULL;
|
|
8101
9122
|
__ctx.ca = ca;
|
|
@@ -8116,7 +9137,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8116
9137
|
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
8117
9138
|
&& ! ca_has_mask(ca)
|
|
8118
9139
|
&& ca_attach_is_alias(ca)
|
|
8119
|
-
&&
|
|
9140
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
8120
9141
|
int8_t __li_ax = slab_axes[0];
|
|
8121
9142
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
8122
9143
|
ca_size_t __li_INNER = 1;
|
|
@@ -8189,7 +9210,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8189
9210
|
&& ! ca_has_mask(ca)
|
|
8190
9211
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
8191
9212
|
&& ((CAStack *) ca)->k_axis == 0
|
|
8192
|
-
&&
|
|
9213
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
8193
9214
|
CAStack *__li_st = (CAStack *) ca;
|
|
8194
9215
|
int8_t __li_ax = slab_axes[0];
|
|
8195
9216
|
ca_size_t __li_M = ca->dim[__li_ax];
|
|
@@ -8217,7 +9238,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8217
9238
|
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
8218
9239
|
const ca_size_t __li_TILE = 512;
|
|
8219
9240
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
8220
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
9241
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
8221
9242
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
8222
9243
|
ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
8223
9244
|
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
@@ -8277,7 +9298,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8277
9298
|
&& ((CAStack *) ca)->k_axis > 0
|
|
8278
9299
|
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
8279
9300
|
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
8280
|
-
&&
|
|
9301
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
8281
9302
|
CAStack *__li_st = (CAStack *) ca;
|
|
8282
9303
|
int8_t __li_ax = slab_axes[0];
|
|
8283
9304
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -8334,7 +9355,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8334
9355
|
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
8335
9356
|
const ca_size_t __li_TILE = 512;
|
|
8336
9357
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
8337
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
9358
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
8338
9359
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
8339
9360
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
8340
9361
|
|
|
@@ -8422,7 +9443,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8422
9443
|
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
8423
9444
|
&& ((CAStack *) ca)->k_axis > 0
|
|
8424
9445
|
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
8425
|
-
&&
|
|
9446
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
8426
9447
|
CAStack *__li_st = (CAStack *) ca;
|
|
8427
9448
|
int8_t __li_ax = slab_axes[0];
|
|
8428
9449
|
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
@@ -8494,7 +9515,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8494
9515
|
|
|
8495
9516
|
const ca_size_t __li_TILE = 512;
|
|
8496
9517
|
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
8497
|
-
ca_attach(__li_st->parents[__kk]); /* window:
|
|
9518
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
8498
9519
|
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
8499
9520
|
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
8500
9521
|
|
|
@@ -8543,12 +9564,41 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
|
|
|
8543
9564
|
return vout;
|
|
8544
9565
|
}
|
|
8545
9566
|
}
|
|
9567
|
+
if ( naxes == 1
|
|
9568
|
+
&& ca->ndim >= 2
|
|
9569
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
9570
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
9571
|
+
&& ca->dim[ca->ndim - 1] <= 32
|
|
9572
|
+
&& ! ca_has_mask(ca)
|
|
9573
|
+
&& ca_attach_is_alias(ca)
|
|
9574
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
9575
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
9576
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
9577
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9578
|
+
CArray *co;
|
|
9579
|
+
GetCArray(vout, co);
|
|
9580
|
+
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9581
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
9582
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
9583
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
9584
|
+
const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
9585
|
+
ca_u_int64_t acc = (ca_u_int64_t) (0);
|
|
9586
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
9587
|
+
boolean8_t v = __sf_row[__j];
|
|
9588
|
+
acc += !v;
|
|
9589
|
+
(void) v;
|
|
9590
|
+
}
|
|
9591
|
+
|
|
9592
|
+
op[__sf_o] = (ca_u_int64_t) (acc);
|
|
9593
|
+
}
|
|
9594
|
+
ca_detach(ca);
|
|
9595
|
+
return vout;
|
|
9596
|
+
}
|
|
8546
9597
|
}
|
|
8547
9598
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8548
9599
|
CArray *co;
|
|
8549
9600
|
GetCArray(vout, co);
|
|
8550
9601
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8551
|
-
ca_iter_state st;
|
|
8552
9602
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8553
9603
|
slab_axes, naxes, 0);
|
|
8554
9604
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -8580,7 +9630,22 @@ ca_size_t out_i = 0;
|
|
|
8580
9630
|
return LL2NUM(op[0]);
|
|
8581
9631
|
}
|
|
8582
9632
|
return vout;
|
|
8583
|
-
|
|
9633
|
+
#undef st
|
|
9634
|
+
}
|
|
9635
|
+
|
|
9636
|
+
static VALUE
|
|
9637
|
+
count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
|
|
9638
|
+
{
|
|
9639
|
+
count_false_ki_native_bool_frame fr;
|
|
9640
|
+
fr.self = self;
|
|
9641
|
+
fr.ca = ca;
|
|
9642
|
+
fr.slab_axes = slab_axes;
|
|
9643
|
+
fr.naxes = naxes;
|
|
9644
|
+
fr.keep_axis = keep_axis;
|
|
9645
|
+
fr.min_count = min_count;
|
|
9646
|
+
return ca_iter_ensure(1, fr.states, count_false_ki_native_bool_body, (VALUE) &fr);
|
|
9647
|
+
}
|
|
9648
|
+
|
|
8584
9649
|
|
|
8585
9650
|
VALUE
|
|
8586
9651
|
rb_ca_count_false_ki (int argc, VALUE *argv, VALUE self)
|
|
@@ -8685,27 +9750,56 @@ count_equal_ki_stream_i8 (VALUE arg)
|
|
|
8685
9750
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
8686
9751
|
}
|
|
8687
9752
|
|
|
9753
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
9754
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
9755
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
9756
|
+
typedef struct {
|
|
9757
|
+
ca_iter_state states[1];
|
|
9758
|
+
VALUE self;
|
|
9759
|
+
CArray *ca;
|
|
9760
|
+
int8_t *slab_axes;
|
|
9761
|
+
int8_t naxes;
|
|
9762
|
+
int keep_axis;
|
|
9763
|
+
int8_t value_arg;
|
|
9764
|
+
ca_size_t min_count;
|
|
9765
|
+
} count_equal_ki_native_i8_frame;
|
|
9766
|
+
|
|
8688
9767
|
static VALUE
|
|
8689
|
-
|
|
9768
|
+
count_equal_ki_native_i8_body (VALUE arg)
|
|
8690
9769
|
{
|
|
8691
|
-
|
|
8692
|
-
|
|
8693
|
-
|
|
8694
|
-
|
|
8695
|
-
|
|
8696
|
-
|
|
8697
|
-
|
|
8698
|
-
|
|
8699
|
-
|
|
8700
|
-
|
|
8701
|
-
|
|
9770
|
+
count_equal_ki_native_i8_frame *fr = (count_equal_ki_native_i8_frame *) arg;
|
|
9771
|
+
VALUE self = fr->self;
|
|
9772
|
+
(void) self;
|
|
9773
|
+
CArray *ca = fr->ca;
|
|
9774
|
+
(void) ca;
|
|
9775
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
9776
|
+
(void) slab_axes;
|
|
9777
|
+
int8_t naxes = fr->naxes;
|
|
9778
|
+
(void) naxes;
|
|
9779
|
+
int keep_axis = fr->keep_axis;
|
|
9780
|
+
(void) keep_axis;
|
|
9781
|
+
int8_t value_arg = fr->value_arg;
|
|
9782
|
+
(void) value_arg;
|
|
9783
|
+
ca_size_t min_count = fr->min_count;
|
|
9784
|
+
(void) min_count;
|
|
9785
|
+
#define st (fr->states[0])
|
|
9786
|
+
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9787
|
+
ca->elements > 0 &&
|
|
9788
|
+
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9789
|
+
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9790
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9791
|
+
count_equal_ki_stream_i8_t __ctx;
|
|
9792
|
+
__ctx.chunk = NULL;
|
|
9793
|
+
__ctx.ca = ca;
|
|
9794
|
+
__ctx.value_arg = value_arg;
|
|
9795
|
+
__ctx.min_count = min_count;
|
|
9796
|
+
return ca_lazy_arena_protect(count_equal_ki_stream_i8, ca_lazy_arena_release_held,
|
|
8702
9797
|
(VALUE) &__ctx);
|
|
8703
9798
|
}
|
|
8704
9799
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
8705
9800
|
CArray *co;
|
|
8706
9801
|
GetCArray(vout, co);
|
|
8707
9802
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8708
|
-
ca_iter_state st;
|
|
8709
9803
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8710
9804
|
slab_axes, naxes, 0);
|
|
8711
9805
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -8737,7 +9831,23 @@ ca_size_t out_i = 0;
|
|
|
8737
9831
|
return LL2NUM(op[0]);
|
|
8738
9832
|
}
|
|
8739
9833
|
return vout;
|
|
8740
|
-
|
|
9834
|
+
#undef st
|
|
9835
|
+
}
|
|
9836
|
+
|
|
9837
|
+
static VALUE
|
|
9838
|
+
count_equal_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, int8_t value_arg, ca_size_t min_count)
|
|
9839
|
+
{
|
|
9840
|
+
count_equal_ki_native_i8_frame fr;
|
|
9841
|
+
fr.self = self;
|
|
9842
|
+
fr.ca = ca;
|
|
9843
|
+
fr.slab_axes = slab_axes;
|
|
9844
|
+
fr.naxes = naxes;
|
|
9845
|
+
fr.keep_axis = keep_axis;
|
|
9846
|
+
fr.value_arg = value_arg;
|
|
9847
|
+
fr.min_count = min_count;
|
|
9848
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_i8_body, (VALUE) &fr);
|
|
9849
|
+
}
|
|
9850
|
+
|
|
8741
9851
|
|
|
8742
9852
|
typedef struct {
|
|
8743
9853
|
void *chunk;
|
|
@@ -8795,14 +9905,44 @@ count_equal_ki_stream_u8 (VALUE arg)
|
|
|
8795
9905
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
8796
9906
|
}
|
|
8797
9907
|
|
|
9908
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
9909
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
9910
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
9911
|
+
typedef struct {
|
|
9912
|
+
ca_iter_state states[1];
|
|
9913
|
+
VALUE self;
|
|
9914
|
+
CArray *ca;
|
|
9915
|
+
int8_t *slab_axes;
|
|
9916
|
+
int8_t naxes;
|
|
9917
|
+
int keep_axis;
|
|
9918
|
+
uint8_t value_arg;
|
|
9919
|
+
ca_size_t min_count;
|
|
9920
|
+
} count_equal_ki_native_u8_frame;
|
|
9921
|
+
|
|
8798
9922
|
static VALUE
|
|
8799
|
-
|
|
9923
|
+
count_equal_ki_native_u8_body (VALUE arg)
|
|
8800
9924
|
{
|
|
9925
|
+
count_equal_ki_native_u8_frame *fr = (count_equal_ki_native_u8_frame *) arg;
|
|
9926
|
+
VALUE self = fr->self;
|
|
9927
|
+
(void) self;
|
|
9928
|
+
CArray *ca = fr->ca;
|
|
9929
|
+
(void) ca;
|
|
9930
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
9931
|
+
(void) slab_axes;
|
|
9932
|
+
int8_t naxes = fr->naxes;
|
|
9933
|
+
(void) naxes;
|
|
9934
|
+
int keep_axis = fr->keep_axis;
|
|
9935
|
+
(void) keep_axis;
|
|
9936
|
+
uint8_t value_arg = fr->value_arg;
|
|
9937
|
+
(void) value_arg;
|
|
9938
|
+
ca_size_t min_count = fr->min_count;
|
|
9939
|
+
(void) min_count;
|
|
9940
|
+
#define st (fr->states[0])
|
|
8801
9941
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
8802
9942
|
ca->elements > 0 &&
|
|
8803
9943
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
8804
9944
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
8805
|
-
|
|
9945
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
8806
9946
|
count_equal_ki_stream_u8_t __ctx;
|
|
8807
9947
|
__ctx.chunk = NULL;
|
|
8808
9948
|
__ctx.ca = ca;
|
|
@@ -8815,7 +9955,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
8815
9955
|
CArray *co;
|
|
8816
9956
|
GetCArray(vout, co);
|
|
8817
9957
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8818
|
-
ca_iter_state st;
|
|
8819
9958
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8820
9959
|
slab_axes, naxes, 0);
|
|
8821
9960
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -8847,7 +9986,23 @@ ca_size_t out_i = 0;
|
|
|
8847
9986
|
return LL2NUM(op[0]);
|
|
8848
9987
|
}
|
|
8849
9988
|
return vout;
|
|
8850
|
-
|
|
9989
|
+
#undef st
|
|
9990
|
+
}
|
|
9991
|
+
|
|
9992
|
+
static VALUE
|
|
9993
|
+
count_equal_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, uint8_t value_arg, ca_size_t min_count)
|
|
9994
|
+
{
|
|
9995
|
+
count_equal_ki_native_u8_frame fr;
|
|
9996
|
+
fr.self = self;
|
|
9997
|
+
fr.ca = ca;
|
|
9998
|
+
fr.slab_axes = slab_axes;
|
|
9999
|
+
fr.naxes = naxes;
|
|
10000
|
+
fr.keep_axis = keep_axis;
|
|
10001
|
+
fr.value_arg = value_arg;
|
|
10002
|
+
fr.min_count = min_count;
|
|
10003
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_u8_body, (VALUE) &fr);
|
|
10004
|
+
}
|
|
10005
|
+
|
|
8851
10006
|
|
|
8852
10007
|
typedef struct {
|
|
8853
10008
|
void *chunk;
|
|
@@ -8905,14 +10060,44 @@ count_equal_ki_stream_i16 (VALUE arg)
|
|
|
8905
10060
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
8906
10061
|
}
|
|
8907
10062
|
|
|
10063
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10064
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10065
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10066
|
+
typedef struct {
|
|
10067
|
+
ca_iter_state states[1];
|
|
10068
|
+
VALUE self;
|
|
10069
|
+
CArray *ca;
|
|
10070
|
+
int8_t *slab_axes;
|
|
10071
|
+
int8_t naxes;
|
|
10072
|
+
int keep_axis;
|
|
10073
|
+
ca_u_int16_t value_arg;
|
|
10074
|
+
ca_size_t min_count;
|
|
10075
|
+
} count_equal_ki_native_i16_frame;
|
|
10076
|
+
|
|
8908
10077
|
static VALUE
|
|
8909
|
-
|
|
10078
|
+
count_equal_ki_native_i16_body (VALUE arg)
|
|
8910
10079
|
{
|
|
10080
|
+
count_equal_ki_native_i16_frame *fr = (count_equal_ki_native_i16_frame *) arg;
|
|
10081
|
+
VALUE self = fr->self;
|
|
10082
|
+
(void) self;
|
|
10083
|
+
CArray *ca = fr->ca;
|
|
10084
|
+
(void) ca;
|
|
10085
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
10086
|
+
(void) slab_axes;
|
|
10087
|
+
int8_t naxes = fr->naxes;
|
|
10088
|
+
(void) naxes;
|
|
10089
|
+
int keep_axis = fr->keep_axis;
|
|
10090
|
+
(void) keep_axis;
|
|
10091
|
+
ca_u_int16_t value_arg = fr->value_arg;
|
|
10092
|
+
(void) value_arg;
|
|
10093
|
+
ca_size_t min_count = fr->min_count;
|
|
10094
|
+
(void) min_count;
|
|
10095
|
+
#define st (fr->states[0])
|
|
8911
10096
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
8912
10097
|
ca->elements > 0 &&
|
|
8913
10098
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
8914
10099
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
8915
|
-
|
|
10100
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
8916
10101
|
count_equal_ki_stream_i16_t __ctx;
|
|
8917
10102
|
__ctx.chunk = NULL;
|
|
8918
10103
|
__ctx.ca = ca;
|
|
@@ -8925,7 +10110,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
8925
10110
|
CArray *co;
|
|
8926
10111
|
GetCArray(vout, co);
|
|
8927
10112
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
8928
|
-
ca_iter_state st;
|
|
8929
10113
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
8930
10114
|
slab_axes, naxes, 0);
|
|
8931
10115
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -8957,7 +10141,23 @@ ca_size_t out_i = 0;
|
|
|
8957
10141
|
return LL2NUM(op[0]);
|
|
8958
10142
|
}
|
|
8959
10143
|
return vout;
|
|
8960
|
-
|
|
10144
|
+
#undef st
|
|
10145
|
+
}
|
|
10146
|
+
|
|
10147
|
+
static VALUE
|
|
10148
|
+
count_equal_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int16_t value_arg, ca_size_t min_count)
|
|
10149
|
+
{
|
|
10150
|
+
count_equal_ki_native_i16_frame fr;
|
|
10151
|
+
fr.self = self;
|
|
10152
|
+
fr.ca = ca;
|
|
10153
|
+
fr.slab_axes = slab_axes;
|
|
10154
|
+
fr.naxes = naxes;
|
|
10155
|
+
fr.keep_axis = keep_axis;
|
|
10156
|
+
fr.value_arg = value_arg;
|
|
10157
|
+
fr.min_count = min_count;
|
|
10158
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_i16_body, (VALUE) &fr);
|
|
10159
|
+
}
|
|
10160
|
+
|
|
8961
10161
|
|
|
8962
10162
|
typedef struct {
|
|
8963
10163
|
void *chunk;
|
|
@@ -9015,14 +10215,44 @@ count_equal_ki_stream_u16 (VALUE arg)
|
|
|
9015
10215
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9016
10216
|
}
|
|
9017
10217
|
|
|
10218
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10219
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10220
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10221
|
+
typedef struct {
|
|
10222
|
+
ca_iter_state states[1];
|
|
10223
|
+
VALUE self;
|
|
10224
|
+
CArray *ca;
|
|
10225
|
+
int8_t *slab_axes;
|
|
10226
|
+
int8_t naxes;
|
|
10227
|
+
int keep_axis;
|
|
10228
|
+
ca_u_uint16_t value_arg;
|
|
10229
|
+
ca_size_t min_count;
|
|
10230
|
+
} count_equal_ki_native_u16_frame;
|
|
10231
|
+
|
|
9018
10232
|
static VALUE
|
|
9019
|
-
|
|
10233
|
+
count_equal_ki_native_u16_body (VALUE arg)
|
|
9020
10234
|
{
|
|
10235
|
+
count_equal_ki_native_u16_frame *fr = (count_equal_ki_native_u16_frame *) arg;
|
|
10236
|
+
VALUE self = fr->self;
|
|
10237
|
+
(void) self;
|
|
10238
|
+
CArray *ca = fr->ca;
|
|
10239
|
+
(void) ca;
|
|
10240
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
10241
|
+
(void) slab_axes;
|
|
10242
|
+
int8_t naxes = fr->naxes;
|
|
10243
|
+
(void) naxes;
|
|
10244
|
+
int keep_axis = fr->keep_axis;
|
|
10245
|
+
(void) keep_axis;
|
|
10246
|
+
ca_u_uint16_t value_arg = fr->value_arg;
|
|
10247
|
+
(void) value_arg;
|
|
10248
|
+
ca_size_t min_count = fr->min_count;
|
|
10249
|
+
(void) min_count;
|
|
10250
|
+
#define st (fr->states[0])
|
|
9021
10251
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9022
10252
|
ca->elements > 0 &&
|
|
9023
10253
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9024
10254
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9025
|
-
|
|
10255
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9026
10256
|
count_equal_ki_stream_u16_t __ctx;
|
|
9027
10257
|
__ctx.chunk = NULL;
|
|
9028
10258
|
__ctx.ca = ca;
|
|
@@ -9035,7 +10265,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9035
10265
|
CArray *co;
|
|
9036
10266
|
GetCArray(vout, co);
|
|
9037
10267
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9038
|
-
ca_iter_state st;
|
|
9039
10268
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9040
10269
|
slab_axes, naxes, 0);
|
|
9041
10270
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9067,7 +10296,23 @@ ca_size_t out_i = 0;
|
|
|
9067
10296
|
return LL2NUM(op[0]);
|
|
9068
10297
|
}
|
|
9069
10298
|
return vout;
|
|
9070
|
-
|
|
10299
|
+
#undef st
|
|
10300
|
+
}
|
|
10301
|
+
|
|
10302
|
+
static VALUE
|
|
10303
|
+
count_equal_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint16_t value_arg, ca_size_t min_count)
|
|
10304
|
+
{
|
|
10305
|
+
count_equal_ki_native_u16_frame fr;
|
|
10306
|
+
fr.self = self;
|
|
10307
|
+
fr.ca = ca;
|
|
10308
|
+
fr.slab_axes = slab_axes;
|
|
10309
|
+
fr.naxes = naxes;
|
|
10310
|
+
fr.keep_axis = keep_axis;
|
|
10311
|
+
fr.value_arg = value_arg;
|
|
10312
|
+
fr.min_count = min_count;
|
|
10313
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_u16_body, (VALUE) &fr);
|
|
10314
|
+
}
|
|
10315
|
+
|
|
9071
10316
|
|
|
9072
10317
|
typedef struct {
|
|
9073
10318
|
void *chunk;
|
|
@@ -9125,14 +10370,44 @@ count_equal_ki_stream_i32 (VALUE arg)
|
|
|
9125
10370
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9126
10371
|
}
|
|
9127
10372
|
|
|
10373
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10374
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10375
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10376
|
+
typedef struct {
|
|
10377
|
+
ca_iter_state states[1];
|
|
10378
|
+
VALUE self;
|
|
10379
|
+
CArray *ca;
|
|
10380
|
+
int8_t *slab_axes;
|
|
10381
|
+
int8_t naxes;
|
|
10382
|
+
int keep_axis;
|
|
10383
|
+
ca_u_int32_t value_arg;
|
|
10384
|
+
ca_size_t min_count;
|
|
10385
|
+
} count_equal_ki_native_i32_frame;
|
|
10386
|
+
|
|
9128
10387
|
static VALUE
|
|
9129
|
-
|
|
10388
|
+
count_equal_ki_native_i32_body (VALUE arg)
|
|
9130
10389
|
{
|
|
10390
|
+
count_equal_ki_native_i32_frame *fr = (count_equal_ki_native_i32_frame *) arg;
|
|
10391
|
+
VALUE self = fr->self;
|
|
10392
|
+
(void) self;
|
|
10393
|
+
CArray *ca = fr->ca;
|
|
10394
|
+
(void) ca;
|
|
10395
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
10396
|
+
(void) slab_axes;
|
|
10397
|
+
int8_t naxes = fr->naxes;
|
|
10398
|
+
(void) naxes;
|
|
10399
|
+
int keep_axis = fr->keep_axis;
|
|
10400
|
+
(void) keep_axis;
|
|
10401
|
+
ca_u_int32_t value_arg = fr->value_arg;
|
|
10402
|
+
(void) value_arg;
|
|
10403
|
+
ca_size_t min_count = fr->min_count;
|
|
10404
|
+
(void) min_count;
|
|
10405
|
+
#define st (fr->states[0])
|
|
9131
10406
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9132
10407
|
ca->elements > 0 &&
|
|
9133
10408
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9134
10409
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9135
|
-
|
|
10410
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9136
10411
|
count_equal_ki_stream_i32_t __ctx;
|
|
9137
10412
|
__ctx.chunk = NULL;
|
|
9138
10413
|
__ctx.ca = ca;
|
|
@@ -9145,7 +10420,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9145
10420
|
CArray *co;
|
|
9146
10421
|
GetCArray(vout, co);
|
|
9147
10422
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9148
|
-
ca_iter_state st;
|
|
9149
10423
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9150
10424
|
slab_axes, naxes, 0);
|
|
9151
10425
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9177,7 +10451,23 @@ ca_size_t out_i = 0;
|
|
|
9177
10451
|
return LL2NUM(op[0]);
|
|
9178
10452
|
}
|
|
9179
10453
|
return vout;
|
|
9180
|
-
|
|
10454
|
+
#undef st
|
|
10455
|
+
}
|
|
10456
|
+
|
|
10457
|
+
static VALUE
|
|
10458
|
+
count_equal_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int32_t value_arg, ca_size_t min_count)
|
|
10459
|
+
{
|
|
10460
|
+
count_equal_ki_native_i32_frame fr;
|
|
10461
|
+
fr.self = self;
|
|
10462
|
+
fr.ca = ca;
|
|
10463
|
+
fr.slab_axes = slab_axes;
|
|
10464
|
+
fr.naxes = naxes;
|
|
10465
|
+
fr.keep_axis = keep_axis;
|
|
10466
|
+
fr.value_arg = value_arg;
|
|
10467
|
+
fr.min_count = min_count;
|
|
10468
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_i32_body, (VALUE) &fr);
|
|
10469
|
+
}
|
|
10470
|
+
|
|
9181
10471
|
|
|
9182
10472
|
typedef struct {
|
|
9183
10473
|
void *chunk;
|
|
@@ -9235,14 +10525,44 @@ count_equal_ki_stream_u32 (VALUE arg)
|
|
|
9235
10525
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9236
10526
|
}
|
|
9237
10527
|
|
|
10528
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10529
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10530
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10531
|
+
typedef struct {
|
|
10532
|
+
ca_iter_state states[1];
|
|
10533
|
+
VALUE self;
|
|
10534
|
+
CArray *ca;
|
|
10535
|
+
int8_t *slab_axes;
|
|
10536
|
+
int8_t naxes;
|
|
10537
|
+
int keep_axis;
|
|
10538
|
+
ca_u_uint32_t value_arg;
|
|
10539
|
+
ca_size_t min_count;
|
|
10540
|
+
} count_equal_ki_native_u32_frame;
|
|
10541
|
+
|
|
9238
10542
|
static VALUE
|
|
9239
|
-
|
|
10543
|
+
count_equal_ki_native_u32_body (VALUE arg)
|
|
9240
10544
|
{
|
|
10545
|
+
count_equal_ki_native_u32_frame *fr = (count_equal_ki_native_u32_frame *) arg;
|
|
10546
|
+
VALUE self = fr->self;
|
|
10547
|
+
(void) self;
|
|
10548
|
+
CArray *ca = fr->ca;
|
|
10549
|
+
(void) ca;
|
|
10550
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
10551
|
+
(void) slab_axes;
|
|
10552
|
+
int8_t naxes = fr->naxes;
|
|
10553
|
+
(void) naxes;
|
|
10554
|
+
int keep_axis = fr->keep_axis;
|
|
10555
|
+
(void) keep_axis;
|
|
10556
|
+
ca_u_uint32_t value_arg = fr->value_arg;
|
|
10557
|
+
(void) value_arg;
|
|
10558
|
+
ca_size_t min_count = fr->min_count;
|
|
10559
|
+
(void) min_count;
|
|
10560
|
+
#define st (fr->states[0])
|
|
9241
10561
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9242
10562
|
ca->elements > 0 &&
|
|
9243
10563
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9244
10564
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9245
|
-
|
|
10565
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9246
10566
|
count_equal_ki_stream_u32_t __ctx;
|
|
9247
10567
|
__ctx.chunk = NULL;
|
|
9248
10568
|
__ctx.ca = ca;
|
|
@@ -9255,7 +10575,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9255
10575
|
CArray *co;
|
|
9256
10576
|
GetCArray(vout, co);
|
|
9257
10577
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9258
|
-
ca_iter_state st;
|
|
9259
10578
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9260
10579
|
slab_axes, naxes, 0);
|
|
9261
10580
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9287,7 +10606,23 @@ ca_size_t out_i = 0;
|
|
|
9287
10606
|
return LL2NUM(op[0]);
|
|
9288
10607
|
}
|
|
9289
10608
|
return vout;
|
|
9290
|
-
|
|
10609
|
+
#undef st
|
|
10610
|
+
}
|
|
10611
|
+
|
|
10612
|
+
static VALUE
|
|
10613
|
+
count_equal_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint32_t value_arg, ca_size_t min_count)
|
|
10614
|
+
{
|
|
10615
|
+
count_equal_ki_native_u32_frame fr;
|
|
10616
|
+
fr.self = self;
|
|
10617
|
+
fr.ca = ca;
|
|
10618
|
+
fr.slab_axes = slab_axes;
|
|
10619
|
+
fr.naxes = naxes;
|
|
10620
|
+
fr.keep_axis = keep_axis;
|
|
10621
|
+
fr.value_arg = value_arg;
|
|
10622
|
+
fr.min_count = min_count;
|
|
10623
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_u32_body, (VALUE) &fr);
|
|
10624
|
+
}
|
|
10625
|
+
|
|
9291
10626
|
|
|
9292
10627
|
typedef struct {
|
|
9293
10628
|
void *chunk;
|
|
@@ -9345,14 +10680,44 @@ count_equal_ki_stream_i64 (VALUE arg)
|
|
|
9345
10680
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9346
10681
|
}
|
|
9347
10682
|
|
|
10683
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10684
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10685
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10686
|
+
typedef struct {
|
|
10687
|
+
ca_iter_state states[1];
|
|
10688
|
+
VALUE self;
|
|
10689
|
+
CArray *ca;
|
|
10690
|
+
int8_t *slab_axes;
|
|
10691
|
+
int8_t naxes;
|
|
10692
|
+
int keep_axis;
|
|
10693
|
+
ca_u_int64_t value_arg;
|
|
10694
|
+
ca_size_t min_count;
|
|
10695
|
+
} count_equal_ki_native_i64_frame;
|
|
10696
|
+
|
|
9348
10697
|
static VALUE
|
|
9349
|
-
|
|
10698
|
+
count_equal_ki_native_i64_body (VALUE arg)
|
|
9350
10699
|
{
|
|
10700
|
+
count_equal_ki_native_i64_frame *fr = (count_equal_ki_native_i64_frame *) arg;
|
|
10701
|
+
VALUE self = fr->self;
|
|
10702
|
+
(void) self;
|
|
10703
|
+
CArray *ca = fr->ca;
|
|
10704
|
+
(void) ca;
|
|
10705
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
10706
|
+
(void) slab_axes;
|
|
10707
|
+
int8_t naxes = fr->naxes;
|
|
10708
|
+
(void) naxes;
|
|
10709
|
+
int keep_axis = fr->keep_axis;
|
|
10710
|
+
(void) keep_axis;
|
|
10711
|
+
ca_u_int64_t value_arg = fr->value_arg;
|
|
10712
|
+
(void) value_arg;
|
|
10713
|
+
ca_size_t min_count = fr->min_count;
|
|
10714
|
+
(void) min_count;
|
|
10715
|
+
#define st (fr->states[0])
|
|
9351
10716
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9352
10717
|
ca->elements > 0 &&
|
|
9353
10718
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9354
10719
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9355
|
-
|
|
10720
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9356
10721
|
count_equal_ki_stream_i64_t __ctx;
|
|
9357
10722
|
__ctx.chunk = NULL;
|
|
9358
10723
|
__ctx.ca = ca;
|
|
@@ -9365,7 +10730,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9365
10730
|
CArray *co;
|
|
9366
10731
|
GetCArray(vout, co);
|
|
9367
10732
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9368
|
-
ca_iter_state st;
|
|
9369
10733
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9370
10734
|
slab_axes, naxes, 0);
|
|
9371
10735
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9397,7 +10761,23 @@ ca_size_t out_i = 0;
|
|
|
9397
10761
|
return LL2NUM(op[0]);
|
|
9398
10762
|
}
|
|
9399
10763
|
return vout;
|
|
9400
|
-
|
|
10764
|
+
#undef st
|
|
10765
|
+
}
|
|
10766
|
+
|
|
10767
|
+
static VALUE
|
|
10768
|
+
count_equal_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int64_t value_arg, ca_size_t min_count)
|
|
10769
|
+
{
|
|
10770
|
+
count_equal_ki_native_i64_frame fr;
|
|
10771
|
+
fr.self = self;
|
|
10772
|
+
fr.ca = ca;
|
|
10773
|
+
fr.slab_axes = slab_axes;
|
|
10774
|
+
fr.naxes = naxes;
|
|
10775
|
+
fr.keep_axis = keep_axis;
|
|
10776
|
+
fr.value_arg = value_arg;
|
|
10777
|
+
fr.min_count = min_count;
|
|
10778
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_i64_body, (VALUE) &fr);
|
|
10779
|
+
}
|
|
10780
|
+
|
|
9401
10781
|
|
|
9402
10782
|
typedef struct {
|
|
9403
10783
|
void *chunk;
|
|
@@ -9455,14 +10835,44 @@ count_equal_ki_stream_u64 (VALUE arg)
|
|
|
9455
10835
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9456
10836
|
}
|
|
9457
10837
|
|
|
10838
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10839
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10840
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10841
|
+
typedef struct {
|
|
10842
|
+
ca_iter_state states[1];
|
|
10843
|
+
VALUE self;
|
|
10844
|
+
CArray *ca;
|
|
10845
|
+
int8_t *slab_axes;
|
|
10846
|
+
int8_t naxes;
|
|
10847
|
+
int keep_axis;
|
|
10848
|
+
ca_u_uint64_t value_arg;
|
|
10849
|
+
ca_size_t min_count;
|
|
10850
|
+
} count_equal_ki_native_u64_frame;
|
|
10851
|
+
|
|
9458
10852
|
static VALUE
|
|
9459
|
-
|
|
10853
|
+
count_equal_ki_native_u64_body (VALUE arg)
|
|
9460
10854
|
{
|
|
10855
|
+
count_equal_ki_native_u64_frame *fr = (count_equal_ki_native_u64_frame *) arg;
|
|
10856
|
+
VALUE self = fr->self;
|
|
10857
|
+
(void) self;
|
|
10858
|
+
CArray *ca = fr->ca;
|
|
10859
|
+
(void) ca;
|
|
10860
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
10861
|
+
(void) slab_axes;
|
|
10862
|
+
int8_t naxes = fr->naxes;
|
|
10863
|
+
(void) naxes;
|
|
10864
|
+
int keep_axis = fr->keep_axis;
|
|
10865
|
+
(void) keep_axis;
|
|
10866
|
+
ca_u_uint64_t value_arg = fr->value_arg;
|
|
10867
|
+
(void) value_arg;
|
|
10868
|
+
ca_size_t min_count = fr->min_count;
|
|
10869
|
+
(void) min_count;
|
|
10870
|
+
#define st (fr->states[0])
|
|
9461
10871
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9462
10872
|
ca->elements > 0 &&
|
|
9463
10873
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9464
10874
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9465
|
-
|
|
10875
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9466
10876
|
count_equal_ki_stream_u64_t __ctx;
|
|
9467
10877
|
__ctx.chunk = NULL;
|
|
9468
10878
|
__ctx.ca = ca;
|
|
@@ -9475,7 +10885,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9475
10885
|
CArray *co;
|
|
9476
10886
|
GetCArray(vout, co);
|
|
9477
10887
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9478
|
-
ca_iter_state st;
|
|
9479
10888
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9480
10889
|
slab_axes, naxes, 0);
|
|
9481
10890
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9507,7 +10916,23 @@ ca_size_t out_i = 0;
|
|
|
9507
10916
|
return LL2NUM(op[0]);
|
|
9508
10917
|
}
|
|
9509
10918
|
return vout;
|
|
9510
|
-
|
|
10919
|
+
#undef st
|
|
10920
|
+
}
|
|
10921
|
+
|
|
10922
|
+
static VALUE
|
|
10923
|
+
count_equal_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint64_t value_arg, ca_size_t min_count)
|
|
10924
|
+
{
|
|
10925
|
+
count_equal_ki_native_u64_frame fr;
|
|
10926
|
+
fr.self = self;
|
|
10927
|
+
fr.ca = ca;
|
|
10928
|
+
fr.slab_axes = slab_axes;
|
|
10929
|
+
fr.naxes = naxes;
|
|
10930
|
+
fr.keep_axis = keep_axis;
|
|
10931
|
+
fr.value_arg = value_arg;
|
|
10932
|
+
fr.min_count = min_count;
|
|
10933
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_u64_body, (VALUE) &fr);
|
|
10934
|
+
}
|
|
10935
|
+
|
|
9511
10936
|
|
|
9512
10937
|
typedef struct {
|
|
9513
10938
|
void *chunk;
|
|
@@ -9565,14 +10990,44 @@ count_equal_ki_stream_f32 (VALUE arg)
|
|
|
9565
10990
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9566
10991
|
}
|
|
9567
10992
|
|
|
10993
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
10994
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
10995
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
10996
|
+
typedef struct {
|
|
10997
|
+
ca_iter_state states[1];
|
|
10998
|
+
VALUE self;
|
|
10999
|
+
CArray *ca;
|
|
11000
|
+
int8_t *slab_axes;
|
|
11001
|
+
int8_t naxes;
|
|
11002
|
+
int keep_axis;
|
|
11003
|
+
ca_u_float32_t value_arg;
|
|
11004
|
+
ca_size_t min_count;
|
|
11005
|
+
} count_equal_ki_native_f32_frame;
|
|
11006
|
+
|
|
9568
11007
|
static VALUE
|
|
9569
|
-
|
|
11008
|
+
count_equal_ki_native_f32_body (VALUE arg)
|
|
9570
11009
|
{
|
|
11010
|
+
count_equal_ki_native_f32_frame *fr = (count_equal_ki_native_f32_frame *) arg;
|
|
11011
|
+
VALUE self = fr->self;
|
|
11012
|
+
(void) self;
|
|
11013
|
+
CArray *ca = fr->ca;
|
|
11014
|
+
(void) ca;
|
|
11015
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
11016
|
+
(void) slab_axes;
|
|
11017
|
+
int8_t naxes = fr->naxes;
|
|
11018
|
+
(void) naxes;
|
|
11019
|
+
int keep_axis = fr->keep_axis;
|
|
11020
|
+
(void) keep_axis;
|
|
11021
|
+
ca_u_float32_t value_arg = fr->value_arg;
|
|
11022
|
+
(void) value_arg;
|
|
11023
|
+
ca_size_t min_count = fr->min_count;
|
|
11024
|
+
(void) min_count;
|
|
11025
|
+
#define st (fr->states[0])
|
|
9571
11026
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9572
11027
|
ca->elements > 0 &&
|
|
9573
11028
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9574
11029
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9575
|
-
|
|
11030
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9576
11031
|
count_equal_ki_stream_f32_t __ctx;
|
|
9577
11032
|
__ctx.chunk = NULL;
|
|
9578
11033
|
__ctx.ca = ca;
|
|
@@ -9585,7 +11040,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9585
11040
|
CArray *co;
|
|
9586
11041
|
GetCArray(vout, co);
|
|
9587
11042
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9588
|
-
ca_iter_state st;
|
|
9589
11043
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9590
11044
|
slab_axes, naxes, 0);
|
|
9591
11045
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9617,7 +11071,23 @@ ca_size_t out_i = 0;
|
|
|
9617
11071
|
return LL2NUM(op[0]);
|
|
9618
11072
|
}
|
|
9619
11073
|
return vout;
|
|
9620
|
-
|
|
11074
|
+
#undef st
|
|
11075
|
+
}
|
|
11076
|
+
|
|
11077
|
+
static VALUE
|
|
11078
|
+
count_equal_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float32_t value_arg, ca_size_t min_count)
|
|
11079
|
+
{
|
|
11080
|
+
count_equal_ki_native_f32_frame fr;
|
|
11081
|
+
fr.self = self;
|
|
11082
|
+
fr.ca = ca;
|
|
11083
|
+
fr.slab_axes = slab_axes;
|
|
11084
|
+
fr.naxes = naxes;
|
|
11085
|
+
fr.keep_axis = keep_axis;
|
|
11086
|
+
fr.value_arg = value_arg;
|
|
11087
|
+
fr.min_count = min_count;
|
|
11088
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_f32_body, (VALUE) &fr);
|
|
11089
|
+
}
|
|
11090
|
+
|
|
9621
11091
|
|
|
9622
11092
|
typedef struct {
|
|
9623
11093
|
void *chunk;
|
|
@@ -9675,14 +11145,44 @@ count_equal_ki_stream_f64 (VALUE arg)
|
|
|
9675
11145
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9676
11146
|
}
|
|
9677
11147
|
|
|
11148
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
11149
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
11150
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
11151
|
+
typedef struct {
|
|
11152
|
+
ca_iter_state states[1];
|
|
11153
|
+
VALUE self;
|
|
11154
|
+
CArray *ca;
|
|
11155
|
+
int8_t *slab_axes;
|
|
11156
|
+
int8_t naxes;
|
|
11157
|
+
int keep_axis;
|
|
11158
|
+
ca_u_float64_t value_arg;
|
|
11159
|
+
ca_size_t min_count;
|
|
11160
|
+
} count_equal_ki_native_f64_frame;
|
|
11161
|
+
|
|
9678
11162
|
static VALUE
|
|
9679
|
-
|
|
11163
|
+
count_equal_ki_native_f64_body (VALUE arg)
|
|
9680
11164
|
{
|
|
11165
|
+
count_equal_ki_native_f64_frame *fr = (count_equal_ki_native_f64_frame *) arg;
|
|
11166
|
+
VALUE self = fr->self;
|
|
11167
|
+
(void) self;
|
|
11168
|
+
CArray *ca = fr->ca;
|
|
11169
|
+
(void) ca;
|
|
11170
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
11171
|
+
(void) slab_axes;
|
|
11172
|
+
int8_t naxes = fr->naxes;
|
|
11173
|
+
(void) naxes;
|
|
11174
|
+
int keep_axis = fr->keep_axis;
|
|
11175
|
+
(void) keep_axis;
|
|
11176
|
+
ca_u_float64_t value_arg = fr->value_arg;
|
|
11177
|
+
(void) value_arg;
|
|
11178
|
+
ca_size_t min_count = fr->min_count;
|
|
11179
|
+
(void) min_count;
|
|
11180
|
+
#define st (fr->states[0])
|
|
9681
11181
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
9682
11182
|
ca->elements > 0 &&
|
|
9683
11183
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9684
11184
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9685
|
-
|
|
11185
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9686
11186
|
count_equal_ki_stream_f64_t __ctx;
|
|
9687
11187
|
__ctx.chunk = NULL;
|
|
9688
11188
|
__ctx.ca = ca;
|
|
@@ -9695,7 +11195,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
|
9695
11195
|
CArray *co;
|
|
9696
11196
|
GetCArray(vout, co);
|
|
9697
11197
|
ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
|
|
9698
|
-
ca_iter_state st;
|
|
9699
11198
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
9700
11199
|
slab_axes, naxes, 0);
|
|
9701
11200
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -9727,7 +11226,23 @@ ca_size_t out_i = 0;
|
|
|
9727
11226
|
return LL2NUM(op[0]);
|
|
9728
11227
|
}
|
|
9729
11228
|
return vout;
|
|
9730
|
-
|
|
11229
|
+
#undef st
|
|
11230
|
+
}
|
|
11231
|
+
|
|
11232
|
+
static VALUE
|
|
11233
|
+
count_equal_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float64_t value_arg, ca_size_t min_count)
|
|
11234
|
+
{
|
|
11235
|
+
count_equal_ki_native_f64_frame fr;
|
|
11236
|
+
fr.self = self;
|
|
11237
|
+
fr.ca = ca;
|
|
11238
|
+
fr.slab_axes = slab_axes;
|
|
11239
|
+
fr.naxes = naxes;
|
|
11240
|
+
fr.keep_axis = keep_axis;
|
|
11241
|
+
fr.value_arg = value_arg;
|
|
11242
|
+
fr.min_count = min_count;
|
|
11243
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_f64_body, (VALUE) &fr);
|
|
11244
|
+
}
|
|
11245
|
+
|
|
9731
11246
|
|
|
9732
11247
|
typedef struct {
|
|
9733
11248
|
void *chunk;
|
|
@@ -9785,9 +11300,9 @@ count_equal_ki_stream_object (VALUE arg)
|
|
|
9785
11300
|
return LL2NUM((ca_u_int64_t) (acc));
|
|
9786
11301
|
}
|
|
9787
11302
|
|
|
9788
|
-
/* The
|
|
9789
|
-
|
|
9790
|
-
finishes them however the body leaves. */
|
|
11303
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
11304
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
11305
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
9791
11306
|
typedef struct {
|
|
9792
11307
|
ca_iter_state states[1];
|
|
9793
11308
|
VALUE self;
|
|
@@ -9822,7 +11337,7 @@ count_equal_ki_native_object_body (VALUE arg)
|
|
|
9822
11337
|
ca->elements > 0 &&
|
|
9823
11338
|
! ca_too_few_present(ca->elements, min_count, 0) &&
|
|
9824
11339
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
9825
|
-
|
|
11340
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
9826
11341
|
count_equal_ki_stream_object_t __ctx;
|
|
9827
11342
|
__ctx.chunk = NULL;
|
|
9828
11343
|
__ctx.ca = ca;
|
|
@@ -9880,19 +11395,48 @@ count_equal_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
9880
11395
|
fr.keep_axis = keep_axis;
|
|
9881
11396
|
fr.value_arg = value_arg;
|
|
9882
11397
|
fr.min_count = min_count;
|
|
9883
|
-
return
|
|
11398
|
+
return ca_iter_ensure_computing(ca, 1, fr.states, count_equal_ki_native_object_body, (VALUE) &fr);
|
|
9884
11399
|
}
|
|
9885
11400
|
|
|
9886
11401
|
|
|
11402
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
11403
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
11404
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
11405
|
+
typedef struct {
|
|
11406
|
+
ca_iter_state states[1];
|
|
11407
|
+
VALUE self;
|
|
11408
|
+
CArray *ca;
|
|
11409
|
+
int8_t *slab_axes;
|
|
11410
|
+
int8_t naxes;
|
|
11411
|
+
int keep_axis;
|
|
11412
|
+
const char *value_arg;
|
|
11413
|
+
ca_size_t min_count;
|
|
11414
|
+
} count_equal_ki_native_fixlen_frame;
|
|
11415
|
+
|
|
9887
11416
|
static VALUE
|
|
9888
|
-
|
|
11417
|
+
count_equal_ki_native_fixlen_body (VALUE arg)
|
|
9889
11418
|
{
|
|
11419
|
+
count_equal_ki_native_fixlen_frame *fr = (count_equal_ki_native_fixlen_frame *) arg;
|
|
11420
|
+
VALUE self = fr->self;
|
|
11421
|
+
(void) self;
|
|
11422
|
+
CArray *ca = fr->ca;
|
|
11423
|
+
(void) ca;
|
|
11424
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
11425
|
+
(void) slab_axes;
|
|
11426
|
+
int8_t naxes = fr->naxes;
|
|
11427
|
+
(void) naxes;
|
|
11428
|
+
int keep_axis = fr->keep_axis;
|
|
11429
|
+
(void) keep_axis;
|
|
11430
|
+
const char *value_arg = fr->value_arg;
|
|
11431
|
+
(void) value_arg;
|
|
11432
|
+
ca_size_t min_count = fr->min_count;
|
|
11433
|
+
(void) min_count;
|
|
11434
|
+
#define st (fr->states[0])
|
|
9890
11435
|
ca_size_t K = ca->bytes; /* uniform fixlen byte width */
|
|
9891
11436
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
|
|
9892
11437
|
CArray *co;
|
|
9893
11438
|
GetCArray(vout, co);
|
|
9894
11439
|
char *op = (char *) co->ptr;
|
|
9895
|
-
ca_iter_state st;
|
|
9896
11440
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES, slab_axes, naxes, 0);
|
|
9897
11441
|
if ( rc != CA_ITER_OK ) {
|
|
9898
11442
|
rb_raise(rb_eRuntimeError, "%s: kernel_iterator init failed rc=%d", ca_calling_method_name(), rc);
|
|
@@ -9949,8 +11493,24 @@ count_equal_ki_native_fixlen (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
|
|
|
9949
11493
|
return LL2NUM(((int64_t *) op)[0]);
|
|
9950
11494
|
}
|
|
9951
11495
|
return vout;
|
|
11496
|
+
#undef st
|
|
9952
11497
|
}
|
|
9953
11498
|
|
|
11499
|
+
static VALUE
|
|
11500
|
+
count_equal_ki_native_fixlen (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, const char *value_arg, ca_size_t min_count)
|
|
11501
|
+
{
|
|
11502
|
+
count_equal_ki_native_fixlen_frame fr;
|
|
11503
|
+
fr.self = self;
|
|
11504
|
+
fr.ca = ca;
|
|
11505
|
+
fr.slab_axes = slab_axes;
|
|
11506
|
+
fr.naxes = naxes;
|
|
11507
|
+
fr.keep_axis = keep_axis;
|
|
11508
|
+
fr.value_arg = value_arg;
|
|
11509
|
+
fr.min_count = min_count;
|
|
11510
|
+
return ca_iter_ensure(1, fr.states, count_equal_ki_native_fixlen_body, (VALUE) &fr);
|
|
11511
|
+
}
|
|
11512
|
+
|
|
11513
|
+
|
|
9954
11514
|
VALUE
|
|
9955
11515
|
rb_ca_count_equal_ki (int argc, VALUE *argv, VALUE self)
|
|
9956
11516
|
{
|