carray 3.0.5 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +233 -0
  3. data/README.md +1 -1
  4. data/carray.gemspec +3 -2
  5. data/ext/ca_axis_dispatch.c +6 -0
  6. data/ext/ca_axis_group.c +1 -1
  7. data/ext/ca_categorical_iterator.c +7 -7
  8. data/ext/ca_kernel_iterator.c +116 -231
  9. data/ext/ca_kernel_iterator.h +410 -254
  10. data/ext/ca_obj_bincmp.c +9 -1
  11. data/ext/ca_obj_binop.c +10 -1
  12. data/ext/ca_obj_bitfield.c +2 -0
  13. data/ext/ca_obj_byte_swap.c +2 -0
  14. data/ext/ca_obj_face.c +2 -0
  15. data/ext/ca_obj_fake.c +3 -1
  16. data/ext/ca_obj_grid.c +167 -12
  17. data/ext/ca_obj_meld.c +69 -113
  18. data/ext/ca_obj_moncmp.c +9 -1
  19. data/ext/ca_obj_monop.c +40 -2
  20. data/ext/ca_obj_object.c +54 -20
  21. data/ext/ca_obj_reduce.c +15 -1
  22. data/ext/ca_obj_refer.c +65 -20
  23. data/ext/ca_obj_remap.c +69 -10
  24. data/ext/ca_obj_roll.c +1 -0
  25. data/ext/ca_obj_select.c +4 -1
  26. data/ext/ca_obj_select_axis.c +1 -0
  27. data/ext/ca_obj_shift.c +1 -0
  28. data/ext/ca_obj_stack.c +310 -111
  29. data/ext/ca_obj_stride.c +131 -45
  30. data/ext/ca_obj_string.c +18 -2
  31. data/ext/ca_obj_tile.c +1 -0
  32. data/ext/ca_obj_triop.c +9 -1
  33. data/ext/ca_obj_window.c +1 -0
  34. data/ext/ca_op_ipower.c +8 -8
  35. data/ext/caframe_csv_reader.c +1 -1
  36. data/ext/carray.h +2 -1
  37. data/ext/carray_access.c +27 -64
  38. data/ext/carray_address_basis.c +134 -0
  39. data/ext/carray_attribute.c +8 -0
  40. data/ext/carray_broadcast.c +3 -4
  41. data/ext/carray_build_flags.h +1 -1
  42. data/ext/carray_cast.c +8 -11
  43. data/ext/carray_conversion.c +9 -5
  44. data/ext/carray_core.c +811 -72
  45. data/ext/carray_element.c +13 -9
  46. data/ext/carray_histogram.c +25 -3
  47. data/ext/carray_internal.h +147 -12
  48. data/ext/carray_kernels_bincmp.c +2007 -3
  49. data/ext/carray_kernels_binop.c +25365 -5199
  50. data/ext/carray_kernels_init.c +8 -8
  51. data/ext/carray_kernels_monop.c +10696 -3678
  52. data/ext/carray_kernels_reduce_aggregate.c +4429 -702
  53. data/ext/carray_kernels_reduce_boolean.c +1533 -15
  54. data/ext/carray_kernels_reduce_cumulative.c +1808 -248
  55. data/ext/carray_kernels_reduce_extreme.c +4508 -511
  56. data/ext/carray_kernels_reduce_variance.c +2284 -152
  57. data/ext/carray_kernels_scan.c +240 -240
  58. data/ext/carray_kernels_search.c +4211 -264
  59. data/ext/carray_kernels_sort.c +367 -450
  60. data/ext/carray_lazy.c +47 -0
  61. data/ext/carray_loop.c +1 -1
  62. data/ext/carray_mask.c +130 -139
  63. data/ext/carray_median_percentile.c +4 -4
  64. data/ext/carray_memory_view.c +13 -7
  65. data/ext/carray_operator.c +147 -44
  66. data/ext/carray_order.c +3 -2
  67. data/ext/carray_random.c +4 -1
  68. data/ext/carray_slab.c +97 -21
  69. data/ext/carray_sort.c +1 -1
  70. data/ext/carray_test.c +74 -19
  71. data/ext/carray_time_text.c +2 -2
  72. data/ext/extconf.rb +11 -0
  73. data/ext/mkkernel.rb +346 -94
  74. data/ext/ruby_carray.c +4 -1
  75. data/ext/version.h +4 -4
  76. data/lib/carray/attribute.rb +25 -1
  77. data/lib/carray/autoload_carray.rb +3 -3
  78. data/lib/carray/autoload_method_extension.rb +5 -3
  79. data/lib/carray/axis_group.rb +13 -7
  80. data/lib/carray/basics.rb +1 -1
  81. data/lib/carray/block_iterator.rb +17 -3
  82. data/lib/carray/categorical.rb +6 -8
  83. data/lib/carray/categorical_iterator.rb +2 -2
  84. data/lib/carray/complex.rb +2 -2
  85. data/lib/carray/conditional.rb +3 -2
  86. data/lib/carray/frame/column_verbs.rb +45 -14
  87. data/lib/carray/frame/convert.rb +19 -0
  88. data/lib/carray/frame/csv_parser.rb +32 -32
  89. data/lib/carray/frame/describe.rb +4 -2
  90. data/lib/carray/frame/frame.rb +399 -78
  91. data/lib/carray/frame/group.rb +679 -53
  92. data/lib/carray/frame/io.rb +209 -71
  93. data/lib/carray/frame/join.rb +146 -37
  94. data/lib/carray/frame/records.rb +3 -3
  95. data/lib/carray/frame/reshape.rb +89 -11
  96. data/lib/carray/frame/sort.rb +57 -9
  97. data/lib/carray/frame/verbs.rb +384 -49
  98. data/lib/carray/frame/windows.rb +44 -6
  99. data/lib/carray/inspect.rb +7 -2
  100. data/lib/carray/iterator.rb +32 -0
  101. data/lib/carray/lazy.rb +17 -8
  102. data/lib/carray/mask_gap_fill.rb +37 -17
  103. data/lib/carray/meld_reduce.rb +1 -1
  104. data/lib/carray/methods/choose.rb +3 -2
  105. data/lib/carray/methods/meshgrid.rb +3 -1
  106. data/lib/carray/runtime.rb +47 -8
  107. data/lib/carray/segment_iterator.rb +23 -8
  108. data/lib/carray/serialize.rb +34 -17
  109. data/lib/carray/stack.rb +18 -6
  110. data/lib/carray/time.rb +5 -0
  111. data/lib/carray/window_iterator.rb +16 -4
  112. data/yard-stubs/ca_obj_array.rb +26 -1
  113. data/yard-stubs/carray_cast.rb +8 -5
  114. data/yard-stubs/carray_conversion.rb +8 -2
  115. data/yard-stubs/carray_mask.rb +20 -1
  116. data/yard-stubs/carray_math.rb +31 -16
  117. data/yard-stubs/carray_memory_view.rb +7 -0
  118. data/yard-stubs/carray_test.rb +7 -4
  119. metadata +18 -4
@@ -98,14 +98,41 @@ accumulate_ki_stream_i8 (VALUE arg)
98
98
  return LL2NUM((int8_t) (acc));
99
99
  }
100
100
 
101
+ /* The walks can raise part way (a Ruby call per cell in the object
102
+ lane, an allocation in any lane). Their states live in the caller's
103
+ frame, and ca_iter_ensure finishes them however the body leaves. */
104
+ typedef struct {
105
+ ca_iter_state states[1];
106
+ VALUE self;
107
+ CArray *ca;
108
+ int8_t *slab_axes;
109
+ int8_t naxes;
110
+ int keep_axis;
111
+ ca_size_t min_count;
112
+ } accumulate_ki_native_i8_frame;
113
+
101
114
  static VALUE
102
- accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
115
+ accumulate_ki_native_i8_body (VALUE arg)
103
116
  {
117
+ accumulate_ki_native_i8_frame *fr = (accumulate_ki_native_i8_frame *) arg;
118
+ VALUE self = fr->self;
119
+ (void) self;
120
+ CArray *ca = fr->ca;
121
+ (void) ca;
122
+ int8_t *slab_axes = fr->slab_axes;
123
+ (void) slab_axes;
124
+ int8_t naxes = fr->naxes;
125
+ (void) naxes;
126
+ int keep_axis = fr->keep_axis;
127
+ (void) keep_axis;
128
+ ca_size_t min_count = fr->min_count;
129
+ (void) min_count;
130
+ #define st (fr->states[0])
104
131
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
105
132
  ca->elements > 0 &&
106
133
  ! ca_too_few_present(ca->elements, min_count, 0) &&
107
134
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
108
- ca_fast_paths_enabled() ) {
135
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
109
136
  accumulate_ki_stream_i8_t __ctx;
110
137
  __ctx.chunk = NULL;
111
138
  __ctx.ca = ca;
@@ -126,7 +153,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
126
153
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
127
154
  && ! ca_has_mask(ca)
128
155
  && ca_attach_is_alias(ca)
129
- && ca_fast_paths_enabled() ) {
156
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
130
157
  int8_t __li_ax = slab_axes[0];
131
158
  ca_size_t __li_M = ca->dim[__li_ax];
132
159
  ca_size_t __li_INNER = 1;
@@ -199,7 +226,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
199
226
  && ! ca_has_mask(ca)
200
227
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
201
228
  && ((CAStack *) ca)->k_axis == 0
202
- && ca_fast_paths_enabled() ) {
229
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
203
230
  CAStack *__li_st = (CAStack *) ca;
204
231
  int8_t __li_ax = slab_axes[0];
205
232
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -227,7 +254,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
227
254
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
228
255
  const ca_size_t __li_TILE = 512;
229
256
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
230
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
257
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
231
258
  const int8_t *p_root = (const int8_t *) __li_st->parents[__kk]->ptr;
232
259
  int8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
233
260
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -287,7 +314,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
287
314
  && ((CAStack *) ca)->k_axis > 0
288
315
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
289
316
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
290
- && ca_fast_paths_enabled() ) {
317
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
291
318
  CAStack *__li_st = (CAStack *) ca;
292
319
  int8_t __li_ax = slab_axes[0];
293
320
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -344,7 +371,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
344
371
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
345
372
  const ca_size_t __li_TILE = 512;
346
373
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
347
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
374
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
348
375
  const int8_t *p_root = (const int8_t *) __li_st->parents[__kk]->ptr;
349
376
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
350
377
 
@@ -432,7 +459,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
432
459
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
433
460
  && ((CAStack *) ca)->k_axis > 0
434
461
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
435
- && ca_fast_paths_enabled() ) {
462
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
436
463
  CAStack *__li_st = (CAStack *) ca;
437
464
  int8_t __li_ax = slab_axes[0];
438
465
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -504,7 +531,7 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
504
531
 
505
532
  const ca_size_t __li_TILE = 512;
506
533
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
507
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
534
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
508
535
  const int8_t *p_root = (const int8_t *) __li_st->parents[__kk]->ptr;
509
536
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
510
537
 
@@ -553,12 +580,41 @@ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
553
580
  return vout;
554
581
  }
555
582
  }
583
+ if ( naxes == 1
584
+ && ca->ndim >= 2
585
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
586
+ && ca->dim[ca->ndim - 1] >= 1
587
+ && ca->dim[ca->ndim - 1] <= 32
588
+ && ! ca_has_mask(ca)
589
+ && ca_attach_is_alias(ca)
590
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
591
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
592
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
593
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT8, keep_axis);
594
+ CArray *co;
595
+ GetCArray(vout, co);
596
+ int8_t *op = (int8_t *) co->ptr;
597
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
598
+ const int8_t *p_root = (const int8_t *) ca->ptr;
599
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
600
+ const int8_t *__sf_row = p_root + __sf_o * __sf_M;
601
+ int8_t acc = (int8_t) (0);
602
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
603
+ int8_t v = __sf_row[__j];
604
+ acc += v;
605
+ (void) v;
606
+ }
607
+
608
+ op[__sf_o] = (int8_t) (acc);
609
+ }
610
+ ca_detach(ca);
611
+ return vout;
612
+ }
556
613
  }
557
614
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT8, keep_axis);
558
615
  CArray *co;
559
616
  GetCArray(vout, co);
560
617
  int8_t *op = (int8_t *) co->ptr;
561
- ca_iter_state st;
562
618
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
563
619
  slab_axes, naxes, 0);
564
620
  if ( rc != CA_ITER_OK ) {
@@ -592,7 +648,22 @@ ca_size_t out_i = 0;
592
648
  return LL2NUM(op[0]);
593
649
  }
594
650
  return vout;
595
- }
651
+ #undef st
652
+ }
653
+
654
+ static VALUE
655
+ accumulate_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
656
+ {
657
+ accumulate_ki_native_i8_frame fr;
658
+ fr.self = self;
659
+ fr.ca = ca;
660
+ fr.slab_axes = slab_axes;
661
+ fr.naxes = naxes;
662
+ fr.keep_axis = keep_axis;
663
+ fr.min_count = min_count;
664
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_i8_body, (VALUE) &fr);
665
+ }
666
+
596
667
 
597
668
  typedef struct {
598
669
  void *chunk;
@@ -647,14 +718,41 @@ accumulate_ki_stream_u8 (VALUE arg)
647
718
  return ULL2NUM((uint8_t) (acc));
648
719
  }
649
720
 
721
+ /* The walks can raise part way (a Ruby call per cell in the object
722
+ lane, an allocation in any lane). Their states live in the caller's
723
+ frame, and ca_iter_ensure finishes them however the body leaves. */
724
+ typedef struct {
725
+ ca_iter_state states[1];
726
+ VALUE self;
727
+ CArray *ca;
728
+ int8_t *slab_axes;
729
+ int8_t naxes;
730
+ int keep_axis;
731
+ ca_size_t min_count;
732
+ } accumulate_ki_native_u8_frame;
733
+
650
734
  static VALUE
651
- accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
735
+ accumulate_ki_native_u8_body (VALUE arg)
652
736
  {
737
+ accumulate_ki_native_u8_frame *fr = (accumulate_ki_native_u8_frame *) arg;
738
+ VALUE self = fr->self;
739
+ (void) self;
740
+ CArray *ca = fr->ca;
741
+ (void) ca;
742
+ int8_t *slab_axes = fr->slab_axes;
743
+ (void) slab_axes;
744
+ int8_t naxes = fr->naxes;
745
+ (void) naxes;
746
+ int keep_axis = fr->keep_axis;
747
+ (void) keep_axis;
748
+ ca_size_t min_count = fr->min_count;
749
+ (void) min_count;
750
+ #define st (fr->states[0])
653
751
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
654
752
  ca->elements > 0 &&
655
753
  ! ca_too_few_present(ca->elements, min_count, 0) &&
656
754
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
657
- ca_fast_paths_enabled() ) {
755
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
658
756
  accumulate_ki_stream_u8_t __ctx;
659
757
  __ctx.chunk = NULL;
660
758
  __ctx.ca = ca;
@@ -675,7 +773,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
675
773
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
676
774
  && ! ca_has_mask(ca)
677
775
  && ca_attach_is_alias(ca)
678
- && ca_fast_paths_enabled() ) {
776
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
679
777
  int8_t __li_ax = slab_axes[0];
680
778
  ca_size_t __li_M = ca->dim[__li_ax];
681
779
  ca_size_t __li_INNER = 1;
@@ -748,7 +846,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
748
846
  && ! ca_has_mask(ca)
749
847
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
750
848
  && ((CAStack *) ca)->k_axis == 0
751
- && ca_fast_paths_enabled() ) {
849
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
752
850
  CAStack *__li_st = (CAStack *) ca;
753
851
  int8_t __li_ax = slab_axes[0];
754
852
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -776,7 +874,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
776
874
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
777
875
  const ca_size_t __li_TILE = 512;
778
876
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
779
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
877
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
780
878
  const uint8_t *p_root = (const uint8_t *) __li_st->parents[__kk]->ptr;
781
879
  uint8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
782
880
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -836,7 +934,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
836
934
  && ((CAStack *) ca)->k_axis > 0
837
935
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
838
936
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
839
- && ca_fast_paths_enabled() ) {
937
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
840
938
  CAStack *__li_st = (CAStack *) ca;
841
939
  int8_t __li_ax = slab_axes[0];
842
940
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -893,7 +991,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
893
991
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
894
992
  const ca_size_t __li_TILE = 512;
895
993
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
896
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
994
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
897
995
  const uint8_t *p_root = (const uint8_t *) __li_st->parents[__kk]->ptr;
898
996
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
899
997
 
@@ -981,7 +1079,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
981
1079
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
982
1080
  && ((CAStack *) ca)->k_axis > 0
983
1081
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
984
- && ca_fast_paths_enabled() ) {
1082
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
985
1083
  CAStack *__li_st = (CAStack *) ca;
986
1084
  int8_t __li_ax = slab_axes[0];
987
1085
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -1053,7 +1151,7 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
1053
1151
 
1054
1152
  const ca_size_t __li_TILE = 512;
1055
1153
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
1056
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
1154
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
1057
1155
  const uint8_t *p_root = (const uint8_t *) __li_st->parents[__kk]->ptr;
1058
1156
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
1059
1157
 
@@ -1102,12 +1200,41 @@ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes
1102
1200
  return vout;
1103
1201
  }
1104
1202
  }
1203
+ if ( naxes == 1
1204
+ && ca->ndim >= 2
1205
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
1206
+ && ca->dim[ca->ndim - 1] >= 1
1207
+ && ca->dim[ca->ndim - 1] <= 32
1208
+ && ! ca_has_mask(ca)
1209
+ && ca_attach_is_alias(ca)
1210
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
1211
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
1212
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
1213
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT8, keep_axis);
1214
+ CArray *co;
1215
+ GetCArray(vout, co);
1216
+ uint8_t *op = (uint8_t *) co->ptr;
1217
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
1218
+ const uint8_t *p_root = (const uint8_t *) ca->ptr;
1219
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
1220
+ const uint8_t *__sf_row = p_root + __sf_o * __sf_M;
1221
+ uint8_t acc = (uint8_t) (0);
1222
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
1223
+ uint8_t v = __sf_row[__j];
1224
+ acc += v;
1225
+ (void) v;
1226
+ }
1227
+
1228
+ op[__sf_o] = (uint8_t) (acc);
1229
+ }
1230
+ ca_detach(ca);
1231
+ return vout;
1232
+ }
1105
1233
  }
1106
1234
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT8, keep_axis);
1107
1235
  CArray *co;
1108
1236
  GetCArray(vout, co);
1109
1237
  uint8_t *op = (uint8_t *) co->ptr;
1110
- ca_iter_state st;
1111
1238
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
1112
1239
  slab_axes, naxes, 0);
1113
1240
  if ( rc != CA_ITER_OK ) {
@@ -1141,7 +1268,22 @@ ca_size_t out_i = 0;
1141
1268
  return ULL2NUM(op[0]);
1142
1269
  }
1143
1270
  return vout;
1144
- }
1271
+ #undef st
1272
+ }
1273
+
1274
+ static VALUE
1275
+ accumulate_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
1276
+ {
1277
+ accumulate_ki_native_u8_frame fr;
1278
+ fr.self = self;
1279
+ fr.ca = ca;
1280
+ fr.slab_axes = slab_axes;
1281
+ fr.naxes = naxes;
1282
+ fr.keep_axis = keep_axis;
1283
+ fr.min_count = min_count;
1284
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_u8_body, (VALUE) &fr);
1285
+ }
1286
+
1145
1287
 
1146
1288
  typedef struct {
1147
1289
  void *chunk;
@@ -1196,14 +1338,41 @@ accumulate_ki_stream_i16 (VALUE arg)
1196
1338
  return LL2NUM((ca_u_int16_t) (acc));
1197
1339
  }
1198
1340
 
1341
+ /* The walks can raise part way (a Ruby call per cell in the object
1342
+ lane, an allocation in any lane). Their states live in the caller's
1343
+ frame, and ca_iter_ensure finishes them however the body leaves. */
1344
+ typedef struct {
1345
+ ca_iter_state states[1];
1346
+ VALUE self;
1347
+ CArray *ca;
1348
+ int8_t *slab_axes;
1349
+ int8_t naxes;
1350
+ int keep_axis;
1351
+ ca_size_t min_count;
1352
+ } accumulate_ki_native_i16_frame;
1353
+
1199
1354
  static VALUE
1200
- accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
1355
+ accumulate_ki_native_i16_body (VALUE arg)
1201
1356
  {
1357
+ accumulate_ki_native_i16_frame *fr = (accumulate_ki_native_i16_frame *) arg;
1358
+ VALUE self = fr->self;
1359
+ (void) self;
1360
+ CArray *ca = fr->ca;
1361
+ (void) ca;
1362
+ int8_t *slab_axes = fr->slab_axes;
1363
+ (void) slab_axes;
1364
+ int8_t naxes = fr->naxes;
1365
+ (void) naxes;
1366
+ int keep_axis = fr->keep_axis;
1367
+ (void) keep_axis;
1368
+ ca_size_t min_count = fr->min_count;
1369
+ (void) min_count;
1370
+ #define st (fr->states[0])
1202
1371
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
1203
1372
  ca->elements > 0 &&
1204
1373
  ! ca_too_few_present(ca->elements, min_count, 0) &&
1205
1374
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
1206
- ca_fast_paths_enabled() ) {
1375
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
1207
1376
  accumulate_ki_stream_i16_t __ctx;
1208
1377
  __ctx.chunk = NULL;
1209
1378
  __ctx.ca = ca;
@@ -1224,7 +1393,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1224
1393
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
1225
1394
  && ! ca_has_mask(ca)
1226
1395
  && ca_attach_is_alias(ca)
1227
- && ca_fast_paths_enabled() ) {
1396
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
1228
1397
  int8_t __li_ax = slab_axes[0];
1229
1398
  ca_size_t __li_M = ca->dim[__li_ax];
1230
1399
  ca_size_t __li_INNER = 1;
@@ -1297,7 +1466,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1297
1466
  && ! ca_has_mask(ca)
1298
1467
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
1299
1468
  && ((CAStack *) ca)->k_axis == 0
1300
- && ca_fast_paths_enabled() ) {
1469
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
1301
1470
  CAStack *__li_st = (CAStack *) ca;
1302
1471
  int8_t __li_ax = slab_axes[0];
1303
1472
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -1325,7 +1494,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1325
1494
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
1326
1495
  const ca_size_t __li_TILE = 512;
1327
1496
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
1328
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
1497
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
1329
1498
  const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
1330
1499
  ca_u_int16_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
1331
1500
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -1385,7 +1554,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1385
1554
  && ((CAStack *) ca)->k_axis > 0
1386
1555
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
1387
1556
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
1388
- && ca_fast_paths_enabled() ) {
1557
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
1389
1558
  CAStack *__li_st = (CAStack *) ca;
1390
1559
  int8_t __li_ax = slab_axes[0];
1391
1560
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -1442,7 +1611,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1442
1611
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
1443
1612
  const ca_size_t __li_TILE = 512;
1444
1613
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
1445
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
1614
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
1446
1615
  const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
1447
1616
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
1448
1617
 
@@ -1530,7 +1699,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1530
1699
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
1531
1700
  && ((CAStack *) ca)->k_axis > 0
1532
1701
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
1533
- && ca_fast_paths_enabled() ) {
1702
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
1534
1703
  CAStack *__li_st = (CAStack *) ca;
1535
1704
  int8_t __li_ax = slab_axes[0];
1536
1705
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -1602,7 +1771,7 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1602
1771
 
1603
1772
  const ca_size_t __li_TILE = 512;
1604
1773
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
1605
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
1774
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
1606
1775
  const ca_u_int16_t *p_root = (const ca_u_int16_t *) __li_st->parents[__kk]->ptr;
1607
1776
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
1608
1777
 
@@ -1651,12 +1820,41 @@ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1651
1820
  return vout;
1652
1821
  }
1653
1822
  }
1823
+ if ( naxes == 1
1824
+ && ca->ndim >= 2
1825
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
1826
+ && ca->dim[ca->ndim - 1] >= 1
1827
+ && ca->dim[ca->ndim - 1] <= 32
1828
+ && ! ca_has_mask(ca)
1829
+ && ca_attach_is_alias(ca)
1830
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
1831
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
1832
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
1833
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
1834
+ CArray *co;
1835
+ GetCArray(vout, co);
1836
+ ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
1837
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
1838
+ const ca_u_int16_t *p_root = (const ca_u_int16_t *) ca->ptr;
1839
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
1840
+ const ca_u_int16_t *__sf_row = p_root + __sf_o * __sf_M;
1841
+ ca_u_int16_t acc = (ca_u_int16_t) (0);
1842
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
1843
+ ca_u_int16_t v = __sf_row[__j];
1844
+ acc += v;
1845
+ (void) v;
1846
+ }
1847
+
1848
+ op[__sf_o] = (ca_u_int16_t) (acc);
1849
+ }
1850
+ ca_detach(ca);
1851
+ return vout;
1852
+ }
1654
1853
  }
1655
1854
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT16, keep_axis);
1656
1855
  CArray *co;
1657
1856
  GetCArray(vout, co);
1658
1857
  ca_u_int16_t *op = (ca_u_int16_t *) co->ptr;
1659
- ca_iter_state st;
1660
1858
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
1661
1859
  slab_axes, naxes, 0);
1662
1860
  if ( rc != CA_ITER_OK ) {
@@ -1690,7 +1888,22 @@ ca_size_t out_i = 0;
1690
1888
  return LL2NUM(op[0]);
1691
1889
  }
1692
1890
  return vout;
1693
- }
1891
+ #undef st
1892
+ }
1893
+
1894
+ static VALUE
1895
+ accumulate_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
1896
+ {
1897
+ accumulate_ki_native_i16_frame fr;
1898
+ fr.self = self;
1899
+ fr.ca = ca;
1900
+ fr.slab_axes = slab_axes;
1901
+ fr.naxes = naxes;
1902
+ fr.keep_axis = keep_axis;
1903
+ fr.min_count = min_count;
1904
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_i16_body, (VALUE) &fr);
1905
+ }
1906
+
1694
1907
 
1695
1908
  typedef struct {
1696
1909
  void *chunk;
@@ -1745,14 +1958,41 @@ accumulate_ki_stream_u16 (VALUE arg)
1745
1958
  return ULL2NUM((ca_u_uint16_t) (acc));
1746
1959
  }
1747
1960
 
1961
+ /* The walks can raise part way (a Ruby call per cell in the object
1962
+ lane, an allocation in any lane). Their states live in the caller's
1963
+ frame, and ca_iter_ensure finishes them however the body leaves. */
1964
+ typedef struct {
1965
+ ca_iter_state states[1];
1966
+ VALUE self;
1967
+ CArray *ca;
1968
+ int8_t *slab_axes;
1969
+ int8_t naxes;
1970
+ int keep_axis;
1971
+ ca_size_t min_count;
1972
+ } accumulate_ki_native_u16_frame;
1973
+
1748
1974
  static VALUE
1749
- accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
1975
+ accumulate_ki_native_u16_body (VALUE arg)
1750
1976
  {
1977
+ accumulate_ki_native_u16_frame *fr = (accumulate_ki_native_u16_frame *) arg;
1978
+ VALUE self = fr->self;
1979
+ (void) self;
1980
+ CArray *ca = fr->ca;
1981
+ (void) ca;
1982
+ int8_t *slab_axes = fr->slab_axes;
1983
+ (void) slab_axes;
1984
+ int8_t naxes = fr->naxes;
1985
+ (void) naxes;
1986
+ int keep_axis = fr->keep_axis;
1987
+ (void) keep_axis;
1988
+ ca_size_t min_count = fr->min_count;
1989
+ (void) min_count;
1990
+ #define st (fr->states[0])
1751
1991
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
1752
1992
  ca->elements > 0 &&
1753
1993
  ! ca_too_few_present(ca->elements, min_count, 0) &&
1754
1994
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
1755
- ca_fast_paths_enabled() ) {
1995
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
1756
1996
  accumulate_ki_stream_u16_t __ctx;
1757
1997
  __ctx.chunk = NULL;
1758
1998
  __ctx.ca = ca;
@@ -1773,7 +2013,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1773
2013
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
1774
2014
  && ! ca_has_mask(ca)
1775
2015
  && ca_attach_is_alias(ca)
1776
- && ca_fast_paths_enabled() ) {
2016
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
1777
2017
  int8_t __li_ax = slab_axes[0];
1778
2018
  ca_size_t __li_M = ca->dim[__li_ax];
1779
2019
  ca_size_t __li_INNER = 1;
@@ -1846,7 +2086,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1846
2086
  && ! ca_has_mask(ca)
1847
2087
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
1848
2088
  && ((CAStack *) ca)->k_axis == 0
1849
- && ca_fast_paths_enabled() ) {
2089
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
1850
2090
  CAStack *__li_st = (CAStack *) ca;
1851
2091
  int8_t __li_ax = slab_axes[0];
1852
2092
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -1874,7 +2114,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1874
2114
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
1875
2115
  const ca_size_t __li_TILE = 512;
1876
2116
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
1877
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
2117
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
1878
2118
  const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
1879
2119
  ca_u_uint16_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
1880
2120
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -1934,7 +2174,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1934
2174
  && ((CAStack *) ca)->k_axis > 0
1935
2175
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
1936
2176
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
1937
- && ca_fast_paths_enabled() ) {
2177
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
1938
2178
  CAStack *__li_st = (CAStack *) ca;
1939
2179
  int8_t __li_ax = slab_axes[0];
1940
2180
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -1991,7 +2231,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
1991
2231
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
1992
2232
  const ca_size_t __li_TILE = 512;
1993
2233
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
1994
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
2234
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
1995
2235
  const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
1996
2236
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
1997
2237
 
@@ -2079,7 +2319,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2079
2319
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
2080
2320
  && ((CAStack *) ca)->k_axis > 0
2081
2321
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
2082
- && ca_fast_paths_enabled() ) {
2322
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
2083
2323
  CAStack *__li_st = (CAStack *) ca;
2084
2324
  int8_t __li_ax = slab_axes[0];
2085
2325
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -2151,7 +2391,7 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2151
2391
 
2152
2392
  const ca_size_t __li_TILE = 512;
2153
2393
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
2154
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
2394
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
2155
2395
  const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) __li_st->parents[__kk]->ptr;
2156
2396
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
2157
2397
 
@@ -2200,12 +2440,41 @@ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2200
2440
  return vout;
2201
2441
  }
2202
2442
  }
2443
+ if ( naxes == 1
2444
+ && ca->ndim >= 2
2445
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
2446
+ && ca->dim[ca->ndim - 1] >= 1
2447
+ && ca->dim[ca->ndim - 1] <= 32
2448
+ && ! ca_has_mask(ca)
2449
+ && ca_attach_is_alias(ca)
2450
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
2451
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
2452
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
2453
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
2454
+ CArray *co;
2455
+ GetCArray(vout, co);
2456
+ ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
2457
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
2458
+ const ca_u_uint16_t *p_root = (const ca_u_uint16_t *) ca->ptr;
2459
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
2460
+ const ca_u_uint16_t *__sf_row = p_root + __sf_o * __sf_M;
2461
+ ca_u_uint16_t acc = (ca_u_uint16_t) (0);
2462
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
2463
+ ca_u_uint16_t v = __sf_row[__j];
2464
+ acc += v;
2465
+ (void) v;
2466
+ }
2467
+
2468
+ op[__sf_o] = (ca_u_uint16_t) (acc);
2469
+ }
2470
+ ca_detach(ca);
2471
+ return vout;
2472
+ }
2203
2473
  }
2204
2474
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT16, keep_axis);
2205
2475
  CArray *co;
2206
2476
  GetCArray(vout, co);
2207
2477
  ca_u_uint16_t *op = (ca_u_uint16_t *) co->ptr;
2208
- ca_iter_state st;
2209
2478
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
2210
2479
  slab_axes, naxes, 0);
2211
2480
  if ( rc != CA_ITER_OK ) {
@@ -2239,7 +2508,22 @@ ca_size_t out_i = 0;
2239
2508
  return ULL2NUM(op[0]);
2240
2509
  }
2241
2510
  return vout;
2242
- }
2511
+ #undef st
2512
+ }
2513
+
2514
+ static VALUE
2515
+ accumulate_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
2516
+ {
2517
+ accumulate_ki_native_u16_frame fr;
2518
+ fr.self = self;
2519
+ fr.ca = ca;
2520
+ fr.slab_axes = slab_axes;
2521
+ fr.naxes = naxes;
2522
+ fr.keep_axis = keep_axis;
2523
+ fr.min_count = min_count;
2524
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_u16_body, (VALUE) &fr);
2525
+ }
2526
+
2243
2527
 
2244
2528
  typedef struct {
2245
2529
  void *chunk;
@@ -2294,14 +2578,41 @@ accumulate_ki_stream_i32 (VALUE arg)
2294
2578
  return LL2NUM((ca_u_int32_t) (acc));
2295
2579
  }
2296
2580
 
2581
+ /* The walks can raise part way (a Ruby call per cell in the object
2582
+ lane, an allocation in any lane). Their states live in the caller's
2583
+ frame, and ca_iter_ensure finishes them however the body leaves. */
2584
+ typedef struct {
2585
+ ca_iter_state states[1];
2586
+ VALUE self;
2587
+ CArray *ca;
2588
+ int8_t *slab_axes;
2589
+ int8_t naxes;
2590
+ int keep_axis;
2591
+ ca_size_t min_count;
2592
+ } accumulate_ki_native_i32_frame;
2593
+
2297
2594
  static VALUE
2298
- accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
2595
+ accumulate_ki_native_i32_body (VALUE arg)
2299
2596
  {
2597
+ accumulate_ki_native_i32_frame *fr = (accumulate_ki_native_i32_frame *) arg;
2598
+ VALUE self = fr->self;
2599
+ (void) self;
2600
+ CArray *ca = fr->ca;
2601
+ (void) ca;
2602
+ int8_t *slab_axes = fr->slab_axes;
2603
+ (void) slab_axes;
2604
+ int8_t naxes = fr->naxes;
2605
+ (void) naxes;
2606
+ int keep_axis = fr->keep_axis;
2607
+ (void) keep_axis;
2608
+ ca_size_t min_count = fr->min_count;
2609
+ (void) min_count;
2610
+ #define st (fr->states[0])
2300
2611
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
2301
2612
  ca->elements > 0 &&
2302
2613
  ! ca_too_few_present(ca->elements, min_count, 0) &&
2303
2614
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
2304
- ca_fast_paths_enabled() ) {
2615
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
2305
2616
  accumulate_ki_stream_i32_t __ctx;
2306
2617
  __ctx.chunk = NULL;
2307
2618
  __ctx.ca = ca;
@@ -2322,7 +2633,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2322
2633
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
2323
2634
  && ! ca_has_mask(ca)
2324
2635
  && ca_attach_is_alias(ca)
2325
- && ca_fast_paths_enabled() ) {
2636
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
2326
2637
  int8_t __li_ax = slab_axes[0];
2327
2638
  ca_size_t __li_M = ca->dim[__li_ax];
2328
2639
  ca_size_t __li_INNER = 1;
@@ -2395,7 +2706,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2395
2706
  && ! ca_has_mask(ca)
2396
2707
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
2397
2708
  && ((CAStack *) ca)->k_axis == 0
2398
- && ca_fast_paths_enabled() ) {
2709
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
2399
2710
  CAStack *__li_st = (CAStack *) ca;
2400
2711
  int8_t __li_ax = slab_axes[0];
2401
2712
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -2423,7 +2734,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2423
2734
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
2424
2735
  const ca_size_t __li_TILE = 512;
2425
2736
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
2426
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
2737
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
2427
2738
  const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
2428
2739
  ca_u_int32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
2429
2740
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -2483,7 +2794,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2483
2794
  && ((CAStack *) ca)->k_axis > 0
2484
2795
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
2485
2796
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
2486
- && ca_fast_paths_enabled() ) {
2797
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
2487
2798
  CAStack *__li_st = (CAStack *) ca;
2488
2799
  int8_t __li_ax = slab_axes[0];
2489
2800
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -2540,7 +2851,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2540
2851
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
2541
2852
  const ca_size_t __li_TILE = 512;
2542
2853
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
2543
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
2854
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
2544
2855
  const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
2545
2856
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
2546
2857
 
@@ -2628,7 +2939,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2628
2939
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
2629
2940
  && ((CAStack *) ca)->k_axis > 0
2630
2941
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
2631
- && ca_fast_paths_enabled() ) {
2942
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
2632
2943
  CAStack *__li_st = (CAStack *) ca;
2633
2944
  int8_t __li_ax = slab_axes[0];
2634
2945
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -2700,7 +3011,7 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2700
3011
 
2701
3012
  const ca_size_t __li_TILE = 512;
2702
3013
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
2703
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
3014
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
2704
3015
  const ca_u_int32_t *p_root = (const ca_u_int32_t *) __li_st->parents[__kk]->ptr;
2705
3016
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
2706
3017
 
@@ -2749,12 +3060,41 @@ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2749
3060
  return vout;
2750
3061
  }
2751
3062
  }
3063
+ if ( naxes == 1
3064
+ && ca->ndim >= 2
3065
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
3066
+ && ca->dim[ca->ndim - 1] >= 1
3067
+ && ca->dim[ca->ndim - 1] <= 32
3068
+ && ! ca_has_mask(ca)
3069
+ && ca_attach_is_alias(ca)
3070
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
3071
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
3072
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
3073
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
3074
+ CArray *co;
3075
+ GetCArray(vout, co);
3076
+ ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
3077
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
3078
+ const ca_u_int32_t *p_root = (const ca_u_int32_t *) ca->ptr;
3079
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
3080
+ const ca_u_int32_t *__sf_row = p_root + __sf_o * __sf_M;
3081
+ ca_u_int32_t acc = (ca_u_int32_t) (0);
3082
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
3083
+ ca_u_int32_t v = __sf_row[__j];
3084
+ acc += v;
3085
+ (void) v;
3086
+ }
3087
+
3088
+ op[__sf_o] = (ca_u_int32_t) (acc);
3089
+ }
3090
+ ca_detach(ca);
3091
+ return vout;
3092
+ }
2752
3093
  }
2753
3094
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT32, keep_axis);
2754
3095
  CArray *co;
2755
3096
  GetCArray(vout, co);
2756
3097
  ca_u_int32_t *op = (ca_u_int32_t *) co->ptr;
2757
- ca_iter_state st;
2758
3098
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
2759
3099
  slab_axes, naxes, 0);
2760
3100
  if ( rc != CA_ITER_OK ) {
@@ -2788,7 +3128,22 @@ ca_size_t out_i = 0;
2788
3128
  return LL2NUM(op[0]);
2789
3129
  }
2790
3130
  return vout;
2791
- }
3131
+ #undef st
3132
+ }
3133
+
3134
+ static VALUE
3135
+ accumulate_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
3136
+ {
3137
+ accumulate_ki_native_i32_frame fr;
3138
+ fr.self = self;
3139
+ fr.ca = ca;
3140
+ fr.slab_axes = slab_axes;
3141
+ fr.naxes = naxes;
3142
+ fr.keep_axis = keep_axis;
3143
+ fr.min_count = min_count;
3144
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_i32_body, (VALUE) &fr);
3145
+ }
3146
+
2792
3147
 
2793
3148
  typedef struct {
2794
3149
  void *chunk;
@@ -2843,21 +3198,48 @@ accumulate_ki_stream_u32 (VALUE arg)
2843
3198
  return ULL2NUM((ca_u_uint32_t) (acc));
2844
3199
  }
2845
3200
 
3201
+ /* The walks can raise part way (a Ruby call per cell in the object
3202
+ lane, an allocation in any lane). Their states live in the caller's
3203
+ frame, and ca_iter_ensure finishes them however the body leaves. */
3204
+ typedef struct {
3205
+ ca_iter_state states[1];
3206
+ VALUE self;
3207
+ CArray *ca;
3208
+ int8_t *slab_axes;
3209
+ int8_t naxes;
3210
+ int keep_axis;
3211
+ ca_size_t min_count;
3212
+ } accumulate_ki_native_u32_frame;
3213
+
2846
3214
  static VALUE
2847
- accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
3215
+ accumulate_ki_native_u32_body (VALUE arg)
2848
3216
  {
2849
- if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
2850
- ca->elements > 0 &&
2851
- ! ca_too_few_present(ca->elements, min_count, 0) &&
2852
- ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
2853
- ca_fast_paths_enabled() ) {
2854
- accumulate_ki_stream_u32_t __ctx;
2855
- __ctx.chunk = NULL;
2856
- __ctx.ca = ca;
2857
- __ctx.min_count = min_count;
2858
- return ca_lazy_arena_protect(accumulate_ki_stream_u32, ca_lazy_arena_release_held,
2859
- (VALUE) &__ctx);
2860
- }
3217
+ accumulate_ki_native_u32_frame *fr = (accumulate_ki_native_u32_frame *) arg;
3218
+ VALUE self = fr->self;
3219
+ (void) self;
3220
+ CArray *ca = fr->ca;
3221
+ (void) ca;
3222
+ int8_t *slab_axes = fr->slab_axes;
3223
+ (void) slab_axes;
3224
+ int8_t naxes = fr->naxes;
3225
+ (void) naxes;
3226
+ int keep_axis = fr->keep_axis;
3227
+ (void) keep_axis;
3228
+ ca_size_t min_count = fr->min_count;
3229
+ (void) min_count;
3230
+ #define st (fr->states[0])
3231
+ if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
3232
+ ca->elements > 0 &&
3233
+ ! ca_too_few_present(ca->elements, min_count, 0) &&
3234
+ ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
3235
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
3236
+ accumulate_ki_stream_u32_t __ctx;
3237
+ __ctx.chunk = NULL;
3238
+ __ctx.ca = ca;
3239
+ __ctx.min_count = min_count;
3240
+ return ca_lazy_arena_protect(accumulate_ki_stream_u32, ca_lazy_arena_release_held,
3241
+ (VALUE) &__ctx);
3242
+ }
2861
3243
  if ( naxes != 1 || ! ca_too_few_present(ca->dim[slab_axes[0]], min_count, 0) ) {
2862
3244
  /* L.1 / L.7: loop-interchange fast path with inner-tiling
2863
3245
  (PROPOSAL_REDUCTION_LOOP_INTERCHANGE).
@@ -2871,7 +3253,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2871
3253
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
2872
3254
  && ! ca_has_mask(ca)
2873
3255
  && ca_attach_is_alias(ca)
2874
- && ca_fast_paths_enabled() ) {
3256
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
2875
3257
  int8_t __li_ax = slab_axes[0];
2876
3258
  ca_size_t __li_M = ca->dim[__li_ax];
2877
3259
  ca_size_t __li_INNER = 1;
@@ -2944,7 +3326,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2944
3326
  && ! ca_has_mask(ca)
2945
3327
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
2946
3328
  && ((CAStack *) ca)->k_axis == 0
2947
- && ca_fast_paths_enabled() ) {
3329
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
2948
3330
  CAStack *__li_st = (CAStack *) ca;
2949
3331
  int8_t __li_ax = slab_axes[0];
2950
3332
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -2972,7 +3354,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
2972
3354
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
2973
3355
  const ca_size_t __li_TILE = 512;
2974
3356
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
2975
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
3357
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
2976
3358
  const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
2977
3359
  ca_u_uint32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
2978
3360
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -3032,7 +3414,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3032
3414
  && ((CAStack *) ca)->k_axis > 0
3033
3415
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
3034
3416
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
3035
- && ca_fast_paths_enabled() ) {
3417
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
3036
3418
  CAStack *__li_st = (CAStack *) ca;
3037
3419
  int8_t __li_ax = slab_axes[0];
3038
3420
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -3089,7 +3471,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3089
3471
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
3090
3472
  const ca_size_t __li_TILE = 512;
3091
3473
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
3092
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
3474
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
3093
3475
  const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
3094
3476
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
3095
3477
 
@@ -3177,7 +3559,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3177
3559
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
3178
3560
  && ((CAStack *) ca)->k_axis > 0
3179
3561
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
3180
- && ca_fast_paths_enabled() ) {
3562
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
3181
3563
  CAStack *__li_st = (CAStack *) ca;
3182
3564
  int8_t __li_ax = slab_axes[0];
3183
3565
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -3249,7 +3631,7 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3249
3631
 
3250
3632
  const ca_size_t __li_TILE = 512;
3251
3633
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
3252
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
3634
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
3253
3635
  const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) __li_st->parents[__kk]->ptr;
3254
3636
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
3255
3637
 
@@ -3298,12 +3680,41 @@ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3298
3680
  return vout;
3299
3681
  }
3300
3682
  }
3683
+ if ( naxes == 1
3684
+ && ca->ndim >= 2
3685
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
3686
+ && ca->dim[ca->ndim - 1] >= 1
3687
+ && ca->dim[ca->ndim - 1] <= 32
3688
+ && ! ca_has_mask(ca)
3689
+ && ca_attach_is_alias(ca)
3690
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
3691
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
3692
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
3693
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
3694
+ CArray *co;
3695
+ GetCArray(vout, co);
3696
+ ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
3697
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
3698
+ const ca_u_uint32_t *p_root = (const ca_u_uint32_t *) ca->ptr;
3699
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
3700
+ const ca_u_uint32_t *__sf_row = p_root + __sf_o * __sf_M;
3701
+ ca_u_uint32_t acc = (ca_u_uint32_t) (0);
3702
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
3703
+ ca_u_uint32_t v = __sf_row[__j];
3704
+ acc += v;
3705
+ (void) v;
3706
+ }
3707
+
3708
+ op[__sf_o] = (ca_u_uint32_t) (acc);
3709
+ }
3710
+ ca_detach(ca);
3711
+ return vout;
3712
+ }
3301
3713
  }
3302
3714
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT32, keep_axis);
3303
3715
  CArray *co;
3304
3716
  GetCArray(vout, co);
3305
3717
  ca_u_uint32_t *op = (ca_u_uint32_t *) co->ptr;
3306
- ca_iter_state st;
3307
3718
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
3308
3719
  slab_axes, naxes, 0);
3309
3720
  if ( rc != CA_ITER_OK ) {
@@ -3337,7 +3748,22 @@ ca_size_t out_i = 0;
3337
3748
  return ULL2NUM(op[0]);
3338
3749
  }
3339
3750
  return vout;
3340
- }
3751
+ #undef st
3752
+ }
3753
+
3754
+ static VALUE
3755
+ accumulate_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
3756
+ {
3757
+ accumulate_ki_native_u32_frame fr;
3758
+ fr.self = self;
3759
+ fr.ca = ca;
3760
+ fr.slab_axes = slab_axes;
3761
+ fr.naxes = naxes;
3762
+ fr.keep_axis = keep_axis;
3763
+ fr.min_count = min_count;
3764
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_u32_body, (VALUE) &fr);
3765
+ }
3766
+
3341
3767
 
3342
3768
  typedef struct {
3343
3769
  void *chunk;
@@ -3392,14 +3818,41 @@ accumulate_ki_stream_i64 (VALUE arg)
3392
3818
  return LL2NUM((ca_u_int64_t) (acc));
3393
3819
  }
3394
3820
 
3821
+ /* The walks can raise part way (a Ruby call per cell in the object
3822
+ lane, an allocation in any lane). Their states live in the caller's
3823
+ frame, and ca_iter_ensure finishes them however the body leaves. */
3824
+ typedef struct {
3825
+ ca_iter_state states[1];
3826
+ VALUE self;
3827
+ CArray *ca;
3828
+ int8_t *slab_axes;
3829
+ int8_t naxes;
3830
+ int keep_axis;
3831
+ ca_size_t min_count;
3832
+ } accumulate_ki_native_i64_frame;
3833
+
3395
3834
  static VALUE
3396
- accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
3835
+ accumulate_ki_native_i64_body (VALUE arg)
3397
3836
  {
3837
+ accumulate_ki_native_i64_frame *fr = (accumulate_ki_native_i64_frame *) arg;
3838
+ VALUE self = fr->self;
3839
+ (void) self;
3840
+ CArray *ca = fr->ca;
3841
+ (void) ca;
3842
+ int8_t *slab_axes = fr->slab_axes;
3843
+ (void) slab_axes;
3844
+ int8_t naxes = fr->naxes;
3845
+ (void) naxes;
3846
+ int keep_axis = fr->keep_axis;
3847
+ (void) keep_axis;
3848
+ ca_size_t min_count = fr->min_count;
3849
+ (void) min_count;
3850
+ #define st (fr->states[0])
3398
3851
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
3399
3852
  ca->elements > 0 &&
3400
3853
  ! ca_too_few_present(ca->elements, min_count, 0) &&
3401
3854
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
3402
- ca_fast_paths_enabled() ) {
3855
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
3403
3856
  accumulate_ki_stream_i64_t __ctx;
3404
3857
  __ctx.chunk = NULL;
3405
3858
  __ctx.ca = ca;
@@ -3420,7 +3873,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3420
3873
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
3421
3874
  && ! ca_has_mask(ca)
3422
3875
  && ca_attach_is_alias(ca)
3423
- && ca_fast_paths_enabled() ) {
3876
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
3424
3877
  int8_t __li_ax = slab_axes[0];
3425
3878
  ca_size_t __li_M = ca->dim[__li_ax];
3426
3879
  ca_size_t __li_INNER = 1;
@@ -3493,7 +3946,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3493
3946
  && ! ca_has_mask(ca)
3494
3947
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
3495
3948
  && ((CAStack *) ca)->k_axis == 0
3496
- && ca_fast_paths_enabled() ) {
3949
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
3497
3950
  CAStack *__li_st = (CAStack *) ca;
3498
3951
  int8_t __li_ax = slab_axes[0];
3499
3952
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -3521,7 +3974,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3521
3974
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
3522
3975
  const ca_size_t __li_TILE = 512;
3523
3976
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
3524
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
3977
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
3525
3978
  const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
3526
3979
  ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
3527
3980
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -3581,7 +4034,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3581
4034
  && ((CAStack *) ca)->k_axis > 0
3582
4035
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
3583
4036
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
3584
- && ca_fast_paths_enabled() ) {
4037
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
3585
4038
  CAStack *__li_st = (CAStack *) ca;
3586
4039
  int8_t __li_ax = slab_axes[0];
3587
4040
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -3638,7 +4091,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3638
4091
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
3639
4092
  const ca_size_t __li_TILE = 512;
3640
4093
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
3641
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
4094
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
3642
4095
  const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
3643
4096
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
3644
4097
 
@@ -3726,7 +4179,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3726
4179
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
3727
4180
  && ((CAStack *) ca)->k_axis > 0
3728
4181
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
3729
- && ca_fast_paths_enabled() ) {
4182
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
3730
4183
  CAStack *__li_st = (CAStack *) ca;
3731
4184
  int8_t __li_ax = slab_axes[0];
3732
4185
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -3798,7 +4251,7 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3798
4251
 
3799
4252
  const ca_size_t __li_TILE = 512;
3800
4253
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
3801
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
4254
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
3802
4255
  const ca_u_int64_t *p_root = (const ca_u_int64_t *) __li_st->parents[__kk]->ptr;
3803
4256
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
3804
4257
 
@@ -3847,12 +4300,41 @@ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3847
4300
  return vout;
3848
4301
  }
3849
4302
  }
4303
+ if ( naxes == 1
4304
+ && ca->ndim >= 2
4305
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
4306
+ && ca->dim[ca->ndim - 1] >= 1
4307
+ && ca->dim[ca->ndim - 1] <= 32
4308
+ && ! ca_has_mask(ca)
4309
+ && ca_attach_is_alias(ca)
4310
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
4311
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
4312
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
4313
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
4314
+ CArray *co;
4315
+ GetCArray(vout, co);
4316
+ ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
4317
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
4318
+ const ca_u_int64_t *p_root = (const ca_u_int64_t *) ca->ptr;
4319
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
4320
+ const ca_u_int64_t *__sf_row = p_root + __sf_o * __sf_M;
4321
+ ca_u_int64_t acc = (ca_u_int64_t) (0);
4322
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
4323
+ ca_u_int64_t v = __sf_row[__j];
4324
+ acc += v;
4325
+ (void) v;
4326
+ }
4327
+
4328
+ op[__sf_o] = (ca_u_int64_t) (acc);
4329
+ }
4330
+ ca_detach(ca);
4331
+ return vout;
4332
+ }
3850
4333
  }
3851
4334
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
3852
4335
  CArray *co;
3853
4336
  GetCArray(vout, co);
3854
4337
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
3855
- ca_iter_state st;
3856
4338
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
3857
4339
  slab_axes, naxes, 0);
3858
4340
  if ( rc != CA_ITER_OK ) {
@@ -3886,7 +4368,22 @@ ca_size_t out_i = 0;
3886
4368
  return LL2NUM(op[0]);
3887
4369
  }
3888
4370
  return vout;
3889
- }
4371
+ #undef st
4372
+ }
4373
+
4374
+ static VALUE
4375
+ accumulate_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
4376
+ {
4377
+ accumulate_ki_native_i64_frame fr;
4378
+ fr.self = self;
4379
+ fr.ca = ca;
4380
+ fr.slab_axes = slab_axes;
4381
+ fr.naxes = naxes;
4382
+ fr.keep_axis = keep_axis;
4383
+ fr.min_count = min_count;
4384
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_i64_body, (VALUE) &fr);
4385
+ }
4386
+
3890
4387
 
3891
4388
  typedef struct {
3892
4389
  void *chunk;
@@ -3941,14 +4438,41 @@ accumulate_ki_stream_u64 (VALUE arg)
3941
4438
  return ULL2NUM((ca_u_uint64_t) (acc));
3942
4439
  }
3943
4440
 
4441
+ /* The walks can raise part way (a Ruby call per cell in the object
4442
+ lane, an allocation in any lane). Their states live in the caller's
4443
+ frame, and ca_iter_ensure finishes them however the body leaves. */
4444
+ typedef struct {
4445
+ ca_iter_state states[1];
4446
+ VALUE self;
4447
+ CArray *ca;
4448
+ int8_t *slab_axes;
4449
+ int8_t naxes;
4450
+ int keep_axis;
4451
+ ca_size_t min_count;
4452
+ } accumulate_ki_native_u64_frame;
4453
+
3944
4454
  static VALUE
3945
- accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
4455
+ accumulate_ki_native_u64_body (VALUE arg)
3946
4456
  {
4457
+ accumulate_ki_native_u64_frame *fr = (accumulate_ki_native_u64_frame *) arg;
4458
+ VALUE self = fr->self;
4459
+ (void) self;
4460
+ CArray *ca = fr->ca;
4461
+ (void) ca;
4462
+ int8_t *slab_axes = fr->slab_axes;
4463
+ (void) slab_axes;
4464
+ int8_t naxes = fr->naxes;
4465
+ (void) naxes;
4466
+ int keep_axis = fr->keep_axis;
4467
+ (void) keep_axis;
4468
+ ca_size_t min_count = fr->min_count;
4469
+ (void) min_count;
4470
+ #define st (fr->states[0])
3947
4471
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
3948
4472
  ca->elements > 0 &&
3949
4473
  ! ca_too_few_present(ca->elements, min_count, 0) &&
3950
4474
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
3951
- ca_fast_paths_enabled() ) {
4475
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
3952
4476
  accumulate_ki_stream_u64_t __ctx;
3953
4477
  __ctx.chunk = NULL;
3954
4478
  __ctx.ca = ca;
@@ -3969,7 +4493,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
3969
4493
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
3970
4494
  && ! ca_has_mask(ca)
3971
4495
  && ca_attach_is_alias(ca)
3972
- && ca_fast_paths_enabled() ) {
4496
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
3973
4497
  int8_t __li_ax = slab_axes[0];
3974
4498
  ca_size_t __li_M = ca->dim[__li_ax];
3975
4499
  ca_size_t __li_INNER = 1;
@@ -4042,7 +4566,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4042
4566
  && ! ca_has_mask(ca)
4043
4567
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
4044
4568
  && ((CAStack *) ca)->k_axis == 0
4045
- && ca_fast_paths_enabled() ) {
4569
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
4046
4570
  CAStack *__li_st = (CAStack *) ca;
4047
4571
  int8_t __li_ax = slab_axes[0];
4048
4572
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -4070,7 +4594,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4070
4594
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
4071
4595
  const ca_size_t __li_TILE = 512;
4072
4596
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
4073
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
4597
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
4074
4598
  const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
4075
4599
  ca_u_uint64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
4076
4600
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -4130,7 +4654,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4130
4654
  && ((CAStack *) ca)->k_axis > 0
4131
4655
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
4132
4656
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
4133
- && ca_fast_paths_enabled() ) {
4657
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
4134
4658
  CAStack *__li_st = (CAStack *) ca;
4135
4659
  int8_t __li_ax = slab_axes[0];
4136
4660
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -4187,7 +4711,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4187
4711
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
4188
4712
  const ca_size_t __li_TILE = 512;
4189
4713
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
4190
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
4714
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
4191
4715
  const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
4192
4716
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
4193
4717
 
@@ -4275,7 +4799,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4275
4799
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
4276
4800
  && ((CAStack *) ca)->k_axis > 0
4277
4801
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
4278
- && ca_fast_paths_enabled() ) {
4802
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
4279
4803
  CAStack *__li_st = (CAStack *) ca;
4280
4804
  int8_t __li_ax = slab_axes[0];
4281
4805
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -4347,7 +4871,7 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4347
4871
 
4348
4872
  const ca_size_t __li_TILE = 512;
4349
4873
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
4350
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
4874
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
4351
4875
  const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) __li_st->parents[__kk]->ptr;
4352
4876
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
4353
4877
 
@@ -4396,12 +4920,41 @@ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4396
4920
  return vout;
4397
4921
  }
4398
4922
  }
4923
+ if ( naxes == 1
4924
+ && ca->ndim >= 2
4925
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
4926
+ && ca->dim[ca->ndim - 1] >= 1
4927
+ && ca->dim[ca->ndim - 1] <= 32
4928
+ && ! ca_has_mask(ca)
4929
+ && ca_attach_is_alias(ca)
4930
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
4931
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
4932
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
4933
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
4934
+ CArray *co;
4935
+ GetCArray(vout, co);
4936
+ ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
4937
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
4938
+ const ca_u_uint64_t *p_root = (const ca_u_uint64_t *) ca->ptr;
4939
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
4940
+ const ca_u_uint64_t *__sf_row = p_root + __sf_o * __sf_M;
4941
+ ca_u_uint64_t acc = (ca_u_uint64_t) (0);
4942
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
4943
+ ca_u_uint64_t v = __sf_row[__j];
4944
+ acc += v;
4945
+ (void) v;
4946
+ }
4947
+
4948
+ op[__sf_o] = (ca_u_uint64_t) (acc);
4949
+ }
4950
+ ca_detach(ca);
4951
+ return vout;
4952
+ }
4399
4953
  }
4400
4954
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_UINT64, keep_axis);
4401
4955
  CArray *co;
4402
4956
  GetCArray(vout, co);
4403
4957
  ca_u_uint64_t *op = (ca_u_uint64_t *) co->ptr;
4404
- ca_iter_state st;
4405
4958
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
4406
4959
  slab_axes, naxes, 0);
4407
4960
  if ( rc != CA_ITER_OK ) {
@@ -4435,7 +4988,22 @@ ca_size_t out_i = 0;
4435
4988
  return ULL2NUM(op[0]);
4436
4989
  }
4437
4990
  return vout;
4438
- }
4991
+ #undef st
4992
+ }
4993
+
4994
+ static VALUE
4995
+ accumulate_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
4996
+ {
4997
+ accumulate_ki_native_u64_frame fr;
4998
+ fr.self = self;
4999
+ fr.ca = ca;
5000
+ fr.slab_axes = slab_axes;
5001
+ fr.naxes = naxes;
5002
+ fr.keep_axis = keep_axis;
5003
+ fr.min_count = min_count;
5004
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_u64_body, (VALUE) &fr);
5005
+ }
5006
+
4439
5007
 
4440
5008
  typedef struct {
4441
5009
  void *chunk;
@@ -4490,14 +5058,41 @@ accumulate_ki_stream_f32 (VALUE arg)
4490
5058
  return rb_float_new((ca_u_float32_t) (acc));
4491
5059
  }
4492
5060
 
5061
+ /* The walks can raise part way (a Ruby call per cell in the object
5062
+ lane, an allocation in any lane). Their states live in the caller's
5063
+ frame, and ca_iter_ensure finishes them however the body leaves. */
5064
+ typedef struct {
5065
+ ca_iter_state states[1];
5066
+ VALUE self;
5067
+ CArray *ca;
5068
+ int8_t *slab_axes;
5069
+ int8_t naxes;
5070
+ int keep_axis;
5071
+ ca_size_t min_count;
5072
+ } accumulate_ki_native_f32_frame;
5073
+
4493
5074
  static VALUE
4494
- accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
5075
+ accumulate_ki_native_f32_body (VALUE arg)
4495
5076
  {
5077
+ accumulate_ki_native_f32_frame *fr = (accumulate_ki_native_f32_frame *) arg;
5078
+ VALUE self = fr->self;
5079
+ (void) self;
5080
+ CArray *ca = fr->ca;
5081
+ (void) ca;
5082
+ int8_t *slab_axes = fr->slab_axes;
5083
+ (void) slab_axes;
5084
+ int8_t naxes = fr->naxes;
5085
+ (void) naxes;
5086
+ int keep_axis = fr->keep_axis;
5087
+ (void) keep_axis;
5088
+ ca_size_t min_count = fr->min_count;
5089
+ (void) min_count;
5090
+ #define st (fr->states[0])
4496
5091
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
4497
5092
  ca->elements > 0 &&
4498
5093
  ! ca_too_few_present(ca->elements, min_count, 0) &&
4499
5094
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
4500
- ca_fast_paths_enabled() ) {
5095
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
4501
5096
  accumulate_ki_stream_f32_t __ctx;
4502
5097
  __ctx.chunk = NULL;
4503
5098
  __ctx.ca = ca;
@@ -4518,7 +5113,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4518
5113
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
4519
5114
  && ! ca_has_mask(ca)
4520
5115
  && ca_attach_is_alias(ca)
4521
- && ca_fast_paths_enabled() ) {
5116
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
4522
5117
  int8_t __li_ax = slab_axes[0];
4523
5118
  ca_size_t __li_M = ca->dim[__li_ax];
4524
5119
  ca_size_t __li_INNER = 1;
@@ -4591,7 +5186,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4591
5186
  && ! ca_has_mask(ca)
4592
5187
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
4593
5188
  && ((CAStack *) ca)->k_axis == 0
4594
- && ca_fast_paths_enabled() ) {
5189
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
4595
5190
  CAStack *__li_st = (CAStack *) ca;
4596
5191
  int8_t __li_ax = slab_axes[0];
4597
5192
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -4619,7 +5214,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4619
5214
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
4620
5215
  const ca_size_t __li_TILE = 512;
4621
5216
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
4622
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
5217
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
4623
5218
  const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
4624
5219
  ca_u_float32_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
4625
5220
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -4679,7 +5274,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4679
5274
  && ((CAStack *) ca)->k_axis > 0
4680
5275
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
4681
5276
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
4682
- && ca_fast_paths_enabled() ) {
5277
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
4683
5278
  CAStack *__li_st = (CAStack *) ca;
4684
5279
  int8_t __li_ax = slab_axes[0];
4685
5280
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -4736,7 +5331,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4736
5331
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
4737
5332
  const ca_size_t __li_TILE = 512;
4738
5333
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
4739
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
5334
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
4740
5335
  const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
4741
5336
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
4742
5337
 
@@ -4824,7 +5419,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4824
5419
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
4825
5420
  && ((CAStack *) ca)->k_axis > 0
4826
5421
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
4827
- && ca_fast_paths_enabled() ) {
5422
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
4828
5423
  CAStack *__li_st = (CAStack *) ca;
4829
5424
  int8_t __li_ax = slab_axes[0];
4830
5425
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -4896,7 +5491,7 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4896
5491
 
4897
5492
  const ca_size_t __li_TILE = 512;
4898
5493
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
4899
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
5494
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
4900
5495
  const ca_u_float32_t *p_root = (const ca_u_float32_t *) __li_st->parents[__kk]->ptr;
4901
5496
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
4902
5497
 
@@ -4945,12 +5540,41 @@ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
4945
5540
  return vout;
4946
5541
  }
4947
5542
  }
5543
+ if ( naxes == 1
5544
+ && ca->ndim >= 2
5545
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
5546
+ && ca->dim[ca->ndim - 1] >= 1
5547
+ && ca->dim[ca->ndim - 1] <= 32
5548
+ && ! ca_has_mask(ca)
5549
+ && ca_attach_is_alias(ca)
5550
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
5551
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
5552
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
5553
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
5554
+ CArray *co;
5555
+ GetCArray(vout, co);
5556
+ ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
5557
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
5558
+ const ca_u_float32_t *p_root = (const ca_u_float32_t *) ca->ptr;
5559
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
5560
+ const ca_u_float32_t *__sf_row = p_root + __sf_o * __sf_M;
5561
+ ca_u_float32_t acc = (ca_u_float32_t) (0);
5562
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
5563
+ ca_u_float32_t v = __sf_row[__j];
5564
+ acc += v;
5565
+ (void) v;
5566
+ }
5567
+
5568
+ op[__sf_o] = (ca_u_float32_t) (acc);
5569
+ }
5570
+ ca_detach(ca);
5571
+ return vout;
5572
+ }
4948
5573
  }
4949
5574
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT32, keep_axis);
4950
5575
  CArray *co;
4951
5576
  GetCArray(vout, co);
4952
5577
  ca_u_float32_t *op = (ca_u_float32_t *) co->ptr;
4953
- ca_iter_state st;
4954
5578
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
4955
5579
  slab_axes, naxes, 0);
4956
5580
  if ( rc != CA_ITER_OK ) {
@@ -4984,7 +5608,22 @@ ca_size_t out_i = 0;
4984
5608
  return rb_float_new(op[0]);
4985
5609
  }
4986
5610
  return vout;
4987
- }
5611
+ #undef st
5612
+ }
5613
+
5614
+ static VALUE
5615
+ accumulate_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
5616
+ {
5617
+ accumulate_ki_native_f32_frame fr;
5618
+ fr.self = self;
5619
+ fr.ca = ca;
5620
+ fr.slab_axes = slab_axes;
5621
+ fr.naxes = naxes;
5622
+ fr.keep_axis = keep_axis;
5623
+ fr.min_count = min_count;
5624
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_f32_body, (VALUE) &fr);
5625
+ }
5626
+
4988
5627
 
4989
5628
  typedef struct {
4990
5629
  void *chunk;
@@ -5039,14 +5678,41 @@ accumulate_ki_stream_f64 (VALUE arg)
5039
5678
  return rb_float_new((ca_u_float64_t) (acc));
5040
5679
  }
5041
5680
 
5681
+ /* The walks can raise part way (a Ruby call per cell in the object
5682
+ lane, an allocation in any lane). Their states live in the caller's
5683
+ frame, and ca_iter_ensure finishes them however the body leaves. */
5684
+ typedef struct {
5685
+ ca_iter_state states[1];
5686
+ VALUE self;
5687
+ CArray *ca;
5688
+ int8_t *slab_axes;
5689
+ int8_t naxes;
5690
+ int keep_axis;
5691
+ ca_size_t min_count;
5692
+ } accumulate_ki_native_f64_frame;
5693
+
5042
5694
  static VALUE
5043
- accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
5695
+ accumulate_ki_native_f64_body (VALUE arg)
5044
5696
  {
5697
+ accumulate_ki_native_f64_frame *fr = (accumulate_ki_native_f64_frame *) arg;
5698
+ VALUE self = fr->self;
5699
+ (void) self;
5700
+ CArray *ca = fr->ca;
5701
+ (void) ca;
5702
+ int8_t *slab_axes = fr->slab_axes;
5703
+ (void) slab_axes;
5704
+ int8_t naxes = fr->naxes;
5705
+ (void) naxes;
5706
+ int keep_axis = fr->keep_axis;
5707
+ (void) keep_axis;
5708
+ ca_size_t min_count = fr->min_count;
5709
+ (void) min_count;
5710
+ #define st (fr->states[0])
5045
5711
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
5046
5712
  ca->elements > 0 &&
5047
5713
  ! ca_too_few_present(ca->elements, min_count, 0) &&
5048
5714
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
5049
- ca_fast_paths_enabled() ) {
5715
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
5050
5716
  accumulate_ki_stream_f64_t __ctx;
5051
5717
  __ctx.chunk = NULL;
5052
5718
  __ctx.ca = ca;
@@ -5067,7 +5733,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5067
5733
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
5068
5734
  && ! ca_has_mask(ca)
5069
5735
  && ca_attach_is_alias(ca)
5070
- && ca_fast_paths_enabled() ) {
5736
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
5071
5737
  int8_t __li_ax = slab_axes[0];
5072
5738
  ca_size_t __li_M = ca->dim[__li_ax];
5073
5739
  ca_size_t __li_INNER = 1;
@@ -5140,7 +5806,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5140
5806
  && ! ca_has_mask(ca)
5141
5807
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
5142
5808
  && ((CAStack *) ca)->k_axis == 0
5143
- && ca_fast_paths_enabled() ) {
5809
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
5144
5810
  CAStack *__li_st = (CAStack *) ca;
5145
5811
  int8_t __li_ax = slab_axes[0];
5146
5812
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -5168,7 +5834,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5168
5834
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
5169
5835
  const ca_size_t __li_TILE = 512;
5170
5836
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
5171
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
5837
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
5172
5838
  const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
5173
5839
  ca_u_float64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
5174
5840
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -5228,7 +5894,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5228
5894
  && ((CAStack *) ca)->k_axis > 0
5229
5895
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
5230
5896
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
5231
- && ca_fast_paths_enabled() ) {
5897
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
5232
5898
  CAStack *__li_st = (CAStack *) ca;
5233
5899
  int8_t __li_ax = slab_axes[0];
5234
5900
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -5285,7 +5951,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5285
5951
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
5286
5952
  const ca_size_t __li_TILE = 512;
5287
5953
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
5288
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
5954
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
5289
5955
  const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
5290
5956
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
5291
5957
 
@@ -5373,7 +6039,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5373
6039
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
5374
6040
  && ((CAStack *) ca)->k_axis > 0
5375
6041
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
5376
- && ca_fast_paths_enabled() ) {
6042
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
5377
6043
  CAStack *__li_st = (CAStack *) ca;
5378
6044
  int8_t __li_ax = slab_axes[0];
5379
6045
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -5445,7 +6111,7 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5445
6111
 
5446
6112
  const ca_size_t __li_TILE = 512;
5447
6113
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
5448
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
6114
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
5449
6115
  const ca_u_float64_t *p_root = (const ca_u_float64_t *) __li_st->parents[__kk]->ptr;
5450
6116
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
5451
6117
 
@@ -5494,12 +6160,41 @@ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxe
5494
6160
  return vout;
5495
6161
  }
5496
6162
  }
6163
+ if ( naxes == 1
6164
+ && ca->ndim >= 2
6165
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
6166
+ && ca->dim[ca->ndim - 1] >= 1
6167
+ && ca->dim[ca->ndim - 1] <= 32
6168
+ && ! ca_has_mask(ca)
6169
+ && ca_attach_is_alias(ca)
6170
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
6171
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
6172
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
6173
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
6174
+ CArray *co;
6175
+ GetCArray(vout, co);
6176
+ ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
6177
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
6178
+ const ca_u_float64_t *p_root = (const ca_u_float64_t *) ca->ptr;
6179
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
6180
+ const ca_u_float64_t *__sf_row = p_root + __sf_o * __sf_M;
6181
+ ca_u_float64_t acc = (ca_u_float64_t) (0);
6182
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
6183
+ ca_u_float64_t v = __sf_row[__j];
6184
+ acc += v;
6185
+ (void) v;
6186
+ }
6187
+
6188
+ op[__sf_o] = (ca_u_float64_t) (acc);
6189
+ }
6190
+ ca_detach(ca);
6191
+ return vout;
6192
+ }
5497
6193
  }
5498
6194
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_FLOAT64, keep_axis);
5499
6195
  CArray *co;
5500
6196
  GetCArray(vout, co);
5501
6197
  ca_u_float64_t *op = (ca_u_float64_t *) co->ptr;
5502
- ca_iter_state st;
5503
6198
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
5504
6199
  slab_axes, naxes, 0);
5505
6200
  if ( rc != CA_ITER_OK ) {
@@ -5533,7 +6228,22 @@ ca_size_t out_i = 0;
5533
6228
  return rb_float_new(op[0]);
5534
6229
  }
5535
6230
  return vout;
5536
- }
6231
+ #undef st
6232
+ }
6233
+
6234
+ static VALUE
6235
+ accumulate_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
6236
+ {
6237
+ accumulate_ki_native_f64_frame fr;
6238
+ fr.self = self;
6239
+ fr.ca = ca;
6240
+ fr.slab_axes = slab_axes;
6241
+ fr.naxes = naxes;
6242
+ fr.keep_axis = keep_axis;
6243
+ fr.min_count = min_count;
6244
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_f64_body, (VALUE) &fr);
6245
+ }
6246
+
5537
6247
 
5538
6248
  typedef struct {
5539
6249
  void *chunk;
@@ -5588,20 +6298,47 @@ accumulate_ki_stream_cmplx64 (VALUE arg)
5588
6298
  return CC2NUM((ca_u_cmplx64_t) (acc));
5589
6299
  }
5590
6300
 
6301
+ /* The walks can raise part way (a Ruby call per cell in the object
6302
+ lane, an allocation in any lane). Their states live in the caller's
6303
+ frame, and ca_iter_ensure finishes them however the body leaves. */
6304
+ typedef struct {
6305
+ ca_iter_state states[1];
6306
+ VALUE self;
6307
+ CArray *ca;
6308
+ int8_t *slab_axes;
6309
+ int8_t naxes;
6310
+ int keep_axis;
6311
+ ca_size_t min_count;
6312
+ } accumulate_ki_native_cmplx64_frame;
6313
+
5591
6314
  static VALUE
5592
- accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
6315
+ accumulate_ki_native_cmplx64_body (VALUE arg)
5593
6316
  {
5594
- if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
5595
- ca->elements > 0 &&
5596
- ! ca_too_few_present(ca->elements, min_count, 0) &&
5597
- ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
5598
- ca_fast_paths_enabled() ) {
5599
- accumulate_ki_stream_cmplx64_t __ctx;
5600
- __ctx.chunk = NULL;
5601
- __ctx.ca = ca;
5602
- __ctx.min_count = min_count;
5603
- return ca_lazy_arena_protect(accumulate_ki_stream_cmplx64, ca_lazy_arena_release_held,
5604
- (VALUE) &__ctx);
6317
+ accumulate_ki_native_cmplx64_frame *fr = (accumulate_ki_native_cmplx64_frame *) arg;
6318
+ VALUE self = fr->self;
6319
+ (void) self;
6320
+ CArray *ca = fr->ca;
6321
+ (void) ca;
6322
+ int8_t *slab_axes = fr->slab_axes;
6323
+ (void) slab_axes;
6324
+ int8_t naxes = fr->naxes;
6325
+ (void) naxes;
6326
+ int keep_axis = fr->keep_axis;
6327
+ (void) keep_axis;
6328
+ ca_size_t min_count = fr->min_count;
6329
+ (void) min_count;
6330
+ #define st (fr->states[0])
6331
+ if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
6332
+ ca->elements > 0 &&
6333
+ ! ca_too_few_present(ca->elements, min_count, 0) &&
6334
+ ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
6335
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
6336
+ accumulate_ki_stream_cmplx64_t __ctx;
6337
+ __ctx.chunk = NULL;
6338
+ __ctx.ca = ca;
6339
+ __ctx.min_count = min_count;
6340
+ return ca_lazy_arena_protect(accumulate_ki_stream_cmplx64, ca_lazy_arena_release_held,
6341
+ (VALUE) &__ctx);
5605
6342
  }
5606
6343
  if ( naxes != 1 || ! ca_too_few_present(ca->dim[slab_axes[0]], min_count, 0) ) {
5607
6344
  /* L.1 / L.7: loop-interchange fast path with inner-tiling
@@ -5616,7 +6353,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5616
6353
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
5617
6354
  && ! ca_has_mask(ca)
5618
6355
  && ca_attach_is_alias(ca)
5619
- && ca_fast_paths_enabled() ) {
6356
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
5620
6357
  int8_t __li_ax = slab_axes[0];
5621
6358
  ca_size_t __li_M = ca->dim[__li_ax];
5622
6359
  ca_size_t __li_INNER = 1;
@@ -5689,7 +6426,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5689
6426
  && ! ca_has_mask(ca)
5690
6427
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
5691
6428
  && ((CAStack *) ca)->k_axis == 0
5692
- && ca_fast_paths_enabled() ) {
6429
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
5693
6430
  CAStack *__li_st = (CAStack *) ca;
5694
6431
  int8_t __li_ax = slab_axes[0];
5695
6432
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -5717,7 +6454,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5717
6454
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
5718
6455
  const ca_size_t __li_TILE = 512;
5719
6456
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
5720
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
6457
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
5721
6458
  const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
5722
6459
  ca_u_cmplx64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
5723
6460
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -5777,7 +6514,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5777
6514
  && ((CAStack *) ca)->k_axis > 0
5778
6515
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
5779
6516
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
5780
- && ca_fast_paths_enabled() ) {
6517
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
5781
6518
  CAStack *__li_st = (CAStack *) ca;
5782
6519
  int8_t __li_ax = slab_axes[0];
5783
6520
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -5834,7 +6571,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5834
6571
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
5835
6572
  const ca_size_t __li_TILE = 512;
5836
6573
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
5837
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
6574
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
5838
6575
  const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
5839
6576
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
5840
6577
 
@@ -5922,7 +6659,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5922
6659
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
5923
6660
  && ((CAStack *) ca)->k_axis > 0
5924
6661
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
5925
- && ca_fast_paths_enabled() ) {
6662
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
5926
6663
  CAStack *__li_st = (CAStack *) ca;
5927
6664
  int8_t __li_ax = slab_axes[0];
5928
6665
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -5994,7 +6731,7 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
5994
6731
 
5995
6732
  const ca_size_t __li_TILE = 512;
5996
6733
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
5997
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
6734
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
5998
6735
  const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) __li_st->parents[__kk]->ptr;
5999
6736
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
6000
6737
 
@@ -6043,12 +6780,41 @@ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6043
6780
  return vout;
6044
6781
  }
6045
6782
  }
6783
+ if ( naxes == 1
6784
+ && ca->ndim >= 2
6785
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
6786
+ && ca->dim[ca->ndim - 1] >= 1
6787
+ && ca->dim[ca->ndim - 1] <= 32
6788
+ && ! ca_has_mask(ca)
6789
+ && ca_attach_is_alias(ca)
6790
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
6791
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
6792
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
6793
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
6794
+ CArray *co;
6795
+ GetCArray(vout, co);
6796
+ ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
6797
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
6798
+ const ca_u_cmplx64_t *p_root = (const ca_u_cmplx64_t *) ca->ptr;
6799
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
6800
+ const ca_u_cmplx64_t *__sf_row = p_root + __sf_o * __sf_M;
6801
+ ca_u_cmplx64_t acc = (ca_u_cmplx64_t) (0);
6802
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
6803
+ ca_u_cmplx64_t v = __sf_row[__j];
6804
+ acc += v;
6805
+ (void) v;
6806
+ }
6807
+
6808
+ op[__sf_o] = (ca_u_cmplx64_t) (acc);
6809
+ }
6810
+ ca_detach(ca);
6811
+ return vout;
6812
+ }
6046
6813
  }
6047
6814
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX64, keep_axis);
6048
6815
  CArray *co;
6049
6816
  GetCArray(vout, co);
6050
6817
  ca_u_cmplx64_t *op = (ca_u_cmplx64_t *) co->ptr;
6051
- ca_iter_state st;
6052
6818
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
6053
6819
  slab_axes, naxes, 0);
6054
6820
  if ( rc != CA_ITER_OK ) {
@@ -6082,7 +6848,22 @@ ca_size_t out_i = 0;
6082
6848
  return CC2NUM(op[0]);
6083
6849
  }
6084
6850
  return vout;
6085
- }
6851
+ #undef st
6852
+ }
6853
+
6854
+ static VALUE
6855
+ accumulate_ki_native_cmplx64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
6856
+ {
6857
+ accumulate_ki_native_cmplx64_frame fr;
6858
+ fr.self = self;
6859
+ fr.ca = ca;
6860
+ fr.slab_axes = slab_axes;
6861
+ fr.naxes = naxes;
6862
+ fr.keep_axis = keep_axis;
6863
+ fr.min_count = min_count;
6864
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_cmplx64_body, (VALUE) &fr);
6865
+ }
6866
+
6086
6867
 
6087
6868
  typedef struct {
6088
6869
  void *chunk;
@@ -6137,14 +6918,41 @@ accumulate_ki_stream_cmplx128 (VALUE arg)
6137
6918
  return CC2NUM((ca_u_cmplx128_t) (acc));
6138
6919
  }
6139
6920
 
6921
+ /* The walks can raise part way (a Ruby call per cell in the object
6922
+ lane, an allocation in any lane). Their states live in the caller's
6923
+ frame, and ca_iter_ensure finishes them however the body leaves. */
6924
+ typedef struct {
6925
+ ca_iter_state states[1];
6926
+ VALUE self;
6927
+ CArray *ca;
6928
+ int8_t *slab_axes;
6929
+ int8_t naxes;
6930
+ int keep_axis;
6931
+ ca_size_t min_count;
6932
+ } accumulate_ki_native_cmplx128_frame;
6933
+
6140
6934
  static VALUE
6141
- accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
6935
+ accumulate_ki_native_cmplx128_body (VALUE arg)
6142
6936
  {
6937
+ accumulate_ki_native_cmplx128_frame *fr = (accumulate_ki_native_cmplx128_frame *) arg;
6938
+ VALUE self = fr->self;
6939
+ (void) self;
6940
+ CArray *ca = fr->ca;
6941
+ (void) ca;
6942
+ int8_t *slab_axes = fr->slab_axes;
6943
+ (void) slab_axes;
6944
+ int8_t naxes = fr->naxes;
6945
+ (void) naxes;
6946
+ int keep_axis = fr->keep_axis;
6947
+ (void) keep_axis;
6948
+ ca_size_t min_count = fr->min_count;
6949
+ (void) min_count;
6950
+ #define st (fr->states[0])
6143
6951
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
6144
6952
  ca->elements > 0 &&
6145
6953
  ! ca_too_few_present(ca->elements, min_count, 0) &&
6146
6954
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
6147
- ca_fast_paths_enabled() ) {
6955
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
6148
6956
  accumulate_ki_stream_cmplx128_t __ctx;
6149
6957
  __ctx.chunk = NULL;
6150
6958
  __ctx.ca = ca;
@@ -6165,7 +6973,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6165
6973
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
6166
6974
  && ! ca_has_mask(ca)
6167
6975
  && ca_attach_is_alias(ca)
6168
- && ca_fast_paths_enabled() ) {
6976
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
6169
6977
  int8_t __li_ax = slab_axes[0];
6170
6978
  ca_size_t __li_M = ca->dim[__li_ax];
6171
6979
  ca_size_t __li_INNER = 1;
@@ -6238,7 +7046,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6238
7046
  && ! ca_has_mask(ca)
6239
7047
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
6240
7048
  && ((CAStack *) ca)->k_axis == 0
6241
- && ca_fast_paths_enabled() ) {
7049
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
6242
7050
  CAStack *__li_st = (CAStack *) ca;
6243
7051
  int8_t __li_ax = slab_axes[0];
6244
7052
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -6266,7 +7074,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6266
7074
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
6267
7075
  const ca_size_t __li_TILE = 512;
6268
7076
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
6269
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
7077
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
6270
7078
  const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
6271
7079
  ca_u_cmplx128_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
6272
7080
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -6326,7 +7134,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6326
7134
  && ((CAStack *) ca)->k_axis > 0
6327
7135
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
6328
7136
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
6329
- && ca_fast_paths_enabled() ) {
7137
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
6330
7138
  CAStack *__li_st = (CAStack *) ca;
6331
7139
  int8_t __li_ax = slab_axes[0];
6332
7140
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -6383,7 +7191,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6383
7191
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
6384
7192
  const ca_size_t __li_TILE = 512;
6385
7193
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
6386
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
7194
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
6387
7195
  const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
6388
7196
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
6389
7197
 
@@ -6471,7 +7279,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6471
7279
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
6472
7280
  && ((CAStack *) ca)->k_axis > 0
6473
7281
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
6474
- && ca_fast_paths_enabled() ) {
7282
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
6475
7283
  CAStack *__li_st = (CAStack *) ca;
6476
7284
  int8_t __li_ax = slab_axes[0];
6477
7285
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -6543,7 +7351,7 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6543
7351
 
6544
7352
  const ca_size_t __li_TILE = 512;
6545
7353
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
6546
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
7354
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
6547
7355
  const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) __li_st->parents[__kk]->ptr;
6548
7356
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
6549
7357
 
@@ -6592,12 +7400,41 @@ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
6592
7400
  return vout;
6593
7401
  }
6594
7402
  }
7403
+ if ( naxes == 1
7404
+ && ca->ndim >= 2
7405
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
7406
+ && ca->dim[ca->ndim - 1] >= 1
7407
+ && ca->dim[ca->ndim - 1] <= 32
7408
+ && ! ca_has_mask(ca)
7409
+ && ca_attach_is_alias(ca)
7410
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
7411
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
7412
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
7413
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
7414
+ CArray *co;
7415
+ GetCArray(vout, co);
7416
+ ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
7417
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
7418
+ const ca_u_cmplx128_t *p_root = (const ca_u_cmplx128_t *) ca->ptr;
7419
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
7420
+ const ca_u_cmplx128_t *__sf_row = p_root + __sf_o * __sf_M;
7421
+ ca_u_cmplx128_t acc = (ca_u_cmplx128_t) (0);
7422
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
7423
+ ca_u_cmplx128_t v = __sf_row[__j];
7424
+ acc += v;
7425
+ (void) v;
7426
+ }
7427
+
7428
+ op[__sf_o] = (ca_u_cmplx128_t) (acc);
7429
+ }
7430
+ ca_detach(ca);
7431
+ return vout;
7432
+ }
6595
7433
  }
6596
7434
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_CMPLX128, keep_axis);
6597
7435
  CArray *co;
6598
7436
  GetCArray(vout, co);
6599
7437
  ca_u_cmplx128_t *op = (ca_u_cmplx128_t *) co->ptr;
6600
- ca_iter_state st;
6601
7438
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
6602
7439
  slab_axes, naxes, 0);
6603
7440
  if ( rc != CA_ITER_OK ) {
@@ -6631,7 +7468,22 @@ ca_size_t out_i = 0;
6631
7468
  return CC2NUM(op[0]);
6632
7469
  }
6633
7470
  return vout;
6634
- }
7471
+ #undef st
7472
+ }
7473
+
7474
+ static VALUE
7475
+ accumulate_ki_native_cmplx128 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
7476
+ {
7477
+ accumulate_ki_native_cmplx128_frame fr;
7478
+ fr.self = self;
7479
+ fr.ca = ca;
7480
+ fr.slab_axes = slab_axes;
7481
+ fr.naxes = naxes;
7482
+ fr.keep_axis = keep_axis;
7483
+ fr.min_count = min_count;
7484
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_cmplx128_body, (VALUE) &fr);
7485
+ }
7486
+
6635
7487
 
6636
7488
  typedef struct {
6637
7489
  void *chunk;
@@ -6686,14 +7538,41 @@ accumulate_ki_stream_bool (VALUE arg)
6686
7538
  return INT2NUM((boolean8_t) (acc));
6687
7539
  }
6688
7540
 
7541
+ /* The walks can raise part way (a Ruby call per cell in the object
7542
+ lane, an allocation in any lane). Their states live in the caller's
7543
+ frame, and ca_iter_ensure finishes them however the body leaves. */
7544
+ typedef struct {
7545
+ ca_iter_state states[1];
7546
+ VALUE self;
7547
+ CArray *ca;
7548
+ int8_t *slab_axes;
7549
+ int8_t naxes;
7550
+ int keep_axis;
7551
+ ca_size_t min_count;
7552
+ } accumulate_ki_native_bool_frame;
7553
+
6689
7554
  static VALUE
6690
- accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
7555
+ accumulate_ki_native_bool_body (VALUE arg)
6691
7556
  {
7557
+ accumulate_ki_native_bool_frame *fr = (accumulate_ki_native_bool_frame *) arg;
7558
+ VALUE self = fr->self;
7559
+ (void) self;
7560
+ CArray *ca = fr->ca;
7561
+ (void) ca;
7562
+ int8_t *slab_axes = fr->slab_axes;
7563
+ (void) slab_axes;
7564
+ int8_t naxes = fr->naxes;
7565
+ (void) naxes;
7566
+ int keep_axis = fr->keep_axis;
7567
+ (void) keep_axis;
7568
+ ca_size_t min_count = fr->min_count;
7569
+ (void) min_count;
7570
+ #define st (fr->states[0])
6692
7571
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
6693
7572
  ca->elements > 0 &&
6694
7573
  ! ca_too_few_present(ca->elements, min_count, 0) &&
6695
7574
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
6696
- ca_fast_paths_enabled() ) {
7575
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
6697
7576
  accumulate_ki_stream_bool_t __ctx;
6698
7577
  __ctx.chunk = NULL;
6699
7578
  __ctx.ca = ca;
@@ -6714,7 +7593,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
6714
7593
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
6715
7594
  && ! ca_has_mask(ca)
6716
7595
  && ca_attach_is_alias(ca)
6717
- && ca_fast_paths_enabled() ) {
7596
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
6718
7597
  int8_t __li_ax = slab_axes[0];
6719
7598
  ca_size_t __li_M = ca->dim[__li_ax];
6720
7599
  ca_size_t __li_INNER = 1;
@@ -6787,7 +7666,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
6787
7666
  && ! ca_has_mask(ca)
6788
7667
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
6789
7668
  && ((CAStack *) ca)->k_axis == 0
6790
- && ca_fast_paths_enabled() ) {
7669
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
6791
7670
  CAStack *__li_st = (CAStack *) ca;
6792
7671
  int8_t __li_ax = slab_axes[0];
6793
7672
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -6815,7 +7694,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
6815
7694
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
6816
7695
  const ca_size_t __li_TILE = 512;
6817
7696
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
6818
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
7697
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
6819
7698
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
6820
7699
  boolean8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
6821
7700
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -6875,7 +7754,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
6875
7754
  && ((CAStack *) ca)->k_axis > 0
6876
7755
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
6877
7756
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
6878
- && ca_fast_paths_enabled() ) {
7757
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
6879
7758
  CAStack *__li_st = (CAStack *) ca;
6880
7759
  int8_t __li_ax = slab_axes[0];
6881
7760
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -6932,7 +7811,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
6932
7811
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
6933
7812
  const ca_size_t __li_TILE = 512;
6934
7813
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
6935
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
7814
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
6936
7815
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
6937
7816
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
6938
7817
 
@@ -7020,7 +7899,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7020
7899
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
7021
7900
  && ((CAStack *) ca)->k_axis > 0
7022
7901
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
7023
- && ca_fast_paths_enabled() ) {
7902
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
7024
7903
  CAStack *__li_st = (CAStack *) ca;
7025
7904
  int8_t __li_ax = slab_axes[0];
7026
7905
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -7092,7 +7971,7 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7092
7971
 
7093
7972
  const ca_size_t __li_TILE = 512;
7094
7973
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
7095
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
7974
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
7096
7975
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
7097
7976
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
7098
7977
 
@@ -7141,12 +8020,41 @@ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7141
8020
  return vout;
7142
8021
  }
7143
8022
  }
8023
+ if ( naxes == 1
8024
+ && ca->ndim >= 2
8025
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
8026
+ && ca->dim[ca->ndim - 1] >= 1
8027
+ && ca->dim[ca->ndim - 1] <= 32
8028
+ && ! ca_has_mask(ca)
8029
+ && ca_attach_is_alias(ca)
8030
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
8031
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
8032
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
8033
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
8034
+ CArray *co;
8035
+ GetCArray(vout, co);
8036
+ boolean8_t *op = (boolean8_t *) co->ptr;
8037
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
8038
+ const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
8039
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
8040
+ const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
8041
+ boolean8_t acc = (boolean8_t) (0);
8042
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
8043
+ boolean8_t v = __sf_row[__j];
8044
+ acc ^= v;
8045
+ (void) v;
8046
+ }
8047
+
8048
+ op[__sf_o] = (boolean8_t) (acc);
8049
+ }
8050
+ ca_detach(ca);
8051
+ return vout;
8052
+ }
7144
8053
  }
7145
8054
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
7146
8055
  CArray *co;
7147
8056
  GetCArray(vout, co);
7148
8057
  boolean8_t *op = (boolean8_t *) co->ptr;
7149
- ca_iter_state st;
7150
8058
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
7151
8059
  slab_axes, naxes, 0);
7152
8060
  if ( rc != CA_ITER_OK ) {
@@ -7178,7 +8086,22 @@ ca_size_t out_i = 0;
7178
8086
  return INT2NUM(op[0]);
7179
8087
  }
7180
8088
  return vout;
7181
- }
8089
+ #undef st
8090
+ }
8091
+
8092
+ static VALUE
8093
+ accumulate_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
8094
+ {
8095
+ accumulate_ki_native_bool_frame fr;
8096
+ fr.self = self;
8097
+ fr.ca = ca;
8098
+ fr.slab_axes = slab_axes;
8099
+ fr.naxes = naxes;
8100
+ fr.keep_axis = keep_axis;
8101
+ fr.min_count = min_count;
8102
+ return ca_iter_ensure(1, fr.states, accumulate_ki_native_bool_body, (VALUE) &fr);
8103
+ }
8104
+
7182
8105
 
7183
8106
  typedef struct {
7184
8107
  void *chunk;
@@ -7233,9 +8156,9 @@ accumulate_ki_stream_object (VALUE arg)
7233
8156
  return (VALUE)((VALUE) (acc));
7234
8157
  }
7235
8158
 
7236
- /* The object lane calls Ruby for every cell, so a walk can raise part
7237
- way. Its states live in the caller's frame, and ca_iter_ensure
7238
- finishes them however the body leaves. */
8159
+ /* The walks can raise part way (a Ruby call per cell in the object
8160
+ lane, an allocation in any lane). Their states live in the caller's
8161
+ frame, and ca_iter_ensure finishes them however the body leaves. */
7239
8162
  typedef struct {
7240
8163
  ca_iter_state states[1];
7241
8164
  VALUE self;
@@ -7267,7 +8190,7 @@ accumulate_ki_native_object_body (VALUE arg)
7267
8190
  ca->elements > 0 &&
7268
8191
  ! ca_too_few_present(ca->elements, min_count, 0) &&
7269
8192
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
7270
- ca_fast_paths_enabled() ) {
8193
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
7271
8194
  accumulate_ki_stream_object_t __ctx;
7272
8195
  __ctx.chunk = NULL;
7273
8196
  __ctx.ca = ca;
@@ -7323,7 +8246,7 @@ accumulate_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t n
7323
8246
  fr.naxes = naxes;
7324
8247
  fr.keep_axis = keep_axis;
7325
8248
  fr.min_count = min_count;
7326
- return ca_iter_ensure(1, fr.states, accumulate_ki_native_object_body, (VALUE) &fr);
8249
+ return ca_iter_ensure_computing(ca, 1, fr.states, accumulate_ki_native_object_body, (VALUE) &fr);
7327
8250
  }
7328
8251
 
7329
8252
 
@@ -7492,14 +8415,41 @@ count_true_ki_stream_bool (VALUE arg)
7492
8415
  return LL2NUM((ca_u_int64_t) (acc));
7493
8416
  }
7494
8417
 
8418
+ /* The walks can raise part way (a Ruby call per cell in the object
8419
+ lane, an allocation in any lane). Their states live in the caller's
8420
+ frame, and ca_iter_ensure finishes them however the body leaves. */
8421
+ typedef struct {
8422
+ ca_iter_state states[1];
8423
+ VALUE self;
8424
+ CArray *ca;
8425
+ int8_t *slab_axes;
8426
+ int8_t naxes;
8427
+ int keep_axis;
8428
+ ca_size_t min_count;
8429
+ } count_true_ki_native_bool_frame;
8430
+
7495
8431
  static VALUE
7496
- count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
8432
+ count_true_ki_native_bool_body (VALUE arg)
7497
8433
  {
8434
+ count_true_ki_native_bool_frame *fr = (count_true_ki_native_bool_frame *) arg;
8435
+ VALUE self = fr->self;
8436
+ (void) self;
8437
+ CArray *ca = fr->ca;
8438
+ (void) ca;
8439
+ int8_t *slab_axes = fr->slab_axes;
8440
+ (void) slab_axes;
8441
+ int8_t naxes = fr->naxes;
8442
+ (void) naxes;
8443
+ int keep_axis = fr->keep_axis;
8444
+ (void) keep_axis;
8445
+ ca_size_t min_count = fr->min_count;
8446
+ (void) min_count;
8447
+ #define st (fr->states[0])
7498
8448
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
7499
8449
  ca->elements > 0 &&
7500
8450
  ! ca_too_few_present(ca->elements, min_count, 0) &&
7501
8451
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
7502
- ca_fast_paths_enabled() ) {
8452
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
7503
8453
  count_true_ki_stream_bool_t __ctx;
7504
8454
  __ctx.chunk = NULL;
7505
8455
  __ctx.ca = ca;
@@ -7520,7 +8470,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7520
8470
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
7521
8471
  && ! ca_has_mask(ca)
7522
8472
  && ca_attach_is_alias(ca)
7523
- && ca_fast_paths_enabled() ) {
8473
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
7524
8474
  int8_t __li_ax = slab_axes[0];
7525
8475
  ca_size_t __li_M = ca->dim[__li_ax];
7526
8476
  ca_size_t __li_INNER = 1;
@@ -7593,7 +8543,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7593
8543
  && ! ca_has_mask(ca)
7594
8544
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
7595
8545
  && ((CAStack *) ca)->k_axis == 0
7596
- && ca_fast_paths_enabled() ) {
8546
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
7597
8547
  CAStack *__li_st = (CAStack *) ca;
7598
8548
  int8_t __li_ax = slab_axes[0];
7599
8549
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -7621,7 +8571,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7621
8571
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
7622
8572
  const ca_size_t __li_TILE = 512;
7623
8573
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
7624
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
8574
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
7625
8575
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
7626
8576
  ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
7627
8577
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -7681,7 +8631,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7681
8631
  && ((CAStack *) ca)->k_axis > 0
7682
8632
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
7683
8633
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
7684
- && ca_fast_paths_enabled() ) {
8634
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
7685
8635
  CAStack *__li_st = (CAStack *) ca;
7686
8636
  int8_t __li_ax = slab_axes[0];
7687
8637
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -7738,7 +8688,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7738
8688
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
7739
8689
  const ca_size_t __li_TILE = 512;
7740
8690
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
7741
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
8691
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
7742
8692
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
7743
8693
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
7744
8694
 
@@ -7826,7 +8776,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7826
8776
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
7827
8777
  && ((CAStack *) ca)->k_axis > 0
7828
8778
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
7829
- && ca_fast_paths_enabled() ) {
8779
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
7830
8780
  CAStack *__li_st = (CAStack *) ca;
7831
8781
  int8_t __li_ax = slab_axes[0];
7832
8782
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -7898,7 +8848,7 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7898
8848
 
7899
8849
  const ca_size_t __li_TILE = 512;
7900
8850
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
7901
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
8851
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
7902
8852
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
7903
8853
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
7904
8854
 
@@ -7947,12 +8897,41 @@ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t nax
7947
8897
  return vout;
7948
8898
  }
7949
8899
  }
8900
+ if ( naxes == 1
8901
+ && ca->ndim >= 2
8902
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
8903
+ && ca->dim[ca->ndim - 1] >= 1
8904
+ && ca->dim[ca->ndim - 1] <= 32
8905
+ && ! ca_has_mask(ca)
8906
+ && ca_attach_is_alias(ca)
8907
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
8908
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
8909
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
8910
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
8911
+ CArray *co;
8912
+ GetCArray(vout, co);
8913
+ ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
8914
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
8915
+ const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
8916
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
8917
+ const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
8918
+ ca_u_int64_t acc = (ca_u_int64_t) (0);
8919
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
8920
+ boolean8_t v = __sf_row[__j];
8921
+ acc += v;
8922
+ (void) v;
8923
+ }
8924
+
8925
+ op[__sf_o] = (ca_u_int64_t) (acc);
8926
+ }
8927
+ ca_detach(ca);
8928
+ return vout;
8929
+ }
7950
8930
  }
7951
8931
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
7952
8932
  CArray *co;
7953
8933
  GetCArray(vout, co);
7954
8934
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
7955
- ca_iter_state st;
7956
8935
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
7957
8936
  slab_axes, naxes, 0);
7958
8937
  if ( rc != CA_ITER_OK ) {
@@ -7986,7 +8965,22 @@ ca_size_t out_i = 0;
7986
8965
  return LL2NUM(op[0]);
7987
8966
  }
7988
8967
  return vout;
7989
- }
8968
+ #undef st
8969
+ }
8970
+
8971
+ static VALUE
8972
+ count_true_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
8973
+ {
8974
+ count_true_ki_native_bool_frame fr;
8975
+ fr.self = self;
8976
+ fr.ca = ca;
8977
+ fr.slab_axes = slab_axes;
8978
+ fr.naxes = naxes;
8979
+ fr.keep_axis = keep_axis;
8980
+ fr.min_count = min_count;
8981
+ return ca_iter_ensure(1, fr.states, count_true_ki_native_bool_body, (VALUE) &fr);
8982
+ }
8983
+
7990
8984
 
7991
8985
  VALUE
7992
8986
  rb_ca_count_true_ki (int argc, VALUE *argv, VALUE self)
@@ -8088,14 +9082,41 @@ count_false_ki_stream_bool (VALUE arg)
8088
9082
  return LL2NUM((ca_u_int64_t) (acc));
8089
9083
  }
8090
9084
 
9085
+ /* The walks can raise part way (a Ruby call per cell in the object
9086
+ lane, an allocation in any lane). Their states live in the caller's
9087
+ frame, and ca_iter_ensure finishes them however the body leaves. */
9088
+ typedef struct {
9089
+ ca_iter_state states[1];
9090
+ VALUE self;
9091
+ CArray *ca;
9092
+ int8_t *slab_axes;
9093
+ int8_t naxes;
9094
+ int keep_axis;
9095
+ ca_size_t min_count;
9096
+ } count_false_ki_native_bool_frame;
9097
+
8091
9098
  static VALUE
8092
- count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
9099
+ count_false_ki_native_bool_body (VALUE arg)
8093
9100
  {
9101
+ count_false_ki_native_bool_frame *fr = (count_false_ki_native_bool_frame *) arg;
9102
+ VALUE self = fr->self;
9103
+ (void) self;
9104
+ CArray *ca = fr->ca;
9105
+ (void) ca;
9106
+ int8_t *slab_axes = fr->slab_axes;
9107
+ (void) slab_axes;
9108
+ int8_t naxes = fr->naxes;
9109
+ (void) naxes;
9110
+ int keep_axis = fr->keep_axis;
9111
+ (void) keep_axis;
9112
+ ca_size_t min_count = fr->min_count;
9113
+ (void) min_count;
9114
+ #define st (fr->states[0])
8094
9115
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
8095
9116
  ca->elements > 0 &&
8096
9117
  ! ca_too_few_present(ca->elements, min_count, 0) &&
8097
9118
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
8098
- ca_fast_paths_enabled() ) {
9119
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
8099
9120
  count_false_ki_stream_bool_t __ctx;
8100
9121
  __ctx.chunk = NULL;
8101
9122
  __ctx.ca = ca;
@@ -8116,7 +9137,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8116
9137
  && slab_axes[0] != (int8_t)(ca->ndim - 1)
8117
9138
  && ! ca_has_mask(ca)
8118
9139
  && ca_attach_is_alias(ca)
8119
- && ca_fast_paths_enabled() ) {
9140
+ && ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
8120
9141
  int8_t __li_ax = slab_axes[0];
8121
9142
  ca_size_t __li_M = ca->dim[__li_ax];
8122
9143
  ca_size_t __li_INNER = 1;
@@ -8189,7 +9210,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8189
9210
  && ! ca_has_mask(ca)
8190
9211
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
8191
9212
  && ((CAStack *) ca)->k_axis == 0
8192
- && ca_fast_paths_enabled() ) {
9213
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
8193
9214
  CAStack *__li_st = (CAStack *) ca;
8194
9215
  int8_t __li_ax = slab_axes[0];
8195
9216
  ca_size_t __li_M = ca->dim[__li_ax];
@@ -8217,7 +9238,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8217
9238
  ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
8218
9239
  const ca_size_t __li_TILE = 512;
8219
9240
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
8220
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
9241
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
8221
9242
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
8222
9243
  ca_u_int64_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
8223
9244
  for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
@@ -8277,7 +9298,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8277
9298
  && ((CAStack *) ca)->k_axis > 0
8278
9299
  && ( slab_axes[0] > ((CAStack *) ca)->k_axis
8279
9300
  || slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
8280
- && ca_fast_paths_enabled() ) {
9301
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
8281
9302
  CAStack *__li_st = (CAStack *) ca;
8282
9303
  int8_t __li_ax = slab_axes[0];
8283
9304
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -8334,7 +9355,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8334
9355
  ca_size_t __li_plane_elems = __li_M * __li_INNER;
8335
9356
  const ca_size_t __li_TILE = 512;
8336
9357
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
8337
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
9358
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
8338
9359
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
8339
9360
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
8340
9361
 
@@ -8422,7 +9443,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8422
9443
  && ca_func[ca->obj_type].attach == ca_stack_func.attach
8423
9444
  && ((CAStack *) ca)->k_axis > 0
8424
9445
  && slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
8425
- && ca_fast_paths_enabled() ) {
9446
+ && ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
8426
9447
  CAStack *__li_st = (CAStack *) ca;
8427
9448
  int8_t __li_ax = slab_axes[0];
8428
9449
  int8_t __li_kax = (int8_t) __li_st->k_axis;
@@ -8494,7 +9515,7 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8494
9515
 
8495
9516
  const ca_size_t __li_TILE = 512;
8496
9517
  for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
8497
- ca_attach(__li_st->parents[__kk]); /* window: numeric lanes only, nothing raises inside */
9518
+ ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
8498
9519
  const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
8499
9520
  ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
8500
9521
 
@@ -8543,12 +9564,41 @@ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t na
8543
9564
  return vout;
8544
9565
  }
8545
9566
  }
9567
+ if ( naxes == 1
9568
+ && ca->ndim >= 2
9569
+ && slab_axes[0] == (int8_t)(ca->ndim - 1)
9570
+ && ca->dim[ca->ndim - 1] >= 1
9571
+ && ca->dim[ca->ndim - 1] <= 32
9572
+ && ! ca_has_mask(ca)
9573
+ && ca_attach_is_alias(ca)
9574
+ && ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
9575
+ ca_size_t __sf_M = ca->dim[ca->ndim - 1];
9576
+ ca_size_t __sf_OUTER = ca->elements / __sf_M;
9577
+ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9578
+ CArray *co;
9579
+ GetCArray(vout, co);
9580
+ ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9581
+ ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
9582
+ const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
9583
+ for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
9584
+ const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
9585
+ ca_u_int64_t acc = (ca_u_int64_t) (0);
9586
+ for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
9587
+ boolean8_t v = __sf_row[__j];
9588
+ acc += !v;
9589
+ (void) v;
9590
+ }
9591
+
9592
+ op[__sf_o] = (ca_u_int64_t) (acc);
9593
+ }
9594
+ ca_detach(ca);
9595
+ return vout;
9596
+ }
8546
9597
  }
8547
9598
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
8548
9599
  CArray *co;
8549
9600
  GetCArray(vout, co);
8550
9601
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
8551
- ca_iter_state st;
8552
9602
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
8553
9603
  slab_axes, naxes, 0);
8554
9604
  if ( rc != CA_ITER_OK ) {
@@ -8580,7 +9630,22 @@ ca_size_t out_i = 0;
8580
9630
  return LL2NUM(op[0]);
8581
9631
  }
8582
9632
  return vout;
8583
- }
9633
+ #undef st
9634
+ }
9635
+
9636
+ static VALUE
9637
+ count_false_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_size_t min_count)
9638
+ {
9639
+ count_false_ki_native_bool_frame fr;
9640
+ fr.self = self;
9641
+ fr.ca = ca;
9642
+ fr.slab_axes = slab_axes;
9643
+ fr.naxes = naxes;
9644
+ fr.keep_axis = keep_axis;
9645
+ fr.min_count = min_count;
9646
+ return ca_iter_ensure(1, fr.states, count_false_ki_native_bool_body, (VALUE) &fr);
9647
+ }
9648
+
8584
9649
 
8585
9650
  VALUE
8586
9651
  rb_ca_count_false_ki (int argc, VALUE *argv, VALUE self)
@@ -8685,27 +9750,56 @@ count_equal_ki_stream_i8 (VALUE arg)
8685
9750
  return LL2NUM((ca_u_int64_t) (acc));
8686
9751
  }
8687
9752
 
9753
+ /* The walks can raise part way (a Ruby call per cell in the object
9754
+ lane, an allocation in any lane). Their states live in the caller's
9755
+ frame, and ca_iter_ensure finishes them however the body leaves. */
9756
+ typedef struct {
9757
+ ca_iter_state states[1];
9758
+ VALUE self;
9759
+ CArray *ca;
9760
+ int8_t *slab_axes;
9761
+ int8_t naxes;
9762
+ int keep_axis;
9763
+ int8_t value_arg;
9764
+ ca_size_t min_count;
9765
+ } count_equal_ki_native_i8_frame;
9766
+
8688
9767
  static VALUE
8689
- count_equal_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, int8_t value_arg, ca_size_t min_count)
9768
+ count_equal_ki_native_i8_body (VALUE arg)
8690
9769
  {
8691
- if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
8692
- ca->elements > 0 &&
8693
- ! ca_too_few_present(ca->elements, min_count, 0) &&
8694
- ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
8695
- ca_fast_paths_enabled() ) {
8696
- count_equal_ki_stream_i8_t __ctx;
8697
- __ctx.chunk = NULL;
8698
- __ctx.ca = ca;
8699
- __ctx.value_arg = value_arg;
8700
- __ctx.min_count = min_count;
8701
- return ca_lazy_arena_protect(count_equal_ki_stream_i8, ca_lazy_arena_release_held,
9770
+ count_equal_ki_native_i8_frame *fr = (count_equal_ki_native_i8_frame *) arg;
9771
+ VALUE self = fr->self;
9772
+ (void) self;
9773
+ CArray *ca = fr->ca;
9774
+ (void) ca;
9775
+ int8_t *slab_axes = fr->slab_axes;
9776
+ (void) slab_axes;
9777
+ int8_t naxes = fr->naxes;
9778
+ (void) naxes;
9779
+ int keep_axis = fr->keep_axis;
9780
+ (void) keep_axis;
9781
+ int8_t value_arg = fr->value_arg;
9782
+ (void) value_arg;
9783
+ ca_size_t min_count = fr->min_count;
9784
+ (void) min_count;
9785
+ #define st (fr->states[0])
9786
+ if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9787
+ ca->elements > 0 &&
9788
+ ! ca_too_few_present(ca->elements, min_count, 0) &&
9789
+ ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9790
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9791
+ count_equal_ki_stream_i8_t __ctx;
9792
+ __ctx.chunk = NULL;
9793
+ __ctx.ca = ca;
9794
+ __ctx.value_arg = value_arg;
9795
+ __ctx.min_count = min_count;
9796
+ return ca_lazy_arena_protect(count_equal_ki_stream_i8, ca_lazy_arena_release_held,
8702
9797
  (VALUE) &__ctx);
8703
9798
  }
8704
9799
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
8705
9800
  CArray *co;
8706
9801
  GetCArray(vout, co);
8707
9802
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
8708
- ca_iter_state st;
8709
9803
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
8710
9804
  slab_axes, naxes, 0);
8711
9805
  if ( rc != CA_ITER_OK ) {
@@ -8737,7 +9831,23 @@ ca_size_t out_i = 0;
8737
9831
  return LL2NUM(op[0]);
8738
9832
  }
8739
9833
  return vout;
8740
- }
9834
+ #undef st
9835
+ }
9836
+
9837
+ static VALUE
9838
+ count_equal_ki_native_i8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, int8_t value_arg, ca_size_t min_count)
9839
+ {
9840
+ count_equal_ki_native_i8_frame fr;
9841
+ fr.self = self;
9842
+ fr.ca = ca;
9843
+ fr.slab_axes = slab_axes;
9844
+ fr.naxes = naxes;
9845
+ fr.keep_axis = keep_axis;
9846
+ fr.value_arg = value_arg;
9847
+ fr.min_count = min_count;
9848
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_i8_body, (VALUE) &fr);
9849
+ }
9850
+
8741
9851
 
8742
9852
  typedef struct {
8743
9853
  void *chunk;
@@ -8795,14 +9905,44 @@ count_equal_ki_stream_u8 (VALUE arg)
8795
9905
  return LL2NUM((ca_u_int64_t) (acc));
8796
9906
  }
8797
9907
 
9908
+ /* The walks can raise part way (a Ruby call per cell in the object
9909
+ lane, an allocation in any lane). Their states live in the caller's
9910
+ frame, and ca_iter_ensure finishes them however the body leaves. */
9911
+ typedef struct {
9912
+ ca_iter_state states[1];
9913
+ VALUE self;
9914
+ CArray *ca;
9915
+ int8_t *slab_axes;
9916
+ int8_t naxes;
9917
+ int keep_axis;
9918
+ uint8_t value_arg;
9919
+ ca_size_t min_count;
9920
+ } count_equal_ki_native_u8_frame;
9921
+
8798
9922
  static VALUE
8799
- count_equal_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, uint8_t value_arg, ca_size_t min_count)
9923
+ count_equal_ki_native_u8_body (VALUE arg)
8800
9924
  {
9925
+ count_equal_ki_native_u8_frame *fr = (count_equal_ki_native_u8_frame *) arg;
9926
+ VALUE self = fr->self;
9927
+ (void) self;
9928
+ CArray *ca = fr->ca;
9929
+ (void) ca;
9930
+ int8_t *slab_axes = fr->slab_axes;
9931
+ (void) slab_axes;
9932
+ int8_t naxes = fr->naxes;
9933
+ (void) naxes;
9934
+ int keep_axis = fr->keep_axis;
9935
+ (void) keep_axis;
9936
+ uint8_t value_arg = fr->value_arg;
9937
+ (void) value_arg;
9938
+ ca_size_t min_count = fr->min_count;
9939
+ (void) min_count;
9940
+ #define st (fr->states[0])
8801
9941
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
8802
9942
  ca->elements > 0 &&
8803
9943
  ! ca_too_few_present(ca->elements, min_count, 0) &&
8804
9944
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
8805
- ca_fast_paths_enabled() ) {
9945
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
8806
9946
  count_equal_ki_stream_u8_t __ctx;
8807
9947
  __ctx.chunk = NULL;
8808
9948
  __ctx.ca = ca;
@@ -8815,7 +9955,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
8815
9955
  CArray *co;
8816
9956
  GetCArray(vout, co);
8817
9957
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
8818
- ca_iter_state st;
8819
9958
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
8820
9959
  slab_axes, naxes, 0);
8821
9960
  if ( rc != CA_ITER_OK ) {
@@ -8847,7 +9986,23 @@ ca_size_t out_i = 0;
8847
9986
  return LL2NUM(op[0]);
8848
9987
  }
8849
9988
  return vout;
8850
- }
9989
+ #undef st
9990
+ }
9991
+
9992
+ static VALUE
9993
+ count_equal_ki_native_u8 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, uint8_t value_arg, ca_size_t min_count)
9994
+ {
9995
+ count_equal_ki_native_u8_frame fr;
9996
+ fr.self = self;
9997
+ fr.ca = ca;
9998
+ fr.slab_axes = slab_axes;
9999
+ fr.naxes = naxes;
10000
+ fr.keep_axis = keep_axis;
10001
+ fr.value_arg = value_arg;
10002
+ fr.min_count = min_count;
10003
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_u8_body, (VALUE) &fr);
10004
+ }
10005
+
8851
10006
 
8852
10007
  typedef struct {
8853
10008
  void *chunk;
@@ -8905,14 +10060,44 @@ count_equal_ki_stream_i16 (VALUE arg)
8905
10060
  return LL2NUM((ca_u_int64_t) (acc));
8906
10061
  }
8907
10062
 
10063
+ /* The walks can raise part way (a Ruby call per cell in the object
10064
+ lane, an allocation in any lane). Their states live in the caller's
10065
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10066
+ typedef struct {
10067
+ ca_iter_state states[1];
10068
+ VALUE self;
10069
+ CArray *ca;
10070
+ int8_t *slab_axes;
10071
+ int8_t naxes;
10072
+ int keep_axis;
10073
+ ca_u_int16_t value_arg;
10074
+ ca_size_t min_count;
10075
+ } count_equal_ki_native_i16_frame;
10076
+
8908
10077
  static VALUE
8909
- count_equal_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int16_t value_arg, ca_size_t min_count)
10078
+ count_equal_ki_native_i16_body (VALUE arg)
8910
10079
  {
10080
+ count_equal_ki_native_i16_frame *fr = (count_equal_ki_native_i16_frame *) arg;
10081
+ VALUE self = fr->self;
10082
+ (void) self;
10083
+ CArray *ca = fr->ca;
10084
+ (void) ca;
10085
+ int8_t *slab_axes = fr->slab_axes;
10086
+ (void) slab_axes;
10087
+ int8_t naxes = fr->naxes;
10088
+ (void) naxes;
10089
+ int keep_axis = fr->keep_axis;
10090
+ (void) keep_axis;
10091
+ ca_u_int16_t value_arg = fr->value_arg;
10092
+ (void) value_arg;
10093
+ ca_size_t min_count = fr->min_count;
10094
+ (void) min_count;
10095
+ #define st (fr->states[0])
8911
10096
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
8912
10097
  ca->elements > 0 &&
8913
10098
  ! ca_too_few_present(ca->elements, min_count, 0) &&
8914
10099
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
8915
- ca_fast_paths_enabled() ) {
10100
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
8916
10101
  count_equal_ki_stream_i16_t __ctx;
8917
10102
  __ctx.chunk = NULL;
8918
10103
  __ctx.ca = ca;
@@ -8925,7 +10110,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
8925
10110
  CArray *co;
8926
10111
  GetCArray(vout, co);
8927
10112
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
8928
- ca_iter_state st;
8929
10113
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
8930
10114
  slab_axes, naxes, 0);
8931
10115
  if ( rc != CA_ITER_OK ) {
@@ -8957,7 +10141,23 @@ ca_size_t out_i = 0;
8957
10141
  return LL2NUM(op[0]);
8958
10142
  }
8959
10143
  return vout;
8960
- }
10144
+ #undef st
10145
+ }
10146
+
10147
+ static VALUE
10148
+ count_equal_ki_native_i16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int16_t value_arg, ca_size_t min_count)
10149
+ {
10150
+ count_equal_ki_native_i16_frame fr;
10151
+ fr.self = self;
10152
+ fr.ca = ca;
10153
+ fr.slab_axes = slab_axes;
10154
+ fr.naxes = naxes;
10155
+ fr.keep_axis = keep_axis;
10156
+ fr.value_arg = value_arg;
10157
+ fr.min_count = min_count;
10158
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_i16_body, (VALUE) &fr);
10159
+ }
10160
+
8961
10161
 
8962
10162
  typedef struct {
8963
10163
  void *chunk;
@@ -9015,14 +10215,44 @@ count_equal_ki_stream_u16 (VALUE arg)
9015
10215
  return LL2NUM((ca_u_int64_t) (acc));
9016
10216
  }
9017
10217
 
10218
+ /* The walks can raise part way (a Ruby call per cell in the object
10219
+ lane, an allocation in any lane). Their states live in the caller's
10220
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10221
+ typedef struct {
10222
+ ca_iter_state states[1];
10223
+ VALUE self;
10224
+ CArray *ca;
10225
+ int8_t *slab_axes;
10226
+ int8_t naxes;
10227
+ int keep_axis;
10228
+ ca_u_uint16_t value_arg;
10229
+ ca_size_t min_count;
10230
+ } count_equal_ki_native_u16_frame;
10231
+
9018
10232
  static VALUE
9019
- count_equal_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint16_t value_arg, ca_size_t min_count)
10233
+ count_equal_ki_native_u16_body (VALUE arg)
9020
10234
  {
10235
+ count_equal_ki_native_u16_frame *fr = (count_equal_ki_native_u16_frame *) arg;
10236
+ VALUE self = fr->self;
10237
+ (void) self;
10238
+ CArray *ca = fr->ca;
10239
+ (void) ca;
10240
+ int8_t *slab_axes = fr->slab_axes;
10241
+ (void) slab_axes;
10242
+ int8_t naxes = fr->naxes;
10243
+ (void) naxes;
10244
+ int keep_axis = fr->keep_axis;
10245
+ (void) keep_axis;
10246
+ ca_u_uint16_t value_arg = fr->value_arg;
10247
+ (void) value_arg;
10248
+ ca_size_t min_count = fr->min_count;
10249
+ (void) min_count;
10250
+ #define st (fr->states[0])
9021
10251
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9022
10252
  ca->elements > 0 &&
9023
10253
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9024
10254
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9025
- ca_fast_paths_enabled() ) {
10255
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9026
10256
  count_equal_ki_stream_u16_t __ctx;
9027
10257
  __ctx.chunk = NULL;
9028
10258
  __ctx.ca = ca;
@@ -9035,7 +10265,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9035
10265
  CArray *co;
9036
10266
  GetCArray(vout, co);
9037
10267
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9038
- ca_iter_state st;
9039
10268
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9040
10269
  slab_axes, naxes, 0);
9041
10270
  if ( rc != CA_ITER_OK ) {
@@ -9067,7 +10296,23 @@ ca_size_t out_i = 0;
9067
10296
  return LL2NUM(op[0]);
9068
10297
  }
9069
10298
  return vout;
9070
- }
10299
+ #undef st
10300
+ }
10301
+
10302
+ static VALUE
10303
+ count_equal_ki_native_u16 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint16_t value_arg, ca_size_t min_count)
10304
+ {
10305
+ count_equal_ki_native_u16_frame fr;
10306
+ fr.self = self;
10307
+ fr.ca = ca;
10308
+ fr.slab_axes = slab_axes;
10309
+ fr.naxes = naxes;
10310
+ fr.keep_axis = keep_axis;
10311
+ fr.value_arg = value_arg;
10312
+ fr.min_count = min_count;
10313
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_u16_body, (VALUE) &fr);
10314
+ }
10315
+
9071
10316
 
9072
10317
  typedef struct {
9073
10318
  void *chunk;
@@ -9125,14 +10370,44 @@ count_equal_ki_stream_i32 (VALUE arg)
9125
10370
  return LL2NUM((ca_u_int64_t) (acc));
9126
10371
  }
9127
10372
 
10373
+ /* The walks can raise part way (a Ruby call per cell in the object
10374
+ lane, an allocation in any lane). Their states live in the caller's
10375
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10376
+ typedef struct {
10377
+ ca_iter_state states[1];
10378
+ VALUE self;
10379
+ CArray *ca;
10380
+ int8_t *slab_axes;
10381
+ int8_t naxes;
10382
+ int keep_axis;
10383
+ ca_u_int32_t value_arg;
10384
+ ca_size_t min_count;
10385
+ } count_equal_ki_native_i32_frame;
10386
+
9128
10387
  static VALUE
9129
- count_equal_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int32_t value_arg, ca_size_t min_count)
10388
+ count_equal_ki_native_i32_body (VALUE arg)
9130
10389
  {
10390
+ count_equal_ki_native_i32_frame *fr = (count_equal_ki_native_i32_frame *) arg;
10391
+ VALUE self = fr->self;
10392
+ (void) self;
10393
+ CArray *ca = fr->ca;
10394
+ (void) ca;
10395
+ int8_t *slab_axes = fr->slab_axes;
10396
+ (void) slab_axes;
10397
+ int8_t naxes = fr->naxes;
10398
+ (void) naxes;
10399
+ int keep_axis = fr->keep_axis;
10400
+ (void) keep_axis;
10401
+ ca_u_int32_t value_arg = fr->value_arg;
10402
+ (void) value_arg;
10403
+ ca_size_t min_count = fr->min_count;
10404
+ (void) min_count;
10405
+ #define st (fr->states[0])
9131
10406
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9132
10407
  ca->elements > 0 &&
9133
10408
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9134
10409
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9135
- ca_fast_paths_enabled() ) {
10410
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9136
10411
  count_equal_ki_stream_i32_t __ctx;
9137
10412
  __ctx.chunk = NULL;
9138
10413
  __ctx.ca = ca;
@@ -9145,7 +10420,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9145
10420
  CArray *co;
9146
10421
  GetCArray(vout, co);
9147
10422
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9148
- ca_iter_state st;
9149
10423
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9150
10424
  slab_axes, naxes, 0);
9151
10425
  if ( rc != CA_ITER_OK ) {
@@ -9177,7 +10451,23 @@ ca_size_t out_i = 0;
9177
10451
  return LL2NUM(op[0]);
9178
10452
  }
9179
10453
  return vout;
9180
- }
10454
+ #undef st
10455
+ }
10456
+
10457
+ static VALUE
10458
+ count_equal_ki_native_i32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int32_t value_arg, ca_size_t min_count)
10459
+ {
10460
+ count_equal_ki_native_i32_frame fr;
10461
+ fr.self = self;
10462
+ fr.ca = ca;
10463
+ fr.slab_axes = slab_axes;
10464
+ fr.naxes = naxes;
10465
+ fr.keep_axis = keep_axis;
10466
+ fr.value_arg = value_arg;
10467
+ fr.min_count = min_count;
10468
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_i32_body, (VALUE) &fr);
10469
+ }
10470
+
9181
10471
 
9182
10472
  typedef struct {
9183
10473
  void *chunk;
@@ -9235,14 +10525,44 @@ count_equal_ki_stream_u32 (VALUE arg)
9235
10525
  return LL2NUM((ca_u_int64_t) (acc));
9236
10526
  }
9237
10527
 
10528
+ /* The walks can raise part way (a Ruby call per cell in the object
10529
+ lane, an allocation in any lane). Their states live in the caller's
10530
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10531
+ typedef struct {
10532
+ ca_iter_state states[1];
10533
+ VALUE self;
10534
+ CArray *ca;
10535
+ int8_t *slab_axes;
10536
+ int8_t naxes;
10537
+ int keep_axis;
10538
+ ca_u_uint32_t value_arg;
10539
+ ca_size_t min_count;
10540
+ } count_equal_ki_native_u32_frame;
10541
+
9238
10542
  static VALUE
9239
- count_equal_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint32_t value_arg, ca_size_t min_count)
10543
+ count_equal_ki_native_u32_body (VALUE arg)
9240
10544
  {
10545
+ count_equal_ki_native_u32_frame *fr = (count_equal_ki_native_u32_frame *) arg;
10546
+ VALUE self = fr->self;
10547
+ (void) self;
10548
+ CArray *ca = fr->ca;
10549
+ (void) ca;
10550
+ int8_t *slab_axes = fr->slab_axes;
10551
+ (void) slab_axes;
10552
+ int8_t naxes = fr->naxes;
10553
+ (void) naxes;
10554
+ int keep_axis = fr->keep_axis;
10555
+ (void) keep_axis;
10556
+ ca_u_uint32_t value_arg = fr->value_arg;
10557
+ (void) value_arg;
10558
+ ca_size_t min_count = fr->min_count;
10559
+ (void) min_count;
10560
+ #define st (fr->states[0])
9241
10561
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9242
10562
  ca->elements > 0 &&
9243
10563
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9244
10564
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9245
- ca_fast_paths_enabled() ) {
10565
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9246
10566
  count_equal_ki_stream_u32_t __ctx;
9247
10567
  __ctx.chunk = NULL;
9248
10568
  __ctx.ca = ca;
@@ -9255,7 +10575,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9255
10575
  CArray *co;
9256
10576
  GetCArray(vout, co);
9257
10577
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9258
- ca_iter_state st;
9259
10578
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9260
10579
  slab_axes, naxes, 0);
9261
10580
  if ( rc != CA_ITER_OK ) {
@@ -9287,7 +10606,23 @@ ca_size_t out_i = 0;
9287
10606
  return LL2NUM(op[0]);
9288
10607
  }
9289
10608
  return vout;
9290
- }
10609
+ #undef st
10610
+ }
10611
+
10612
+ static VALUE
10613
+ count_equal_ki_native_u32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint32_t value_arg, ca_size_t min_count)
10614
+ {
10615
+ count_equal_ki_native_u32_frame fr;
10616
+ fr.self = self;
10617
+ fr.ca = ca;
10618
+ fr.slab_axes = slab_axes;
10619
+ fr.naxes = naxes;
10620
+ fr.keep_axis = keep_axis;
10621
+ fr.value_arg = value_arg;
10622
+ fr.min_count = min_count;
10623
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_u32_body, (VALUE) &fr);
10624
+ }
10625
+
9291
10626
 
9292
10627
  typedef struct {
9293
10628
  void *chunk;
@@ -9345,14 +10680,44 @@ count_equal_ki_stream_i64 (VALUE arg)
9345
10680
  return LL2NUM((ca_u_int64_t) (acc));
9346
10681
  }
9347
10682
 
10683
+ /* The walks can raise part way (a Ruby call per cell in the object
10684
+ lane, an allocation in any lane). Their states live in the caller's
10685
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10686
+ typedef struct {
10687
+ ca_iter_state states[1];
10688
+ VALUE self;
10689
+ CArray *ca;
10690
+ int8_t *slab_axes;
10691
+ int8_t naxes;
10692
+ int keep_axis;
10693
+ ca_u_int64_t value_arg;
10694
+ ca_size_t min_count;
10695
+ } count_equal_ki_native_i64_frame;
10696
+
9348
10697
  static VALUE
9349
- count_equal_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int64_t value_arg, ca_size_t min_count)
10698
+ count_equal_ki_native_i64_body (VALUE arg)
9350
10699
  {
10700
+ count_equal_ki_native_i64_frame *fr = (count_equal_ki_native_i64_frame *) arg;
10701
+ VALUE self = fr->self;
10702
+ (void) self;
10703
+ CArray *ca = fr->ca;
10704
+ (void) ca;
10705
+ int8_t *slab_axes = fr->slab_axes;
10706
+ (void) slab_axes;
10707
+ int8_t naxes = fr->naxes;
10708
+ (void) naxes;
10709
+ int keep_axis = fr->keep_axis;
10710
+ (void) keep_axis;
10711
+ ca_u_int64_t value_arg = fr->value_arg;
10712
+ (void) value_arg;
10713
+ ca_size_t min_count = fr->min_count;
10714
+ (void) min_count;
10715
+ #define st (fr->states[0])
9351
10716
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9352
10717
  ca->elements > 0 &&
9353
10718
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9354
10719
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9355
- ca_fast_paths_enabled() ) {
10720
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9356
10721
  count_equal_ki_stream_i64_t __ctx;
9357
10722
  __ctx.chunk = NULL;
9358
10723
  __ctx.ca = ca;
@@ -9365,7 +10730,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9365
10730
  CArray *co;
9366
10731
  GetCArray(vout, co);
9367
10732
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9368
- ca_iter_state st;
9369
10733
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9370
10734
  slab_axes, naxes, 0);
9371
10735
  if ( rc != CA_ITER_OK ) {
@@ -9397,7 +10761,23 @@ ca_size_t out_i = 0;
9397
10761
  return LL2NUM(op[0]);
9398
10762
  }
9399
10763
  return vout;
9400
- }
10764
+ #undef st
10765
+ }
10766
+
10767
+ static VALUE
10768
+ count_equal_ki_native_i64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_int64_t value_arg, ca_size_t min_count)
10769
+ {
10770
+ count_equal_ki_native_i64_frame fr;
10771
+ fr.self = self;
10772
+ fr.ca = ca;
10773
+ fr.slab_axes = slab_axes;
10774
+ fr.naxes = naxes;
10775
+ fr.keep_axis = keep_axis;
10776
+ fr.value_arg = value_arg;
10777
+ fr.min_count = min_count;
10778
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_i64_body, (VALUE) &fr);
10779
+ }
10780
+
9401
10781
 
9402
10782
  typedef struct {
9403
10783
  void *chunk;
@@ -9455,14 +10835,44 @@ count_equal_ki_stream_u64 (VALUE arg)
9455
10835
  return LL2NUM((ca_u_int64_t) (acc));
9456
10836
  }
9457
10837
 
10838
+ /* The walks can raise part way (a Ruby call per cell in the object
10839
+ lane, an allocation in any lane). Their states live in the caller's
10840
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10841
+ typedef struct {
10842
+ ca_iter_state states[1];
10843
+ VALUE self;
10844
+ CArray *ca;
10845
+ int8_t *slab_axes;
10846
+ int8_t naxes;
10847
+ int keep_axis;
10848
+ ca_u_uint64_t value_arg;
10849
+ ca_size_t min_count;
10850
+ } count_equal_ki_native_u64_frame;
10851
+
9458
10852
  static VALUE
9459
- count_equal_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint64_t value_arg, ca_size_t min_count)
10853
+ count_equal_ki_native_u64_body (VALUE arg)
9460
10854
  {
10855
+ count_equal_ki_native_u64_frame *fr = (count_equal_ki_native_u64_frame *) arg;
10856
+ VALUE self = fr->self;
10857
+ (void) self;
10858
+ CArray *ca = fr->ca;
10859
+ (void) ca;
10860
+ int8_t *slab_axes = fr->slab_axes;
10861
+ (void) slab_axes;
10862
+ int8_t naxes = fr->naxes;
10863
+ (void) naxes;
10864
+ int keep_axis = fr->keep_axis;
10865
+ (void) keep_axis;
10866
+ ca_u_uint64_t value_arg = fr->value_arg;
10867
+ (void) value_arg;
10868
+ ca_size_t min_count = fr->min_count;
10869
+ (void) min_count;
10870
+ #define st (fr->states[0])
9461
10871
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9462
10872
  ca->elements > 0 &&
9463
10873
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9464
10874
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9465
- ca_fast_paths_enabled() ) {
10875
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9466
10876
  count_equal_ki_stream_u64_t __ctx;
9467
10877
  __ctx.chunk = NULL;
9468
10878
  __ctx.ca = ca;
@@ -9475,7 +10885,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9475
10885
  CArray *co;
9476
10886
  GetCArray(vout, co);
9477
10887
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9478
- ca_iter_state st;
9479
10888
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9480
10889
  slab_axes, naxes, 0);
9481
10890
  if ( rc != CA_ITER_OK ) {
@@ -9507,7 +10916,23 @@ ca_size_t out_i = 0;
9507
10916
  return LL2NUM(op[0]);
9508
10917
  }
9509
10918
  return vout;
9510
- }
10919
+ #undef st
10920
+ }
10921
+
10922
+ static VALUE
10923
+ count_equal_ki_native_u64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_uint64_t value_arg, ca_size_t min_count)
10924
+ {
10925
+ count_equal_ki_native_u64_frame fr;
10926
+ fr.self = self;
10927
+ fr.ca = ca;
10928
+ fr.slab_axes = slab_axes;
10929
+ fr.naxes = naxes;
10930
+ fr.keep_axis = keep_axis;
10931
+ fr.value_arg = value_arg;
10932
+ fr.min_count = min_count;
10933
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_u64_body, (VALUE) &fr);
10934
+ }
10935
+
9511
10936
 
9512
10937
  typedef struct {
9513
10938
  void *chunk;
@@ -9565,14 +10990,44 @@ count_equal_ki_stream_f32 (VALUE arg)
9565
10990
  return LL2NUM((ca_u_int64_t) (acc));
9566
10991
  }
9567
10992
 
10993
+ /* The walks can raise part way (a Ruby call per cell in the object
10994
+ lane, an allocation in any lane). Their states live in the caller's
10995
+ frame, and ca_iter_ensure finishes them however the body leaves. */
10996
+ typedef struct {
10997
+ ca_iter_state states[1];
10998
+ VALUE self;
10999
+ CArray *ca;
11000
+ int8_t *slab_axes;
11001
+ int8_t naxes;
11002
+ int keep_axis;
11003
+ ca_u_float32_t value_arg;
11004
+ ca_size_t min_count;
11005
+ } count_equal_ki_native_f32_frame;
11006
+
9568
11007
  static VALUE
9569
- count_equal_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float32_t value_arg, ca_size_t min_count)
11008
+ count_equal_ki_native_f32_body (VALUE arg)
9570
11009
  {
11010
+ count_equal_ki_native_f32_frame *fr = (count_equal_ki_native_f32_frame *) arg;
11011
+ VALUE self = fr->self;
11012
+ (void) self;
11013
+ CArray *ca = fr->ca;
11014
+ (void) ca;
11015
+ int8_t *slab_axes = fr->slab_axes;
11016
+ (void) slab_axes;
11017
+ int8_t naxes = fr->naxes;
11018
+ (void) naxes;
11019
+ int keep_axis = fr->keep_axis;
11020
+ (void) keep_axis;
11021
+ ca_u_float32_t value_arg = fr->value_arg;
11022
+ (void) value_arg;
11023
+ ca_size_t min_count = fr->min_count;
11024
+ (void) min_count;
11025
+ #define st (fr->states[0])
9571
11026
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9572
11027
  ca->elements > 0 &&
9573
11028
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9574
11029
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9575
- ca_fast_paths_enabled() ) {
11030
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9576
11031
  count_equal_ki_stream_f32_t __ctx;
9577
11032
  __ctx.chunk = NULL;
9578
11033
  __ctx.ca = ca;
@@ -9585,7 +11040,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9585
11040
  CArray *co;
9586
11041
  GetCArray(vout, co);
9587
11042
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9588
- ca_iter_state st;
9589
11043
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9590
11044
  slab_axes, naxes, 0);
9591
11045
  if ( rc != CA_ITER_OK ) {
@@ -9617,7 +11071,23 @@ ca_size_t out_i = 0;
9617
11071
  return LL2NUM(op[0]);
9618
11072
  }
9619
11073
  return vout;
9620
- }
11074
+ #undef st
11075
+ }
11076
+
11077
+ static VALUE
11078
+ count_equal_ki_native_f32 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float32_t value_arg, ca_size_t min_count)
11079
+ {
11080
+ count_equal_ki_native_f32_frame fr;
11081
+ fr.self = self;
11082
+ fr.ca = ca;
11083
+ fr.slab_axes = slab_axes;
11084
+ fr.naxes = naxes;
11085
+ fr.keep_axis = keep_axis;
11086
+ fr.value_arg = value_arg;
11087
+ fr.min_count = min_count;
11088
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_f32_body, (VALUE) &fr);
11089
+ }
11090
+
9621
11091
 
9622
11092
  typedef struct {
9623
11093
  void *chunk;
@@ -9675,14 +11145,44 @@ count_equal_ki_stream_f64 (VALUE arg)
9675
11145
  return LL2NUM((ca_u_int64_t) (acc));
9676
11146
  }
9677
11147
 
11148
+ /* The walks can raise part way (a Ruby call per cell in the object
11149
+ lane, an allocation in any lane). Their states live in the caller's
11150
+ frame, and ca_iter_ensure finishes them however the body leaves. */
11151
+ typedef struct {
11152
+ ca_iter_state states[1];
11153
+ VALUE self;
11154
+ CArray *ca;
11155
+ int8_t *slab_axes;
11156
+ int8_t naxes;
11157
+ int keep_axis;
11158
+ ca_u_float64_t value_arg;
11159
+ ca_size_t min_count;
11160
+ } count_equal_ki_native_f64_frame;
11161
+
9678
11162
  static VALUE
9679
- count_equal_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float64_t value_arg, ca_size_t min_count)
11163
+ count_equal_ki_native_f64_body (VALUE arg)
9680
11164
  {
11165
+ count_equal_ki_native_f64_frame *fr = (count_equal_ki_native_f64_frame *) arg;
11166
+ VALUE self = fr->self;
11167
+ (void) self;
11168
+ CArray *ca = fr->ca;
11169
+ (void) ca;
11170
+ int8_t *slab_axes = fr->slab_axes;
11171
+ (void) slab_axes;
11172
+ int8_t naxes = fr->naxes;
11173
+ (void) naxes;
11174
+ int keep_axis = fr->keep_axis;
11175
+ (void) keep_axis;
11176
+ ca_u_float64_t value_arg = fr->value_arg;
11177
+ (void) value_arg;
11178
+ ca_size_t min_count = fr->min_count;
11179
+ (void) min_count;
11180
+ #define st (fr->states[0])
9681
11181
  if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
9682
11182
  ca->elements > 0 &&
9683
11183
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9684
11184
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9685
- ca_fast_paths_enabled() ) {
11185
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9686
11186
  count_equal_ki_stream_f64_t __ctx;
9687
11187
  __ctx.chunk = NULL;
9688
11188
  __ctx.ca = ca;
@@ -9695,7 +11195,6 @@ VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9695
11195
  CArray *co;
9696
11196
  GetCArray(vout, co);
9697
11197
  ca_u_int64_t *op = (ca_u_int64_t *) co->ptr;
9698
- ca_iter_state st;
9699
11198
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
9700
11199
  slab_axes, naxes, 0);
9701
11200
  if ( rc != CA_ITER_OK ) {
@@ -9727,7 +11226,23 @@ ca_size_t out_i = 0;
9727
11226
  return LL2NUM(op[0]);
9728
11227
  }
9729
11228
  return vout;
9730
- }
11229
+ #undef st
11230
+ }
11231
+
11232
+ static VALUE
11233
+ count_equal_ki_native_f64 (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, ca_u_float64_t value_arg, ca_size_t min_count)
11234
+ {
11235
+ count_equal_ki_native_f64_frame fr;
11236
+ fr.self = self;
11237
+ fr.ca = ca;
11238
+ fr.slab_axes = slab_axes;
11239
+ fr.naxes = naxes;
11240
+ fr.keep_axis = keep_axis;
11241
+ fr.value_arg = value_arg;
11242
+ fr.min_count = min_count;
11243
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_f64_body, (VALUE) &fr);
11244
+ }
11245
+
9731
11246
 
9732
11247
  typedef struct {
9733
11248
  void *chunk;
@@ -9785,9 +11300,9 @@ count_equal_ki_stream_object (VALUE arg)
9785
11300
  return LL2NUM((ca_u_int64_t) (acc));
9786
11301
  }
9787
11302
 
9788
- /* The object lane calls Ruby for every cell, so a walk can raise part
9789
- way. Its states live in the caller's frame, and ca_iter_ensure
9790
- finishes them however the body leaves. */
11303
+ /* The walks can raise part way (a Ruby call per cell in the object
11304
+ lane, an allocation in any lane). Their states live in the caller's
11305
+ frame, and ca_iter_ensure finishes them however the body leaves. */
9791
11306
  typedef struct {
9792
11307
  ca_iter_state states[1];
9793
11308
  VALUE self;
@@ -9822,7 +11337,7 @@ count_equal_ki_native_object_body (VALUE arg)
9822
11337
  ca->elements > 0 &&
9823
11338
  ! ca_too_few_present(ca->elements, min_count, 0) &&
9824
11339
  ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
9825
- ca_fast_paths_enabled() ) {
11340
+ ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
9826
11341
  count_equal_ki_stream_object_t __ctx;
9827
11342
  __ctx.chunk = NULL;
9828
11343
  __ctx.ca = ca;
@@ -9880,19 +11395,48 @@ count_equal_ki_native_object (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
9880
11395
  fr.keep_axis = keep_axis;
9881
11396
  fr.value_arg = value_arg;
9882
11397
  fr.min_count = min_count;
9883
- return ca_iter_ensure(1, fr.states, count_equal_ki_native_object_body, (VALUE) &fr);
11398
+ return ca_iter_ensure_computing(ca, 1, fr.states, count_equal_ki_native_object_body, (VALUE) &fr);
9884
11399
  }
9885
11400
 
9886
11401
 
11402
+ /* The walks can raise part way (a Ruby call per cell in the object
11403
+ lane, an allocation in any lane). Their states live in the caller's
11404
+ frame, and ca_iter_ensure finishes them however the body leaves. */
11405
+ typedef struct {
11406
+ ca_iter_state states[1];
11407
+ VALUE self;
11408
+ CArray *ca;
11409
+ int8_t *slab_axes;
11410
+ int8_t naxes;
11411
+ int keep_axis;
11412
+ const char *value_arg;
11413
+ ca_size_t min_count;
11414
+ } count_equal_ki_native_fixlen_frame;
11415
+
9887
11416
  static VALUE
9888
- count_equal_ki_native_fixlen (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, const char *value_arg, ca_size_t min_count)
11417
+ count_equal_ki_native_fixlen_body (VALUE arg)
9889
11418
  {
11419
+ count_equal_ki_native_fixlen_frame *fr = (count_equal_ki_native_fixlen_frame *) arg;
11420
+ VALUE self = fr->self;
11421
+ (void) self;
11422
+ CArray *ca = fr->ca;
11423
+ (void) ca;
11424
+ int8_t *slab_axes = fr->slab_axes;
11425
+ (void) slab_axes;
11426
+ int8_t naxes = fr->naxes;
11427
+ (void) naxes;
11428
+ int keep_axis = fr->keep_axis;
11429
+ (void) keep_axis;
11430
+ const char *value_arg = fr->value_arg;
11431
+ (void) value_arg;
11432
+ ca_size_t min_count = fr->min_count;
11433
+ (void) min_count;
11434
+ #define st (fr->states[0])
9890
11435
  ca_size_t K = ca->bytes; /* uniform fixlen byte width */
9891
11436
  VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_INT64, keep_axis);
9892
11437
  CArray *co;
9893
11438
  GetCArray(vout, co);
9894
11439
  char *op = (char *) co->ptr;
9895
- ca_iter_state st;
9896
11440
  int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES, slab_axes, naxes, 0);
9897
11441
  if ( rc != CA_ITER_OK ) {
9898
11442
  rb_raise(rb_eRuntimeError, "%s: kernel_iterator init failed rc=%d", ca_calling_method_name(), rc);
@@ -9949,8 +11493,24 @@ count_equal_ki_native_fixlen (VALUE self, CArray *ca, int8_t *slab_axes, int8_t
9949
11493
  return LL2NUM(((int64_t *) op)[0]);
9950
11494
  }
9951
11495
  return vout;
11496
+ #undef st
9952
11497
  }
9953
11498
 
11499
+ static VALUE
11500
+ count_equal_ki_native_fixlen (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis, const char *value_arg, ca_size_t min_count)
11501
+ {
11502
+ count_equal_ki_native_fixlen_frame fr;
11503
+ fr.self = self;
11504
+ fr.ca = ca;
11505
+ fr.slab_axes = slab_axes;
11506
+ fr.naxes = naxes;
11507
+ fr.keep_axis = keep_axis;
11508
+ fr.value_arg = value_arg;
11509
+ fr.min_count = min_count;
11510
+ return ca_iter_ensure(1, fr.states, count_equal_ki_native_fixlen_body, (VALUE) &fr);
11511
+ }
11512
+
11513
+
9954
11514
  VALUE
9955
11515
  rb_ca_count_equal_ki (int argc, VALUE *argv, VALUE self)
9956
11516
  {