carray 3.0.4 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +438 -0
  3. data/README.md +1 -1
  4. data/carray.gemspec +3 -2
  5. data/ext/ca_array_pool.c +25 -1
  6. data/ext/ca_axis_dispatch.c +19 -20
  7. data/ext/ca_axis_group.c +1 -1
  8. data/ext/ca_categorical_iterator.c +34 -27
  9. data/ext/ca_compare.c +4 -4
  10. data/ext/ca_composite_dispatch.c +9 -9
  11. data/ext/ca_kernel_iterator.c +122 -237
  12. data/ext/ca_kernel_iterator.h +522 -337
  13. data/ext/ca_obj_array.c +69 -19
  14. data/ext/ca_obj_bincmp.c +10 -1
  15. data/ext/ca_obj_binop.c +11 -1
  16. data/ext/ca_obj_bitarray.c +5 -1
  17. data/ext/ca_obj_bitfield.c +60 -43
  18. data/ext/ca_obj_byte_swap.c +4 -1
  19. data/ext/ca_obj_const_string.c +15 -3
  20. data/ext/ca_obj_face.c +44 -1
  21. data/ext/ca_obj_fake.c +3 -1
  22. data/ext/ca_obj_field.c +13 -3
  23. data/ext/ca_obj_grid.c +219 -17
  24. data/ext/ca_obj_meld.c +69 -113
  25. data/ext/ca_obj_moncmp.c +9 -1
  26. data/ext/ca_obj_monop.c +86 -12
  27. data/ext/ca_obj_object.c +118 -32
  28. data/ext/ca_obj_record.c +3 -0
  29. data/ext/ca_obj_reduce.c +19 -1
  30. data/ext/ca_obj_refer.c +80 -26
  31. data/ext/ca_obj_remap.c +69 -10
  32. data/ext/ca_obj_roll.c +27 -6
  33. data/ext/ca_obj_select.c +38 -6
  34. data/ext/ca_obj_select_axis.c +1 -0
  35. data/ext/ca_obj_shift.c +32 -7
  36. data/ext/ca_obj_stack.c +311 -111
  37. data/ext/ca_obj_stride.c +159 -64
  38. data/ext/ca_obj_string.c +27 -2
  39. data/ext/ca_obj_tile.c +28 -7
  40. data/ext/ca_obj_transpose.c +8 -4
  41. data/ext/ca_obj_triop.c +12 -1
  42. data/ext/ca_obj_window.c +80 -50
  43. data/ext/ca_op_ipower.c +12 -12
  44. data/ext/ca_sort_kernels.h +6 -3
  45. data/ext/caframe_csv_reader.c +1 -1
  46. data/ext/carray.h +35 -1
  47. data/ext/carray_access.c +43 -67
  48. data/ext/carray_address_basis.c +134 -0
  49. data/ext/carray_attribute.c +32 -0
  50. data/ext/carray_broadcast.c +3 -4
  51. data/ext/carray_build_flags.h +1 -1
  52. data/ext/carray_cast.c +251 -93
  53. data/ext/carray_cast_func.rb +56 -26
  54. data/ext/carray_conversion.c +11 -7
  55. data/ext/carray_core.c +814 -72
  56. data/ext/carray_count.c +6 -0
  57. data/ext/carray_element.c +22 -13
  58. data/ext/carray_factorize.c +20 -20
  59. data/ext/carray_generate.c +8 -2
  60. data/ext/carray_histogram.c +26 -4
  61. data/ext/carray_index_classifier.c +4 -4
  62. data/ext/carray_internal.h +159 -12
  63. data/ext/carray_kernels_bincmp.c +2174 -170
  64. data/ext/carray_kernels_binop.c +25653 -5487
  65. data/ext/carray_kernels_init.c +8 -8
  66. data/ext/carray_kernels_moncmp.c +96 -96
  67. data/ext/carray_kernels_monop.c +12253 -5235
  68. data/ext/carray_kernels_reduce_aggregate.c +6150 -2423
  69. data/ext/carray_kernels_reduce_boolean.c +1533 -15
  70. data/ext/carray_kernels_reduce_cumulative.c +2442 -882
  71. data/ext/carray_kernels_reduce_extreme.c +5658 -1661
  72. data/ext/carray_kernels_reduce_variance.c +2534 -402
  73. data/ext/carray_kernels_scan.c +368 -368
  74. data/ext/carray_kernels_search.c +4551 -604
  75. data/ext/carray_kernels_sort.c +659 -751
  76. data/ext/carray_kernels_triop.c +114 -114
  77. data/ext/carray_key_windows.c +4 -0
  78. data/ext/carray_lazy.c +47 -0
  79. data/ext/carray_loop.c +1 -1
  80. data/ext/carray_mask.c +175 -161
  81. data/ext/carray_median_percentile.c +4 -4
  82. data/ext/carray_memory_view.c +15 -9
  83. data/ext/carray_operator.c +152 -49
  84. data/ext/carray_order.c +230 -10
  85. data/ext/carray_random.c +22 -19
  86. data/ext/carray_scatter.c +24 -7
  87. data/ext/carray_slab.c +128 -29
  88. data/ext/carray_sort.c +224 -16
  89. data/ext/carray_sort_kernel.c +5 -7
  90. data/ext/carray_struct.c +35 -43
  91. data/ext/carray_test.c +74 -19
  92. data/ext/carray_time_text.c +6 -4
  93. data/ext/carray_utils.c +63 -21
  94. data/ext/extconf.rb +11 -0
  95. data/ext/mkkernel.rb +546 -199
  96. data/ext/ruby_carray.c +4 -1
  97. data/ext/version.h +4 -4
  98. data/lib/carray/attribute.rb +42 -5
  99. data/lib/carray/autoload_carray.rb +3 -3
  100. data/lib/carray/autoload_method_extension.rb +5 -3
  101. data/lib/carray/axis_group.rb +13 -7
  102. data/lib/carray/basics.rb +28 -8
  103. data/lib/carray/block_iterator.rb +17 -3
  104. data/lib/carray/categorical.rb +35 -23
  105. data/lib/carray/categorical_iterator.rb +8 -8
  106. data/lib/carray/complex.rb +2 -2
  107. data/lib/carray/conditional.rb +3 -2
  108. data/lib/carray/const_string.rb +10 -0
  109. data/lib/carray/data_type_extension.rb +8 -1
  110. data/lib/carray/fixlen_string.rb +41 -2
  111. data/lib/carray/frame/column_verbs.rb +48 -16
  112. data/lib/carray/frame/concat.rb +17 -3
  113. data/lib/carray/frame/convert.rb +19 -0
  114. data/lib/carray/frame/csv_parser.rb +32 -32
  115. data/lib/carray/frame/describe.rb +4 -2
  116. data/lib/carray/frame/frame.rb +405 -85
  117. data/lib/carray/frame/group.rb +777 -64
  118. data/lib/carray/frame/io.rb +212 -72
  119. data/lib/carray/frame/join.rb +146 -37
  120. data/lib/carray/frame/records.rb +3 -3
  121. data/lib/carray/frame/reshape.rb +89 -11
  122. data/lib/carray/frame/sort.rb +57 -9
  123. data/lib/carray/frame/verbs.rb +384 -49
  124. data/lib/carray/frame/windows.rb +44 -6
  125. data/lib/carray/fuse_source.rb +31 -2
  126. data/lib/carray/inspect.rb +7 -2
  127. data/lib/carray/iterator.rb +32 -0
  128. data/lib/carray/lazy.rb +52 -8
  129. data/lib/carray/mask_gap_fill.rb +37 -17
  130. data/lib/carray/meld_reduce.rb +1 -1
  131. data/lib/carray/methods/choose.rb +3 -2
  132. data/lib/carray/methods/is_in.rb +8 -1
  133. data/lib/carray/methods/meshgrid.rb +3 -1
  134. data/lib/carray/runtime.rb +47 -8
  135. data/lib/carray/segment_iterator.rb +35 -16
  136. data/lib/carray/serialize.rb +53 -19
  137. data/lib/carray/stack.rb +33 -18
  138. data/lib/carray/string.rb +8 -0
  139. data/lib/carray/string_operation_extension.rb +13 -5
  140. data/lib/carray/struct_builder.rb +5 -3
  141. data/lib/carray/time.rb +112 -68
  142. data/lib/carray/window_iterator.rb +16 -4
  143. data/yard-stubs/ca_obj_array.rb +26 -1
  144. data/yard-stubs/ca_obj_transpose.rb +5 -4
  145. data/yard-stubs/carray_attribute.rb +3 -1
  146. data/yard-stubs/carray_cast.rb +10 -5
  147. data/yard-stubs/carray_conversion.rb +13 -3
  148. data/yard-stubs/carray_element.rb +7 -6
  149. data/yard-stubs/carray_mask.rb +28 -4
  150. data/yard-stubs/carray_math.rb +45 -20
  151. data/yard-stubs/carray_memory_view.rb +7 -0
  152. data/yard-stubs/carray_order.rb +100 -4
  153. data/yard-stubs/carray_scatter.rb +7 -5
  154. data/yard-stubs/carray_test.rb +7 -4
  155. metadata +18 -4
data/ext/ca_kernel_iterator.h CHANGED
@@ -49,14 +49,16 @@
49
49
  v, r, w, acc, idx, first
50
50
 
51
51
  INTERNAL (free to refactor across 3.x — engine implementation):
52
- - the state-machine functions' bodies + init_l1 / next_slab /
53
- ca_iter_can_alias
52
+ - the state-machine functions' bodies + ca_iter_can_alias
54
53
  - the macros only the kernel generator (ext/mkkernel.rb) emits:
55
54
  CA_SLAB_REDUCE_SUM8_EX / MIN8_EX / MAX8_EX / STAR8_EX (8-way
56
55
  accumulator split; they take EXPR(x) rather than exposing v and
57
56
  idx), CA_SLAB_REDUCE_T_MIN_EX / _MAX_EX / _STAR_EX (what it emits
58
57
  when a :min / :max / :prod kernel does not take the 8-way form),
59
- CA_SLAB_SCAN_T_GATED, and CA_SLAB_REDUCE_T_VAR_EX (no caller)
58
+ CA_SLAB_REDUCE_SUM8_NARROW_EX / M2_NARROW_EX (8- and 16-bit
59
+ integer sum and centred sum of squares), CA_SLAB_REDUCE_T_SETTLE
60
+ with CA_SLAB_SETTLE_BLOCK (all / any / none), CA_SLAB_SCAN_T_GATED,
61
+ and CA_SLAB_REDUCE_T_VAR_EX (no caller)
60
62
  - alias_mode / src_kind routing (CA_ITER_ALIAS_* / CA_ITER_SRC_* /
61
63
  CA_KERNEL_FIBER_CONTIG / CA_SLAB_FREE)
62
64
  - every ca_iter_state field NOT marked frozen (scratch / stack /
@@ -88,9 +90,18 @@ typedef enum {
88
90
 
89
91
  /* ---- kernel flags ------------------------------------------------------ */
90
92
  #define CA_KERNEL_READ 0x0
91
- #define CA_KERNEL_WRITE 0x1 /* the kernel writes the slab back */
93
+ #define CA_KERNEL_WRITE 0x1 /* the kernel writes the slab back.
94
+ The slab is read first (so a
95
+ kernel may change only some cells),
96
+ and a view that cannot be read --
97
+ CAFake of an object array whose
98
+ cells do not convert -- raises
99
+ there, even for a kernel that
100
+ writes every cell */
92
101
  #define CA_KERNEL_NO_MASK 0x2 /* the kernel cannot take a mask:
93
- a masked source is refused */
102
+ a source that has a mask is
103
+ refused, even with no cell
104
+ masked (no cell is scanned) */
94
105
  #define CA_KERNEL_CHUNK_HINT 0x4 /* reserved: init refuses it
95
106
  with CA_ITER_ERR_FLAGS */
96
107
  #define CA_KERNEL_FIBER_CONTIG 0x8 /* with policy=CA_SLAB_AXES and
@@ -129,13 +140,12 @@ typedef enum {
129
140
  non-innermost slab offsets are
130
141
  computed once per slab row and
131
142
  the inner run is a memcpy. */
132
- #define CA_ITER_ALIAS_PER_FIBER_FUSED 6 /* each fiber is fetched (and on a
133
- write put back) with the view's
134
- ca_xfer_stride, without
143
+ #define CA_ITER_ALIAS_PER_FIBER_FUSED 6 /* each fiber is fetched with the
144
+ view's ca_xfer_stride, without
135
145
  gathering the whole view: a
136
- CAWindow / CAShift fiber, or a
137
146
  read of the innermost fiber of
138
- a conversion view. Chosen by
147
+ a conversion view (a write takes
148
+ the transfer walk). Chosen by
139
149
  ca_iter_should_per_fiber_fused.
140
150
  The fiber is handed out from
141
151
  fiber_data_scratch, its mask
@@ -201,7 +211,8 @@ typedef enum {
201
211
  #define CA_ITER_SRC_ATTACH 3 /* every other view the classifier
202
212
  knows (conversion views, CATile,
203
213
  CARoll, CAStack, CAMeld, CARemap,
204
- CAObject, the lazy views) and any
214
+ CAReduce, CAObject, the lazy
215
+ views) and any
205
216
  registered external obj_type: the
206
217
  view's own transfer gathers it */
207
218
  #define CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE 4
@@ -226,16 +237,19 @@ typedef enum {
226
237
  CA_ITER_SRC_ATTACH is the only kind that may be registered. It is the
227
238
  one whose contract an external class can meet on its own: the engine
228
239
  reads the cells with ca_xfer_all GET and, on a WRITE walk, sends back
229
- the changed cells with ca_xfer_addrs (or ca_xfer_all PUT when every
230
- cell changed), so the view's transfer slots are all it needs -- the
231
- slots every view already implements. The view's attach / sync slots
232
- are not called. The other kinds are not open to registration: SRC_CASTRIDE
240
+ the changed cells with ca_xfer_addrs, so the view's transfer slots
241
+ are all the iterator needs. It does not call the view's attach / sync
242
+ slots, which ca_install_obj_type requires all the same: the paths
243
+ that take a view's buffer, such as a bang operator, call them. The
244
+ other kinds are not open to registration: SRC_CASTRIDE
233
245
  asserts the struct *is* a CAStride (the engine reads its strides
234
246
  directly, and a class that really is one is already classified by its
235
247
  inherited operation table), and SRC_DESCRIPTOR requires a
236
248
  describe_axes function the engine looks up in its own table, which an
237
249
  external type has no way to supply. Passing anything else raises
238
- rather than accepting a routing the iterator cannot honour.
250
+ rather than accepting a routing the iterator cannot honour. An
251
+ obj_type carray installs for itself is refused too: its routing comes
252
+ from its operation table.
239
253
 
240
254
  Registering is additive and does not override the two structural
241
255
  cases: the classifier still decides entity and CAStride-family sources
@@ -265,14 +279,24 @@ typedef struct {
265
279
  /* --- CAStride compose-fold cache (L2 alias path) ---
266
280
  For CAStride-family sources at L2, init_l2 runs
267
281
  ca_stride_compose_to_root once to get the root + per-axis
268
- byte strides + base offset, kept here for the author to read. The
269
- walk itself steps from composed_base by outer_strides (below), which
270
- init builds from the composed strides. Inline CA_RANK_MAX array (no
271
- heap alloc). root is the array attached at
272
- init whose memory the walk reads: the compose-fold root of an L2
273
- CAStride walk (an entity, or a view that lends memory) or the parent
274
- of a descriptor view. NULL when the walk reads src itself or a
275
- scratch of its own. */
282
+ byte strides + base offset, recorded here. These are engine
283
+ bookkeeping, not part of the author contract (a kernel reads the
284
+ slab_* / outer_* fields). The walk itself steps from composed_base
285
+ by outer_strides (below), which init builds from the composed
286
+ strides. Inline CA_RANK_MAX array (no heap alloc).
287
+
288
+ root is the array init attaches and finish detaches:
289
+ - the compose-fold root of an L2 CAStride walk (an entity, or a
290
+ view that lends memory), whose memory the slabs are cut from
291
+ - the parent of a descriptor view, on every descriptor route. On
292
+ the strided route (DESC_STRIDED) the slabs are cut from its
293
+ memory; on the L1 descriptor walk, PER_SLAB / PER_SLAB_HOIST
294
+ and DESC_WHOLE (SHIFT_WHOLE_VIEW included) the descriptor
295
+ engine gathers from it into a scratch of the walk's own (once,
296
+ or per slab for PER_SLAB) and the kernel reads the scratch, but
297
+ root stays attached until finish.
298
+ NULL when the walk reads src itself, a scratch filled through the
299
+ view's own transfer, or a CAStack's parents. */
276
300
  struct _CArray *root; /* attached at init, detached (or
277
301
  synced and detached, on WRITE)
278
302
  by finish */
@@ -282,7 +306,7 @@ typedef struct {
282
306
  /* --- descriptor framework cache (descriptor sources) ---
283
307
  For CASelectAxis / CAGrid / CASelect / CAWindow / CAShift, init runs
284
308
  the view's *_describe_axes once then
285
- reuses the P3 substrate (ca_axis_dispatch_prepare / _layout /
309
+ reuses the descriptor engine's substrate (ca_axis_dispatch_prepare / _layout /
286
310
  _classify_prefix) to derive the slab layout. Cached inline so
287
311
  next_slab walks the prefix axes without re-doing the analysis. */
288
312
  ca_axis_desc_t descs[CA_RANK_MAX]; /* post-merge axes */
@@ -399,9 +423,8 @@ typedef struct {
399
423
  over src->dim used in ca_xfer_stride strides[] argument. */
400
424
  int8_t fiber_axis;
401
425
  ca_size_t fiber_native_strides[CA_RANK_MAX];
402
- /* fiber_region_starts[] cached by next_slab_axes BEFORE outer_idx
403
- advance so sync_slab can reconstruct the same ca_xfer_stride
404
- region for WRITE PUT. The same hazard as last_data_off. */
426
+ /* fiber_region_starts[]: not used; kept so the fields after it stay
427
+ where ext gems built against this header expect them. */
405
428
  ca_size_t fiber_region_starts[CA_RANK_MAX];
406
429
 
407
430
  /* --- CAStack read from its parents (STACK / STACK_OUTER_K) --- */
@@ -519,15 +542,18 @@ int ca_iter_state_init_l1 (ca_iter_state *st,
519
542
  - entity / CAStride contig: alias_mode = CONTIG, the slab reads
520
543
  the source's own memory
521
544
  - CAStride non-contig: alias_mode = STRIDED, the slab reads the
522
- root through the composed strides. No scratch.
545
+ root through the composed strides, without a scratch, when the
546
+ root lends memory; over a root that does not (a conversion view,
547
+ a CAObject, a CAStack, ...) the view is gathered into scratch
523
548
  - descriptor views: aliased when the innermost axis is a STRIDE
524
549
  axis, otherwise gathered into scratch (per slab when a slab axis
525
550
  is an INDEX or SHIFT axis)
526
551
  - SRC_ATTACH views: iterator-owned scratch filled through
527
552
  ca_xfer_all, written back with ki_scratch_push_changed (only the
528
553
  changed cells). A CAStack may instead be walked through its
529
- parents, and a single FIBER_CONTIG fiber of a cast / byte-swap
530
- view through ca_xfer_stride (PER_FIBER_FUSED)
554
+ parents, and on a READ walk a single FIBER_CONTIG fiber along the
555
+ innermost axis of a cast / byte-swap / single-cast lazy view
556
+ through ca_xfer_stride (PER_FIBER_FUSED)
531
557
  - a source with no attach slot: CA_ITER_ERR_NOT_CHEAP
532
558
 
533
559
  policy: CA_SLAB_AXES only. CA_SLAB_WHOLE and CA_SLAB_FREE return
@@ -625,14 +651,14 @@ int ca_iter_state_next_slab_axes (ca_iter_state *st,
625
651
  - per-fiber gather (CA_KERNEL_FIBER_CONTIG on a strided fiber):
626
652
  the fiber is scattered back from its scratch first
627
653
  - CAStack walked through its parents: READ only, no-op
628
- - per-fiber fused (CA_ITER_ALIAS_PER_FIBER_FUSED): the fiber is put
629
- back through the view with ca_xfer_stride
654
+ - per-fiber fused (CA_ITER_ALIAS_PER_FIBER_FUSED): READ only, so a
655
+ WRITE walk never has it
630
656
  - alias path (scratch_ptr == NULL): no-op — kernel wrote through
631
657
  alias_ptr. When the alias is into a view root's own buffer, that
632
658
  buffer is written back once, by finish (ca_sync_detach)
633
659
  - scratch path, at L1 or L2: the cells the kernel changed are sent
634
- back through the source (ki_scratch_push_changed: ca_xfer_addrs,
635
- or ca_xfer_all PUT when every cell changed); a write over a
660
+ back through the source (ki_scratch_push_changed: ca_xfer_addrs);
661
+ a write over a
636
662
  descriptor view takes this path, opened as a transfer walk
637
663
 
638
664
  A view that shows one root cell more than once (a repeated index, a
@@ -693,7 +719,8 @@ void ca_iter_state_close (ca_iter_state *st);
693
719
  int ca_iter_check_init (int rc);
694
720
 
695
721
  /* Raise ArgumentError unless the two arrays have the same shape. The
696
- INOUT and PAIR fiber macros call this before opening either walk. */
722
+ INOUT, SLAB_INOUT and PAIR macros call this before opening either
723
+ walk. */
697
724
  int ca_iter_check_same_shape (struct _CArray *a, struct _CArray *b);
698
725
 
699
726
  /* Run body(arg) so that the walks in states[0..n) are finished however the
@@ -803,18 +830,19 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
803
830
  Plain C99 (a do/while block with for-loop declarations). */
804
831
 
805
832
  #define CA_L2_FOR_EACH(T, ptr, n, stride, p, body) do { \
833
+ typedef T __ca_u_T CA_UNALIGNED; \
806
834
  ca_size_t _l2_n = (n); \
807
835
  ca_size_t _l2_s = (stride); \
808
836
  char *_l2_b = (char *)(ptr); \
809
837
  if (_l2_s == sizeof(T)) { \
810
- T *p = (T *)_l2_b; \
838
+ __ca_u_T *p = (__ca_u_T *)_l2_b; \
811
839
  for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
812
840
  body \
813
841
  p++; \
814
842
  } \
815
843
  } else { \
816
844
  for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
817
- T *p = (T *)(_l2_b + _l2_i * _l2_s); \
845
+ __ca_u_T *p = (__ca_u_T *)(_l2_b + _l2_i * _l2_s); \
818
846
  body \
819
847
  } \
820
848
  } \
@@ -983,8 +1011,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
983
1011
  The INOUT forms cover input + output. These cover input + input, which
984
1012
  is what a routine taking two vectors of the same length wants -- a
985
1013
  correlation, a dot product, a distance. Neither state gets
986
- CA_KERNEL_WRITE, and `flags` must not carry it: use the INOUT forms to
987
- write.
1014
+ CA_KERNEL_WRITE, and `flags` must not carry it (it is refused with
1015
+ CA_ITER_ERR_FLAGS): use the INOUT forms to write.
988
1016
 
989
1017
  Both fibers are contig-delivered, so the pair may be handed straight to
990
1018
  a C routine that walks it itself. The MASKED form yields BOTH mask
@@ -1000,6 +1028,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1000
1028
  #define CA_FOR_EACH_FIBER_PAIR(st_a, st_b, ca_a, ca_b, axis, \
1001
1029
  flags, p_a, p_b, n) \
1002
1030
  for ( int __cffp_init = ( \
1031
+ ca_iter_check_init(((flags) & CA_KERNEL_WRITE) \
1032
+ ? CA_ITER_ERR_FLAGS : CA_ITER_OK), \
1003
1033
  ca_iter_check_same_shape((ca_a), (ca_b)), \
1004
1034
  ca_iter_check_init( \
1005
1035
  ca_iter_state_init_l2(&(st_a), (ca_a), CA_SLAB_AXES, \
@@ -1024,6 +1054,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1024
1054
  #define CA_FOR_EACH_FIBER_PAIR_MASKED(st_a, st_b, ca_a, ca_b, axis, \
1025
1055
  flags, p_a, p_b, n, m_a, m_b) \
1026
1056
  for ( int __cffpm_init = ( \
1057
+ ca_iter_check_init(((flags) & CA_KERNEL_WRITE) \
1058
+ ? CA_ITER_ERR_FLAGS : CA_ITER_OK), \
1027
1059
  ca_iter_check_same_shape((ca_a), (ca_b)), \
1028
1060
  ca_iter_check_init( \
1029
1061
  ca_iter_state_init_l2(&(st_a), (ca_a), CA_SLAB_AXES, \
@@ -1107,6 +1139,38 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1107
1139
 
1108
1140
  /* ---- per-data_type reduction macro suite ------------------------------- */
1109
1141
 
1142
+ /* Whether the stride table `strides` walks the current slab of `st` in
1143
+ row-major order with no gap: the innermost axis steps `elem_bytes`, and
1144
+ each outer axis steps the whole of the axis inside it. A slab that does
1145
+ can be read as one flat run of slab_elements cells. A length-1 axis whose
1146
+ stride does not fit is answered "no", which only costs the flat loop. */
1147
+ static inline int
1148
+ ca_slab_strides_are_flat (const ca_iter_state *st, const ca_size_t *strides,
1149
+ ca_size_t elem_bytes)
1150
+ {
1151
+ int8_t k = (int8_t) (st->slab_ndim - 1);
1152
+ if ( strides[k] != elem_bytes ) {
1153
+ return 0;
1154
+ }
1155
+ for ( k = (int8_t) (k - 1); k >= 0; k-- ) {
1156
+ if ( strides[k] != st->slab_dims[k + 1] * strides[k + 1] ) {
1157
+ return 0;
1158
+ }
1159
+ }
1160
+ return 1;
1161
+ }
1162
+
1163
+ /* Whether the slab reductions may fold all slab axes of `st` into one flat
1164
+ loop: the data strides and, when the slab has a mask, the mask strides
1165
+ (one byte per cell) are row-major with no gap. */
1166
+ static inline int
1167
+ ca_slab_is_flat (const ca_iter_state *st, const ca_size_t *strides,
1168
+ ca_size_t elem_bytes, int has_mask)
1169
+ {
1170
+ return ca_slab_strides_are_flat(st, strides, elem_bytes)
1171
+ && ( ! has_mask || ca_slab_strides_are_flat(st, st->slab_mask_strides, 1) );
1172
+ }
1173
+
1110
1174
  /* CA_SLAB_REDUCE_T(T, ...): generic per-data_type slab reduction. T is the
1111
1175
  element load type (`double`, `float`, `int32_t`, `int64_t`, ...).
1112
1176
  The accumulator `acc` is supplied by the caller and may be a wider
@@ -1150,6 +1214,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1150
1214
  it dead-code-eliminated; the trailing `(void) idx;` silences any
1151
1215
  set-but-not-used warnings. */
1152
1216
  #define CA_SLAB_REDUCE_T_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
1217
+ typedef T __ca_u_T CA_UNALIGNED; \
1153
1218
  (acc) = (INIT); \
1154
1219
  int8_t __srK = (st).slab_ndim; \
1155
1220
  int8_t __srOuterK = __srK - 1; \
@@ -1161,27 +1226,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1161
1226
  ca_size_t __srOC = 1; \
1162
1227
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1163
1228
  __srOC *= (st).slab_dims[__sk]; \
1164
- /* Slab-collapse: when the whole K-D slab is row-major contiguous \
1165
- (data, and mask if present), fold all slab axes into one flat \
1166
- inner loop. Removes the per-outer multi-index offset recompute + \
1167
- carry that otherwise dominates when the innermost slab axis is \
1168
- small (full reduction of [N,1] / [N,small] entity, or trailing- \
1169
- contig multi-axis reduce). Platform-general: structural, not SIMD. \
1170
- No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
1171
- { \
1172
- int __srFlat = __srContig; \
1173
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1174
- if ( (st).slab_strides[__sk] != \
1175
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1176
- __srFlat = 0; \
1177
- if ( __srFlat && (m) != NULL ) { \
1178
- if ( ! __srMaskU ) __srFlat = 0; \
1179
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1180
- if ( (st).slab_mask_strides[__sk] != \
1181
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1182
- __srFlat = 0; \
1183
- } \
1184
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
1229
+ /* fold all slab axes into one flat loop when the slab is row-major \
1230
+ contiguous (ca_slab_is_flat) */ \
1231
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1232
+ __srInnerN = (st).slab_elements; \
1233
+ __srOC = 1; \
1185
1234
  } \
1186
1235
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1187
1236
  ca_size_t idx = 0; \
@@ -1194,7 +1243,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1194
1243
  const char *__srQ = (const char *)(p) + __srDoff; \
1195
1244
  if ( (m) == NULL ) { \
1196
1245
  if ( __srContig ) { \
1197
- const T *__srSrc = (const T *) __srQ; \
1246
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1198
1247
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1199
1248
  T v = __srSrc[__sj]; \
1200
1249
  REDUCE; \
@@ -1202,7 +1251,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1202
1251
  } \
1203
1252
  } else { \
1204
1253
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1205
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
1254
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1206
1255
  REDUCE; \
1207
1256
  idx++; \
1208
1257
  } \
@@ -1210,7 +1259,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1210
1259
  } else { \
1211
1260
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1212
1261
  if ( __srContig && __srMaskU ) { \
1213
- const T *__srSrc = (const T *) __srQ; \
1262
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1214
1263
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1215
1264
  if ( ! __srMM[__sj] ) { \
1216
1265
  T v = __srSrc[__sj]; \
@@ -1223,7 +1272,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1223
1272
  } else { \
1224
1273
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1225
1274
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1226
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
1275
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1227
1276
  REDUCE; \
1228
1277
  } else { \
1229
1278
  (masked_cnt)++; \
@@ -1250,6 +1299,84 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1250
1299
  (void) __sr_throwaway_mc; \
1251
1300
  } while (0)
1252
1301
 
1302
+ /* CA_SLAB_REDUCE_T_SETTLE: a reduction that may stop before the end of
1303
+ the slab. SETTLED is an expression over `acc` that is true once no
1304
+ further cell can change the result (`acc` for any, `!acc` for all).
1305
+ The slab is walked in blocks of CA_SLAB_SETTLE_BLOCK cells; each block
1306
+ is the same branch-free loop as CA_SLAB_REDUCE_T_EX, and SETTLED is
1307
+ tested between blocks, so a slab that never settles costs one extra
1308
+ test per block. Masked cells are skipped as in CA_SLAB_REDUCE_T.
1309
+ `idx` and the masked count are not provided. */
1310
+ #define CA_SLAB_SETTLE_BLOCK 16384
1311
+ #define CA_SLAB_REDUCE_T_SETTLE(T, st, p, m, acc, INIT, REDUCE, SETTLED) do { \
1312
+ typedef T __ca_u_T CA_UNALIGNED; \
1313
+ (acc) = (INIT); \
1314
+ int8_t __srK = (st).slab_ndim; \
1315
+ int8_t __srOuterK = __srK - 1; \
1316
+ ca_size_t __srInnerN = (st).slab_dims[__srK - 1]; \
1317
+ ca_size_t __srInnerS = (st).slab_strides[__srK - 1]; \
1318
+ ca_size_t __srInnerMS = (st).slab_mask_strides[__srK - 1]; \
1319
+ int __srContig = (__srInnerS == (ca_size_t) sizeof(T)); \
1320
+ int __srMaskU = (__srInnerMS == 1); \
1321
+ ca_size_t __srOC = 1; \
1322
+ for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1323
+ __srOC *= (st).slab_dims[__sk]; \
1324
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1325
+ __srInnerN = (st).slab_elements; \
1326
+ __srOC = 1; \
1327
+ } \
1328
+ ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1329
+ int __srDone = 0; \
1330
+ for ( ca_size_t __so = 0; __so < __srOC && ! __srDone; __so++ ) { \
1331
+ ca_size_t __srDoff = 0, __srMoff = 0; \
1332
+ for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) { \
1333
+ __srDoff += __srIdx[__sk] * (st).slab_strides[__sk]; \
1334
+ __srMoff += __srIdx[__sk] * (st).slab_mask_strides[__sk]; \
1335
+ } \
1336
+ const char *__srQ = (const char *)(p) + __srDoff; \
1337
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1338
+ const boolean8_t *__srMM = \
1339
+ ( (m) == NULL ) ? NULL : (const boolean8_t *)(m) + __srMoff; \
1340
+ for ( ca_size_t __sb = 0; __sb < __srInnerN; \
1341
+ __sb += CA_SLAB_SETTLE_BLOCK ) { \
1342
+ ca_size_t __se = ( __srInnerN - __sb > CA_SLAB_SETTLE_BLOCK ) \
1343
+ ? __sb + CA_SLAB_SETTLE_BLOCK : __srInnerN; \
1344
+ if ( __srMM == NULL ) { \
1345
+ if ( __srContig ) { \
1346
+ for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
1347
+ T v = __srSrc[__sj]; \
1348
+ REDUCE; \
1349
+ } \
1350
+ } else { \
1351
+ for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
1352
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1353
+ REDUCE; \
1354
+ } \
1355
+ } \
1356
+ } else if ( __srContig && __srMaskU ) { \
1357
+ for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
1358
+ if ( ! __srMM[__sj] ) { \
1359
+ T v = __srSrc[__sj]; \
1360
+ REDUCE; \
1361
+ } \
1362
+ } \
1363
+ } else { \
1364
+ for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
1365
+ if ( ! __srMM[__sj * __srInnerMS] ) { \
1366
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1367
+ REDUCE; \
1368
+ } \
1369
+ } \
1370
+ } \
1371
+ if ( SETTLED ) { __srDone = 1; break; } \
1372
+ } \
1373
+ for ( int8_t __sk = __srOuterK - 1; __sk >= 0; __sk-- ) { \
1374
+ if ( ++__srIdx[__sk] < (st).slab_dims[__sk] ) break; \
1375
+ __srIdx[__sk] = 0; \
1376
+ } \
1377
+ } \
1378
+ } while (0)
1379
+
1253
1380
  /* ----------------------------------------------------------------------
1254
1381
  * SIMD-licensed reduction variants
1255
1382
  *
@@ -1289,6 +1416,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1289
1416
  * by the compiler when their update is a simple ++ pattern.
1290
1417
  */
1291
1418
  #define CA_SLAB_REDUCE_T_PLUS_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
1419
+ typedef T __ca_u_T CA_UNALIGNED; \
1292
1420
  (acc) = (INIT); \
1293
1421
  int8_t __srK = (st).slab_ndim; \
1294
1422
  int8_t __srOuterK = __srK - 1; \
@@ -1300,27 +1428,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1300
1428
  ca_size_t __srOC = 1; \
1301
1429
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1302
1430
  __srOC *= (st).slab_dims[__sk]; \
1303
- /* Slab-collapse: when the whole K-D slab is row-major contiguous \
1304
- (data, and mask if present), fold all slab axes into one flat \
1305
- inner loop. Removes the per-outer multi-index offset recompute + \
1306
- carry that otherwise dominates when the innermost slab axis is \
1307
- small (full reduction of [N,1] / [N,small] entity, or trailing- \
1308
- contig multi-axis reduce). Platform-general: structural, not SIMD. \
1309
- No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
1310
- { \
1311
- int __srFlat = __srContig; \
1312
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1313
- if ( (st).slab_strides[__sk] != \
1314
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1315
- __srFlat = 0; \
1316
- if ( __srFlat && (m) != NULL ) { \
1317
- if ( ! __srMaskU ) __srFlat = 0; \
1318
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1319
- if ( (st).slab_mask_strides[__sk] != \
1320
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1321
- __srFlat = 0; \
1322
- } \
1323
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
1431
+ /* fold all slab axes into one flat loop when the slab is row-major \
1432
+ contiguous (ca_slab_is_flat) */ \
1433
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1434
+ __srInnerN = (st).slab_elements; \
1435
+ __srOC = 1; \
1324
1436
  } \
1325
1437
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1326
1438
  ca_size_t idx = 0; \
@@ -1333,7 +1445,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1333
1445
  const char *__srQ = (const char *)(p) + __srDoff; \
1334
1446
  if ( (m) == NULL ) { \
1335
1447
  if ( __srContig ) { \
1336
- const T *__srSrc = (const T *) __srQ; \
1448
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1337
1449
  _CA_SIMD_PLUS(acc) \
1338
1450
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1339
1451
  T v = __srSrc[__sj]; \
@@ -1342,7 +1454,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1342
1454
  } \
1343
1455
  } else { \
1344
1456
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1345
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
1457
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1346
1458
  REDUCE; \
1347
1459
  idx++; \
1348
1460
  } \
@@ -1350,7 +1462,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1350
1462
  } else { \
1351
1463
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1352
1464
  if ( __srContig && __srMaskU ) { \
1353
- const T *__srSrc = (const T *) __srQ; \
1465
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1354
1466
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1355
1467
  if ( ! __srMM[__sj] ) { \
1356
1468
  T v = __srSrc[__sj]; \
@@ -1363,7 +1475,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1363
1475
  } else { \
1364
1476
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1365
1477
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1366
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
1478
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1367
1479
  REDUCE; \
1368
1480
  } else { \
1369
1481
  (masked_cnt)++; \
@@ -1409,6 +1521,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1409
1521
  * both passes of variance / stddev).
1410
1522
  */
1411
1523
  #define CA_SLAB_REDUCE_SUM8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
1524
+ typedef T __ca_u_T CA_UNALIGNED; \
1412
1525
  (acc) = (INIT); \
1413
1526
  int8_t __srK = (st).slab_ndim; \
1414
1527
  int8_t __srOuterK = __srK - 1; \
@@ -1420,20 +1533,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1420
1533
  ca_size_t __srOC = 1; \
1421
1534
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1422
1535
  __srOC *= (st).slab_dims[__sk]; \
1423
- { \
1424
- int __srFlat = __srContig; \
1425
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1426
- if ( (st).slab_strides[__sk] != \
1427
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1428
- __srFlat = 0; \
1429
- if ( __srFlat && (m) != NULL ) { \
1430
- if ( ! __srMaskU ) __srFlat = 0; \
1431
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1432
- if ( (st).slab_mask_strides[__sk] != \
1433
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1434
- __srFlat = 0; \
1435
- } \
1436
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
1536
+ /* fold all slab axes into one flat loop when the slab is row-major \
1537
+ contiguous (ca_slab_is_flat) */ \
1538
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1539
+ __srInnerN = (st).slab_elements; \
1540
+ __srOC = 1; \
1437
1541
  } \
1438
1542
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1439
1543
  for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
@@ -1445,7 +1549,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1445
1549
  const char *__srQ = (const char *)(p) + __srDoff; \
1446
1550
  if ( (m) == NULL ) { \
1447
1551
  if ( __srContig ) { \
1448
- const T *__srSrc = (const T *) __srQ; \
1552
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1449
1553
  ACC_T __a0=(ACC_T)0,__a1=(ACC_T)0,__a2=(ACC_T)0,__a3=(ACC_T)0; \
1450
1554
  ACC_T __a4=(ACC_T)0,__a5=(ACC_T)0,__a6=(ACC_T)0,__a7=(ACC_T)0; \
1451
1555
  ca_size_t __sj = 0, __sN = __srInnerN; \
@@ -1464,14 +1568,96 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1464
1568
  + ((__a4 + __a5) + (__a6 + __a7)); \
1465
1569
  } else { \
1466
1570
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1467
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
1571
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1572
+ (acc) += EXPR(v); \
1573
+ } \
1574
+ } \
1575
+ } else { \
1576
+ const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1577
+ if ( __srContig && __srMaskU ) { \
1578
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1579
+ for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1580
+ if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) += EXPR(v); } \
1581
+ else { (masked_cnt)++; } \
1582
+ } \
1583
+ } else { \
1584
+ for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1585
+ if ( ! __srMM[__sj * __srInnerMS] ) { \
1586
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1587
+ (acc) += EXPR(v); \
1588
+ } else { (masked_cnt)++; } \
1589
+ } \
1590
+ } \
1591
+ } \
1592
+ for ( int8_t __sk = __srOuterK - 1; __sk >= 0; __sk-- ) { \
1593
+ if ( ++__srIdx[__sk] < (st).slab_dims[__sk] ) break; \
1594
+ __srIdx[__sk] = 0; \
1595
+ } \
1596
+ } \
1597
+ } while (0)
1598
+
1599
+ /* CA_SLAB_REDUCE_SUM8_NARROW_EX:
1600
+ * CA_SLAB_REDUCE_SUM8_EX for an 8- or 16-bit integer source summed into
1601
+ * a floating accumulator (sum and mean). Converting each cell to double
1602
+ * does not vectorize for these widths, so the contig run is summed in
1603
+ * int32 over blocks of 32768 cells (|cell| < 2^16, so a block sum stays
1604
+ * below 2^31) and only the block sums meet the accumulator. The block
1605
+ * sums are exact, so the answer is at least as close as the per-cell
1606
+ * form. EXPR(x) converts a block sum to ACC_T. The masked and
1607
+ * non-contig branches are the same as CA_SLAB_REDUCE_SUM8_EX.
1608
+ *
1609
+ * The block loop has no omp simd pragma: an integer sum vectorizes on
1610
+ * its own, and with the pragma GCC left a scalar copy of the loop in
1611
+ * the large generated kernels.
1612
+ */
1613
+ #define CA_SLAB_REDUCE_SUM8_NARROW_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
1614
+ typedef T __ca_u_T CA_UNALIGNED; \
1615
+ (acc) = (INIT); \
1616
+ int8_t __srK = (st).slab_ndim; \
1617
+ int8_t __srOuterK = __srK - 1; \
1618
+ ca_size_t __srInnerN = (st).slab_dims[__srK - 1]; \
1619
+ ca_size_t __srInnerS = (st).slab_strides[__srK - 1]; \
1620
+ ca_size_t __srInnerMS = (st).slab_mask_strides[__srK - 1]; \
1621
+ int __srContig = (__srInnerS == (ca_size_t) sizeof(T)); \
1622
+ int __srMaskU = (__srInnerMS == 1); \
1623
+ ca_size_t __srOC = 1; \
1624
+ for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1625
+ __srOC *= (st).slab_dims[__sk]; \
1626
+ /* fold all slab axes into one flat loop when the slab is row-major \
1627
+ contiguous (ca_slab_is_flat) */ \
1628
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1629
+ __srInnerN = (st).slab_elements; \
1630
+ __srOC = 1; \
1631
+ } \
1632
+ ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1633
+ for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
1634
+ ca_size_t __srDoff = 0, __srMoff = 0; \
1635
+ for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) { \
1636
+ __srDoff += __srIdx[__sk] * (st).slab_strides[__sk]; \
1637
+ __srMoff += __srIdx[__sk] * (st).slab_mask_strides[__sk]; \
1638
+ } \
1639
+ const char *__srQ = (const char *)(p) + __srDoff; \
1640
+ if ( (m) == NULL ) { \
1641
+ if ( __srContig ) { \
1642
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1643
+ for ( ca_size_t __sb = 0; __sb < __srInnerN; __sb += 32768 ) { \
1644
+ ca_size_t __se = ( __srInnerN - __sb < 32768 ) \
1645
+ ? __srInnerN : __sb + 32768; \
1646
+ int32_t __sp = 0; \
1647
+ for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) \
1648
+ __sp += __srSrc[__sj]; \
1649
+ (acc) += EXPR(__sp); \
1650
+ } \
1651
+ } else { \
1652
+ for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1653
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1468
1654
  (acc) += EXPR(v); \
1469
1655
  } \
1470
1656
  } \
1471
1657
  } else { \
1472
1658
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1473
1659
  if ( __srContig && __srMaskU ) { \
1474
- const T *__srSrc = (const T *) __srQ; \
1660
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1475
1661
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1476
1662
  if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) += EXPR(v); } \
1477
1663
  else { (masked_cnt)++; } \
@@ -1479,7 +1665,104 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1479
1665
  } else { \
1480
1666
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1481
1667
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1482
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
1668
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1669
+ (acc) += EXPR(v); \
1670
+ } else { (masked_cnt)++; } \
1671
+ } \
1672
+ } \
1673
+ } \
1674
+ for ( int8_t __sk = __srOuterK - 1; __sk >= 0; __sk-- ) { \
1675
+ if ( ++__srIdx[__sk] < (st).slab_dims[__sk] ) break; \
1676
+ __srIdx[__sk] = 0; \
1677
+ } \
1678
+ } \
1679
+ } while (0)
1680
+
1681
+ /* CA_SLAB_REDUCE_M2_NARROW_EX:
1682
+ * The centred sum of squares, sum (x - MEAN)^2, of an 8- or 16-bit
1683
+ * integer source (the second pass of variance and stddev). Converting
1684
+ * each cell to double does not vectorize for these widths, so the
1685
+ * contig run is centred on the integer c nearest MEAN (held within the
1686
+ * 8- and 16-bit range), and D = sum (x - c) and sum (x - c)^2 are added
1687
+ * in integers over blocks of 32768 cells; then
1688
+ * sum (x - MEAN)^2 = sum (x - c)^2 - 2 (MEAN - c) D + n (MEAN - c)^2.
1689
+ * The identity holds for any MEAN, and when MEAN is the mean of the
1690
+ * cells its rounding error enters only squared. Q_T is int32_t for
1691
+ * 8-bit cells (a block is at most 32768 * 255^2 < 2^31) and int64_t for
1692
+ * 16-bit cells. The block sums are exact. The masked and non-contig
1693
+ * branches add EXPR(x) per cell as CA_SLAB_REDUCE_SUM8_EX does; acc is
1694
+ * double.
1695
+ */
1696
+ #define CA_SLAB_REDUCE_M2_NARROW_EX(T, Q_T, st, p, m, acc, INIT, MEAN, EXPR, masked_cnt) do { \
1697
+ typedef T __ca_u_T CA_UNALIGNED; \
1698
+ (acc) = (INIT); \
1699
+ int8_t __srK = (st).slab_ndim; \
1700
+ int8_t __srOuterK = __srK - 1; \
1701
+ ca_size_t __srInnerN = (st).slab_dims[__srK - 1]; \
1702
+ ca_size_t __srInnerS = (st).slab_strides[__srK - 1]; \
1703
+ ca_size_t __srInnerMS = (st).slab_mask_strides[__srK - 1]; \
1704
+ int __srContig = (__srInnerS == (ca_size_t) sizeof(T)); \
1705
+ int __srMaskU = (__srInnerMS == 1); \
1706
+ ca_size_t __srOC = 1; \
1707
+ for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1708
+ __srOC *= (st).slab_dims[__sk]; \
1709
+ /* fold all slab axes into one flat loop when the slab is row-major \
1710
+ contiguous (ca_slab_is_flat) */ \
1711
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1712
+ __srInnerN = (st).slab_elements; \
1713
+ __srOC = 1; \
1714
+ } \
1715
+ ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1716
+ for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
1717
+ ca_size_t __srDoff = 0, __srMoff = 0; \
1718
+ for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) { \
1719
+ __srDoff += __srIdx[__sk] * (st).slab_strides[__sk]; \
1720
+ __srMoff += __srIdx[__sk] * (st).slab_mask_strides[__sk]; \
1721
+ } \
1722
+ const char *__srQ = (const char *)(p) + __srDoff; \
1723
+ if ( (m) == NULL ) { \
1724
+ if ( __srContig ) { \
1725
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1726
+ double __scm = (MEAN); \
1727
+ int32_t __sc = ( __scm > 65535.0 ) ? 65535 \
1728
+ : ( __scm < -32768.0 ) ? -32768 \
1729
+ : (int32_t) lrint(__scm); \
1730
+ double __sdm = __scm - (double) __sc; \
1731
+ int64_t __sdt = 0; \
1732
+ for ( ca_size_t __sb = 0; __sb < __srInnerN; __sb += 32768 ) { \
1733
+ ca_size_t __se = ( __srInnerN - __sb < 32768 ) \
1734
+ ? __srInnerN : __sb + 32768; \
1735
+ Q_T __sq = 0; \
1736
+ int32_t __sl = 0; \
1737
+ for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
1738
+ int32_t __sd = (int32_t) __srSrc[__sj] - __sc; \
1739
+ __sl += __sd; \
1740
+ __sq += (Q_T) __sd * __sd; \
1741
+ } \
1742
+ __sdt += __sl; \
1743
+ (acc) += (double) __sq; \
1744
+ } \
1745
+ if ( __srInnerN > 0 ) \
1746
+ (acc) += __sdm * ( (double) __srInnerN * __sdm \
1747
+ - 2.0 * (double) __sdt ); \
1748
+ } else { \
1749
+ for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1750
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1751
+ (acc) += EXPR(v); \
1752
+ } \
1753
+ } \
1754
+ } else { \
1755
+ const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1756
+ if ( __srContig && __srMaskU ) { \
1757
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1758
+ for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1759
+ if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) += EXPR(v); } \
1760
+ else { (masked_cnt)++; } \
1761
+ } \
1762
+ } else { \
1763
+ for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1764
+ if ( ! __srMM[__sj * __srInnerMS] ) { \
1765
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1483
1766
  (acc) += EXPR(v); \
1484
1767
  } else { (masked_cnt)++; } \
1485
1768
  } \
@@ -1515,6 +1798,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1515
1798
  * single-accumulator case).
1516
1799
  */
1517
1800
  #define CA_SLAB_REDUCE_MIN8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
1801
+ typedef T __ca_u_T CA_UNALIGNED; \
1518
1802
  (acc) = (INIT); \
1519
1803
  int8_t __srK = (st).slab_ndim; \
1520
1804
  int8_t __srOuterK = __srK - 1; \
@@ -1526,20 +1810,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1526
1810
  ca_size_t __srOC = 1; \
1527
1811
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1528
1812
  __srOC *= (st).slab_dims[__sk]; \
1529
- { \
1530
- int __srFlat = __srContig; \
1531
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1532
- if ( (st).slab_strides[__sk] != \
1533
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1534
- __srFlat = 0; \
1535
- if ( __srFlat && (m) != NULL ) { \
1536
- if ( ! __srMaskU ) __srFlat = 0; \
1537
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1538
- if ( (st).slab_mask_strides[__sk] != \
1539
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1540
- __srFlat = 0; \
1541
- } \
1542
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
1813
+ /* fold all slab axes into one flat loop when the slab is row-major \
1814
+ contiguous (ca_slab_is_flat) */ \
1815
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1816
+ __srInnerN = (st).slab_elements; \
1817
+ __srOC = 1; \
1543
1818
  } \
1544
1819
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1545
1820
  for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
@@ -1551,7 +1826,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1551
1826
  const char *__srQ = (const char *)(p) + __srDoff; \
1552
1827
  if ( (m) == NULL ) { \
1553
1828
  if ( __srContig ) { \
1554
- const T *__srSrc = (const T *) __srQ; \
1829
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1555
1830
  ACC_T __a0=(acc),__a1=(acc),__a2=(acc),__a3=(acc); \
1556
1831
  ACC_T __a4=(acc),__a5=(acc),__a6=(acc),__a7=(acc); \
1557
1832
  ca_size_t __sj = 0, __sN = __srInnerN; \
@@ -1577,14 +1852,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1577
1852
  (acc) = (__c0 < __c1) ? __c0 : __c1; \
1578
1853
  } else { \
1579
1854
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1580
- ACC_T __v = EXPR(*(const T *)(__srQ + __sj * __srInnerS)); \
1855
+ ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
1581
1856
  if (__v < (acc)) (acc) = __v; \
1582
1857
  } \
1583
1858
  } \
1584
1859
  } else { \
1585
1860
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1586
1861
  if ( __srContig && __srMaskU ) { \
1587
- const T *__srSrc = (const T *) __srQ; \
1862
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1588
1863
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1589
1864
  if ( ! __srMM[__sj] ) { \
1590
1865
  ACC_T __v = EXPR(__srSrc[__sj]); \
@@ -1594,7 +1869,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1594
1869
  } else { \
1595
1870
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1596
1871
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1597
- ACC_T __v = EXPR(*(const T *)(__srQ + __sj * __srInnerS)); \
1872
+ ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
1598
1873
  if (__v < (acc)) (acc) = __v; \
1599
1874
  } else { (masked_cnt)++; } \
1600
1875
  } \
@@ -1608,6 +1883,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1608
1883
  } while (0)
1609
1884
 
1610
1885
  #define CA_SLAB_REDUCE_MAX8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
1886
+ typedef T __ca_u_T CA_UNALIGNED; \
1611
1887
  (acc) = (INIT); \
1612
1888
  int8_t __srK = (st).slab_ndim; \
1613
1889
  int8_t __srOuterK = __srK - 1; \
@@ -1619,20 +1895,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1619
1895
  ca_size_t __srOC = 1; \
1620
1896
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1621
1897
  __srOC *= (st).slab_dims[__sk]; \
1622
- { \
1623
- int __srFlat = __srContig; \
1624
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1625
- if ( (st).slab_strides[__sk] != \
1626
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1627
- __srFlat = 0; \
1628
- if ( __srFlat && (m) != NULL ) { \
1629
- if ( ! __srMaskU ) __srFlat = 0; \
1630
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1631
- if ( (st).slab_mask_strides[__sk] != \
1632
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1633
- __srFlat = 0; \
1634
- } \
1635
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
1898
+ /* fold all slab axes into one flat loop when the slab is row-major \
1899
+ contiguous (ca_slab_is_flat) */ \
1900
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1901
+ __srInnerN = (st).slab_elements; \
1902
+ __srOC = 1; \
1636
1903
  } \
1637
1904
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1638
1905
  for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
@@ -1644,7 +1911,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1644
1911
  const char *__srQ = (const char *)(p) + __srDoff; \
1645
1912
  if ( (m) == NULL ) { \
1646
1913
  if ( __srContig ) { \
1647
- const T *__srSrc = (const T *) __srQ; \
1914
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1648
1915
  ACC_T __a0=(acc),__a1=(acc),__a2=(acc),__a3=(acc); \
1649
1916
  ACC_T __a4=(acc),__a5=(acc),__a6=(acc),__a7=(acc); \
1650
1917
  ca_size_t __sj = 0, __sN = __srInnerN; \
@@ -1670,14 +1937,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1670
1937
  (acc) = (__c0 > __c1) ? __c0 : __c1; \
1671
1938
  } else { \
1672
1939
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1673
- ACC_T __v = EXPR(*(const T *)(__srQ + __sj * __srInnerS)); \
1940
+ ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
1674
1941
  if (__v > (acc)) (acc) = __v; \
1675
1942
  } \
1676
1943
  } \
1677
1944
  } else { \
1678
1945
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1679
1946
  if ( __srContig && __srMaskU ) { \
1680
- const T *__srSrc = (const T *) __srQ; \
1947
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1681
1948
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1682
1949
  if ( ! __srMM[__sj] ) { \
1683
1950
  ACC_T __v = EXPR(__srSrc[__sj]); \
@@ -1687,7 +1954,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1687
1954
  } else { \
1688
1955
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1689
1956
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1690
- ACC_T __v = EXPR(*(const T *)(__srQ + __sj * __srInnerS)); \
1957
+ ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
1691
1958
  if (__v > (acc)) (acc) = __v; \
1692
1959
  } else { (masked_cnt)++; } \
1693
1960
  } \
@@ -1701,6 +1968,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1701
1968
  } while (0)
1702
1969
 
1703
1970
  #define CA_SLAB_REDUCE_STAR8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
1971
+ typedef T __ca_u_T CA_UNALIGNED; \
1704
1972
  (acc) = (INIT); \
1705
1973
  int8_t __srK = (st).slab_ndim; \
1706
1974
  int8_t __srOuterK = __srK - 1; \
@@ -1712,20 +1980,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1712
1980
  ca_size_t __srOC = 1; \
1713
1981
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1714
1982
  __srOC *= (st).slab_dims[__sk]; \
1715
- { \
1716
- int __srFlat = __srContig; \
1717
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1718
- if ( (st).slab_strides[__sk] != \
1719
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1720
- __srFlat = 0; \
1721
- if ( __srFlat && (m) != NULL ) { \
1722
- if ( ! __srMaskU ) __srFlat = 0; \
1723
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1724
- if ( (st).slab_mask_strides[__sk] != \
1725
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1726
- __srFlat = 0; \
1727
- } \
1728
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
1983
+ /* fold all slab axes into one flat loop when the slab is row-major \
1984
+ contiguous (ca_slab_is_flat) */ \
1985
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
1986
+ __srInnerN = (st).slab_elements; \
1987
+ __srOC = 1; \
1729
1988
  } \
1730
1989
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1731
1990
  for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
@@ -1737,7 +1996,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1737
1996
  const char *__srQ = (const char *)(p) + __srDoff; \
1738
1997
  if ( (m) == NULL ) { \
1739
1998
  if ( __srContig ) { \
1740
- const T *__srSrc = (const T *) __srQ; \
1999
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1741
2000
  ACC_T __a0=(ACC_T)1,__a1=(ACC_T)1,__a2=(ACC_T)1,__a3=(ACC_T)1; \
1742
2001
  ACC_T __a4=(ACC_T)1,__a5=(ACC_T)1,__a6=(ACC_T)1,__a7=(ACC_T)1; \
1743
2002
  ca_size_t __sj = 0, __sN = __srInnerN; \
@@ -1756,14 +2015,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1756
2015
  * ((__a4 * __a5) * (__a6 * __a7)); \
1757
2016
  } else { \
1758
2017
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1759
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2018
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1760
2019
  (acc) *= EXPR(v); \
1761
2020
  } \
1762
2021
  } \
1763
2022
  } else { \
1764
2023
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1765
2024
  if ( __srContig && __srMaskU ) { \
1766
- const T *__srSrc = (const T *) __srQ; \
2025
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1767
2026
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1768
2027
  if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) *= EXPR(v); } \
1769
2028
  else { (masked_cnt)++; } \
@@ -1771,7 +2030,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1771
2030
  } else { \
1772
2031
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1773
2032
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1774
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2033
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1775
2034
  (acc) *= EXPR(v); \
1776
2035
  } else { (masked_cnt)++; } \
1777
2036
  } \
@@ -1797,6 +2056,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1797
2056
  * (T_LIMIT_HI), matching the pre-SIMD path byte-identically.
1798
2057
  */
1799
2058
  #define CA_SLAB_REDUCE_T_MIN_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
2059
+ typedef T __ca_u_T CA_UNALIGNED; \
1800
2060
  (acc) = (INIT); \
1801
2061
  int8_t __srK = (st).slab_ndim; \
1802
2062
  int8_t __srOuterK = __srK - 1; \
@@ -1808,27 +2068,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1808
2068
  ca_size_t __srOC = 1; \
1809
2069
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1810
2070
  __srOC *= (st).slab_dims[__sk]; \
1811
- /* Slab-collapse: when the whole K-D slab is row-major contiguous \
1812
- (data, and mask if present), fold all slab axes into one flat \
1813
- inner loop. Removes the per-outer multi-index offset recompute + \
1814
- carry that otherwise dominates when the innermost slab axis is \
1815
- small (full reduction of [N,1] / [N,small] entity, or trailing- \
1816
- contig multi-axis reduce). Platform-general: structural, not SIMD. \
1817
- No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
1818
- { \
1819
- int __srFlat = __srContig; \
1820
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1821
- if ( (st).slab_strides[__sk] != \
1822
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1823
- __srFlat = 0; \
1824
- if ( __srFlat && (m) != NULL ) { \
1825
- if ( ! __srMaskU ) __srFlat = 0; \
1826
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1827
- if ( (st).slab_mask_strides[__sk] != \
1828
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1829
- __srFlat = 0; \
1830
- } \
1831
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
2071
+ /* fold all slab axes into one flat loop when the slab is row-major \
2072
+ contiguous (ca_slab_is_flat) */ \
2073
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
2074
+ __srInnerN = (st).slab_elements; \
2075
+ __srOC = 1; \
1832
2076
  } \
1833
2077
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1834
2078
  ca_size_t idx = 0; \
@@ -1841,7 +2085,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1841
2085
  const char *__srQ = (const char *)(p) + __srDoff; \
1842
2086
  if ( (m) == NULL ) { \
1843
2087
  if ( __srContig ) { \
1844
- const T *__srSrc = (const T *) __srQ; \
2088
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1845
2089
  _CA_SIMD_MIN(acc) \
1846
2090
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1847
2091
  T v = __srSrc[__sj]; \
@@ -1850,7 +2094,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1850
2094
  } \
1851
2095
  } else { \
1852
2096
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1853
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2097
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1854
2098
  REDUCE; \
1855
2099
  idx++; \
1856
2100
  } \
@@ -1858,7 +2102,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1858
2102
  } else { \
1859
2103
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1860
2104
  if ( __srContig && __srMaskU ) { \
1861
- const T *__srSrc = (const T *) __srQ; \
2105
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1862
2106
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1863
2107
  if ( ! __srMM[__sj] ) { \
1864
2108
  T v = __srSrc[__sj]; \
@@ -1871,7 +2115,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1871
2115
  } else { \
1872
2116
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1873
2117
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1874
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2118
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1875
2119
  REDUCE; \
1876
2120
  } else { \
1877
2121
  (masked_cnt)++; \
@@ -1890,6 +2134,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1890
2134
 
1891
2135
  /* CA_SLAB_REDUCE_T_MAX_EX (generator-only): same as MIN_EX with `_CA_SIMD_MAX`. */
1892
2136
  #define CA_SLAB_REDUCE_T_MAX_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
2137
+ typedef T __ca_u_T CA_UNALIGNED; \
1893
2138
  (acc) = (INIT); \
1894
2139
  int8_t __srK = (st).slab_ndim; \
1895
2140
  int8_t __srOuterK = __srK - 1; \
@@ -1901,27 +2146,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1901
2146
  ca_size_t __srOC = 1; \
1902
2147
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
1903
2148
  __srOC *= (st).slab_dims[__sk]; \
1904
- /* Slab-collapse: when the whole K-D slab is row-major contiguous \
1905
- (data, and mask if present), fold all slab axes into one flat \
1906
- inner loop. Removes the per-outer multi-index offset recompute + \
1907
- carry that otherwise dominates when the innermost slab axis is \
1908
- small (full reduction of [N,1] / [N,small] entity, or trailing- \
1909
- contig multi-axis reduce). Platform-general: structural, not SIMD. \
1910
- No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
1911
- { \
1912
- int __srFlat = __srContig; \
1913
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1914
- if ( (st).slab_strides[__sk] != \
1915
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
1916
- __srFlat = 0; \
1917
- if ( __srFlat && (m) != NULL ) { \
1918
- if ( ! __srMaskU ) __srFlat = 0; \
1919
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
1920
- if ( (st).slab_mask_strides[__sk] != \
1921
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
1922
- __srFlat = 0; \
1923
- } \
1924
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
2149
+ /* fold all slab axes into one flat loop when the slab is row-major \
2150
+ contiguous (ca_slab_is_flat) */ \
2151
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
2152
+ __srInnerN = (st).slab_elements; \
2153
+ __srOC = 1; \
1925
2154
  } \
1926
2155
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
1927
2156
  ca_size_t idx = 0; \
@@ -1934,7 +2163,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1934
2163
  const char *__srQ = (const char *)(p) + __srDoff; \
1935
2164
  if ( (m) == NULL ) { \
1936
2165
  if ( __srContig ) { \
1937
- const T *__srSrc = (const T *) __srQ; \
2166
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1938
2167
  _CA_SIMD_MAX(acc) \
1939
2168
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1940
2169
  T v = __srSrc[__sj]; \
@@ -1943,7 +2172,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1943
2172
  } \
1944
2173
  } else { \
1945
2174
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1946
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2175
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1947
2176
  REDUCE; \
1948
2177
  idx++; \
1949
2178
  } \
@@ -1951,7 +2180,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1951
2180
  } else { \
1952
2181
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
1953
2182
  if ( __srContig && __srMaskU ) { \
1954
- const T *__srSrc = (const T *) __srQ; \
2183
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
1955
2184
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1956
2185
  if ( ! __srMM[__sj] ) { \
1957
2186
  T v = __srSrc[__sj]; \
@@ -1964,7 +2193,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1964
2193
  } else { \
1965
2194
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
1966
2195
  if ( ! __srMM[__sj * __srInnerMS] ) { \
1967
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2196
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
1968
2197
  REDUCE; \
1969
2198
  } else { \
1970
2199
  (masked_cnt)++; \
@@ -1990,6 +2219,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
1990
2219
  * integers).
1991
2220
  */
1992
2221
  #define CA_SLAB_REDUCE_T_STAR_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
2222
+ typedef T __ca_u_T CA_UNALIGNED; \
1993
2223
  (acc) = (INIT); \
1994
2224
  int8_t __srK = (st).slab_ndim; \
1995
2225
  int8_t __srOuterK = __srK - 1; \
@@ -2001,27 +2231,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2001
2231
  ca_size_t __srOC = 1; \
2002
2232
  for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
2003
2233
  __srOC *= (st).slab_dims[__sk]; \
2004
- /* Slab-collapse: when the whole K-D slab is row-major contiguous \
2005
- (data, and mask if present), fold all slab axes into one flat \
2006
- inner loop. Removes the per-outer multi-index offset recompute + \
2007
- carry that otherwise dominates when the innermost slab axis is \
2008
- small (full reduction of [N,1] / [N,small] entity, or trailing- \
2009
- contig multi-axis reduce). Platform-general: structural, not SIMD. \
2010
- No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
2011
- { \
2012
- int __srFlat = __srContig; \
2013
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
2014
- if ( (st).slab_strides[__sk] != \
2015
- (st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
2016
- __srFlat = 0; \
2017
- if ( __srFlat && (m) != NULL ) { \
2018
- if ( ! __srMaskU ) __srFlat = 0; \
2019
- for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
2020
- if ( (st).slab_mask_strides[__sk] != \
2021
- (st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
2022
- __srFlat = 0; \
2023
- } \
2024
- if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
2234
+ /* fold all slab axes into one flat loop when the slab is row-major \
2235
+ contiguous (ca_slab_is_flat) */ \
2236
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
2237
+ __srInnerN = (st).slab_elements; \
2238
+ __srOC = 1; \
2025
2239
  } \
2026
2240
  ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
2027
2241
  ca_size_t idx = 0; \
@@ -2034,7 +2248,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2034
2248
  const char *__srQ = (const char *)(p) + __srDoff; \
2035
2249
  if ( (m) == NULL ) { \
2036
2250
  if ( __srContig ) { \
2037
- const T *__srSrc = (const T *) __srQ; \
2251
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
2038
2252
  _CA_SIMD_STAR(acc) \
2039
2253
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
2040
2254
  T v = __srSrc[__sj]; \
@@ -2043,7 +2257,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2043
2257
  } \
2044
2258
  } else { \
2045
2259
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
2046
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2260
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
2047
2261
  REDUCE; \
2048
2262
  idx++; \
2049
2263
  } \
@@ -2051,7 +2265,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2051
2265
  } else { \
2052
2266
  const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
2053
2267
  if ( __srContig && __srMaskU ) { \
2054
- const T *__srSrc = (const T *) __srQ; \
2268
+ const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
2055
2269
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
2056
2270
  if ( ! __srMM[__sj] ) { \
2057
2271
  T v = __srSrc[__sj]; \
@@ -2064,7 +2278,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2064
2278
  } else { \
2065
2279
  for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
2066
2280
  if ( ! __srMM[__sj * __srInnerMS] ) { \
2067
- T v = *(const T *)(__srQ + __sj * __srInnerS); \
2281
+ T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
2068
2282
  REDUCE; \
2069
2283
  } else { \
2070
2284
  (masked_cnt)++; \
@@ -2129,6 +2343,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2129
2343
  The mask `m` applies to the SOURCE only (weights mask is overlaid onto
2130
2344
  source mask by the dispatcher). */
2131
2345
  #define CA_SLAB_REDUCE_ARRAY_T_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, REDUCE, masked_cnt) do { \
2346
+ typedef T_W __ca_u_T_W CA_UNALIGNED; \
2347
+ typedef T __ca_u_T CA_UNALIGNED; \
2132
2348
  (acc) = (INIT); \
2133
2349
  int8_t __sraK = (st).slab_ndim; \
2134
2350
  int8_t __sraOuterK = __sraK - 1; \
@@ -2142,32 +2358,12 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2142
2358
  ca_size_t __sraOC = 1; \
2143
2359
  for ( int8_t __sak = 0; __sak < __sraOuterK; __sak++ ) \
2144
2360
  __sraOC *= (st).slab_dims[__sak]; \
2145
- /* Slab-collapse: fold all slab axes into one flat inner loop when the \
2146
- whole K-D slab is row-major contiguous for BOTH source and weights \
2147
- (and mask if present). Same structural win as the unweighted \
2148
- CA_SLAB_REDUCE_T_*_EX macros: kills the per-outer multi-index offset \
2149
- recompute + carry that dominates when the innermost slab axis is \
2150
- small. No-op for 1-D slabs (__sraOuterK == 0 leaves __sraOC == 1). \
2151
- Weights share slab_dims with source (shape invariant), so the dim \
2152
- cascade uses (st).slab_dims for both stride tables. */ \
2153
- { \
2154
- int __sraFlat = __sraContig && __sraContigW; \
2155
- for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) { \
2156
- if ( (st).slab_strides[__sak] != \
2157
- (st).slab_dims[__sak + 1] * (st).slab_strides[__sak + 1] ) \
2158
- __sraFlat = 0; \
2159
- if ( (st_w).slab_strides[__sak] != \
2160
- (st).slab_dims[__sak + 1] * (st_w).slab_strides[__sak + 1] ) \
2161
- __sraFlat = 0; \
2162
- } \
2163
- if ( __sraFlat && (m) != NULL ) { \
2164
- if ( ! __sraMaskU ) __sraFlat = 0; \
2165
- for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) \
2166
- if ( (st).slab_mask_strides[__sak] != \
2167
- (st).slab_dims[__sak + 1] * (st).slab_mask_strides[__sak + 1] ) \
2168
- __sraFlat = 0; \
2169
- } \
2170
- if ( __sraFlat ) { __sraInnerN = (st).slab_elements; __sraOC = 1; } \
2361
+ /* fold all slab axes into one flat loop when the slab is row-major \
2362
+ contiguous (ca_slab_is_flat) */ \
2363
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) \
2364
+ && ca_slab_strides_are_flat(&(st), (st_w).slab_strides, sizeof(T_W)) ) { \
2365
+ __sraInnerN = (st).slab_elements; \
2366
+ __sraOC = 1; \
2171
2367
  } \
2172
2368
  ca_size_t __sraIdx[CA_RANK_MAX] = { 0 }; \
2173
2369
  ca_size_t idx = 0; \
@@ -2182,8 +2378,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2182
2378
  const char *__sraWQ = (const char *)(p_w) + __sraWoff; \
2183
2379
  if ( (m) == NULL ) { \
2184
2380
  if ( __sraContig && __sraContigW ) { \
2185
- const T *__sraSrc = (const T *) __sraQ; \
2186
- const T_W *__sraWgt = (const T_W *) __sraWQ; \
2381
+ const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
2382
+ const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
2187
2383
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2188
2384
  T v = __sraSrc[__saj]; \
2189
2385
  T_W w = __sraWgt[__saj]; \
@@ -2192,8 +2388,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2192
2388
  } \
2193
2389
  } else { \
2194
2390
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2195
- T v = *(const T *)(__sraQ + __saj * __sraInnerS); \
2196
- T_W w = *(const T_W *)(__sraWQ + __saj * __sraInnerWS); \
2391
+ T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
2392
+ T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
2197
2393
  REDUCE; \
2198
2394
  idx++; \
2199
2395
  } \
@@ -2201,8 +2397,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2201
2397
  } else { \
2202
2398
  const boolean8_t *__sraMM = (const boolean8_t *)(m) + __sraMoff; \
2203
2399
  if ( __sraContig && __sraContigW && __sraMaskU ) { \
2204
- const T *__sraSrc = (const T *) __sraQ; \
2205
- const T_W *__sraWgt = (const T_W *) __sraWQ; \
2400
+ const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
2401
+ const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
2206
2402
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2207
2403
  if ( ! __sraMM[__saj] ) { \
2208
2404
  T v = __sraSrc[__saj]; \
@@ -2216,8 +2412,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2216
2412
  } else { \
2217
2413
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2218
2414
  if ( ! __sraMM[__saj * __sraInnerMS] ) { \
2219
- T v = *(const T *)(__sraQ + __saj * __sraInnerS); \
2220
- T_W w = *(const T_W *)(__sraWQ + __saj * __sraInnerWS); \
2415
+ T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
2416
+ T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
2221
2417
  REDUCE; \
2222
2418
  } else { \
2223
2419
  (masked_cnt)++; \
@@ -2249,6 +2445,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2249
2445
  */
2250
2446
  #define CA_SLAB_REDUCE_ARRAY_T_PLUS_EX(T, T_W, st, p, m, st_w, p_w, acc, \
2251
2447
  INIT, REDUCE, masked_cnt) do { \
2448
+ typedef T_W __ca_u_T_W CA_UNALIGNED; \
2449
+ typedef T __ca_u_T CA_UNALIGNED; \
2252
2450
  (acc) = (INIT); \
2253
2451
  int8_t __sraK = (st).slab_ndim; \
2254
2452
  int8_t __sraOuterK = __sraK - 1; \
@@ -2262,32 +2460,12 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2262
2460
  ca_size_t __sraOC = 1; \
2263
2461
  for ( int8_t __sak = 0; __sak < __sraOuterK; __sak++ ) \
2264
2462
  __sraOC *= (st).slab_dims[__sak]; \
2265
- /* Slab-collapse: fold all slab axes into one flat inner loop when the \
2266
- whole K-D slab is row-major contiguous for BOTH source and weights \
2267
- (and mask if present). Same structural win as the unweighted \
2268
- CA_SLAB_REDUCE_T_*_EX macros: kills the per-outer multi-index offset \
2269
- recompute + carry that dominates when the innermost slab axis is \
2270
- small. No-op for 1-D slabs (__sraOuterK == 0 leaves __sraOC == 1). \
2271
- Weights share slab_dims with source (shape invariant), so the dim \
2272
- cascade uses (st).slab_dims for both stride tables. */ \
2273
- { \
2274
- int __sraFlat = __sraContig && __sraContigW; \
2275
- for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) { \
2276
- if ( (st).slab_strides[__sak] != \
2277
- (st).slab_dims[__sak + 1] * (st).slab_strides[__sak + 1] ) \
2278
- __sraFlat = 0; \
2279
- if ( (st_w).slab_strides[__sak] != \
2280
- (st).slab_dims[__sak + 1] * (st_w).slab_strides[__sak + 1] ) \
2281
- __sraFlat = 0; \
2282
- } \
2283
- if ( __sraFlat && (m) != NULL ) { \
2284
- if ( ! __sraMaskU ) __sraFlat = 0; \
2285
- for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) \
2286
- if ( (st).slab_mask_strides[__sak] != \
2287
- (st).slab_dims[__sak + 1] * (st).slab_mask_strides[__sak + 1] ) \
2288
- __sraFlat = 0; \
2289
- } \
2290
- if ( __sraFlat ) { __sraInnerN = (st).slab_elements; __sraOC = 1; } \
2463
+ /* fold all slab axes into one flat loop when the slab is row-major \
2464
+ contiguous (ca_slab_is_flat) */ \
2465
+ if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) \
2466
+ && ca_slab_strides_are_flat(&(st), (st_w).slab_strides, sizeof(T_W)) ) { \
2467
+ __sraInnerN = (st).slab_elements; \
2468
+ __sraOC = 1; \
2291
2469
  } \
2292
2470
  ca_size_t __sraIdx[CA_RANK_MAX] = { 0 }; \
2293
2471
  ca_size_t idx = 0; \
@@ -2302,8 +2480,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2302
2480
  const char *__sraWQ = (const char *)(p_w) + __sraWoff; \
2303
2481
  if ( (m) == NULL ) { \
2304
2482
  if ( __sraContig && __sraContigW ) { \
2305
- const T *__sraSrc = (const T *) __sraQ; \
2306
- const T_W *__sraWgt = (const T_W *) __sraWQ; \
2483
+ const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
2484
+ const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
2307
2485
  _CA_SIMD_PLUS(acc) \
2308
2486
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2309
2487
  T v = __sraSrc[__saj]; \
@@ -2313,8 +2491,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2313
2491
  } \
2314
2492
  } else { \
2315
2493
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2316
- T v = *(const T *)(__sraQ + __saj * __sraInnerS); \
2317
- T_W w = *(const T_W *)(__sraWQ + __saj * __sraInnerWS); \
2494
+ T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
2495
+ T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
2318
2496
  REDUCE; \
2319
2497
  idx++; \
2320
2498
  } \
@@ -2322,8 +2500,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2322
2500
  } else { \
2323
2501
  const boolean8_t *__sraMM = (const boolean8_t *)(m) + __sraMoff; \
2324
2502
  if ( __sraContig && __sraContigW && __sraMaskU ) { \
2325
- const T *__sraSrc = (const T *) __sraQ; \
2326
- const T_W *__sraWgt = (const T_W *) __sraWQ; \
2503
+ const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
2504
+ const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
2327
2505
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2328
2506
  if ( ! __sraMM[__saj] ) { \
2329
2507
  T v = __sraSrc[__saj]; \
@@ -2337,8 +2515,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2337
2515
  } else { \
2338
2516
  for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
2339
2517
  if ( ! __sraMM[__saj * __sraInnerMS] ) { \
2340
- T v = *(const T *)(__sraQ + __saj * __sraInnerS); \
2341
- T_W w = *(const T_W *)(__sraWQ + __saj * __sraInnerWS); \
2518
+ T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
2519
+ T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
2342
2520
  REDUCE; \
2343
2521
  } else { \
2344
2522
  (masked_cnt)++; \
@@ -2378,6 +2556,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2378
2556
  docs/authoring/HOW_TO_WRITE_KERNEL.md. Caller is responsible for invoking
2379
2557
  ca_iter_state_sync_slab on the output state after each slab. */
2380
2558
  #define CA_SLAB_MAP_T(T_IN, T_OUT, st_in, p_in, st_out, p_out, MAP_EXPR) do { \
2559
+ typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
2560
+ typedef T_IN __ca_u_T_IN CA_UNALIGNED; \
2381
2561
  int8_t __mK = (st_in).slab_ndim; \
2382
2562
  int8_t __mOuterK = __mK - 1; \
2383
2563
  ca_size_t __mInN = (st_in).slab_dims[__mK - 1]; \
@@ -2398,8 +2578,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2398
2578
  const char *__mQi = (const char *)(p_in) + __mIOff; \
2399
2579
  char *__mQo = (char *) (p_out) + __mOOff; \
2400
2580
  if ( __mIContig && __mOContig ) { \
2401
- const T_IN *__mSi = (const T_IN *) __mQi; \
2402
- T_OUT *__mSo = (T_OUT *) __mQo; \
2581
+ const __ca_u_T_IN *__mSi = (const __ca_u_T_IN *) __mQi; \
2582
+ __ca_u_T_OUT *__mSo = (__ca_u_T_OUT *) __mQo; \
2403
2583
  for ( ca_size_t __mj = 0; __mj < __mInN; __mj++ ) { \
2404
2584
  T_IN v = __mSi[__mj]; \
2405
2585
  T_OUT r; \
@@ -2408,10 +2588,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2408
2588
  } \
2409
2589
  } else { \
2410
2590
  for ( ca_size_t __mj = 0; __mj < __mInN; __mj++ ) { \
2411
- T_IN v = *(const T_IN *)(__mQi + __mj * __mInS); \
2591
+ T_IN v = *(const __ca_u_T_IN *)(__mQi + __mj * __mInS); \
2412
2592
  T_OUT r; \
2413
2593
  MAP_EXPR; \
2414
- *(T_OUT *)(__mQo + __mj * __mOutS) = r; \
2594
+ *(__ca_u_T_OUT *)(__mQo + __mj * __mOutS) = r; \
2415
2595
  } \
2416
2596
  } \
2417
2597
  for ( int8_t __mk = __mOuterK - 1; __mk >= 0; __mk-- ) { \
@@ -2460,6 +2640,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2460
2640
  CA_SLAB_REDUCE_T / CA_SLAB_MAP_T. */
2461
2641
  #define CA_SLAB_SCAN_T(T_LOAD, T_OUT, st_in, p_in, m_in, \
2462
2642
  st_out, p_out, INIT, STEP) do { \
2643
+ typedef T_LOAD __ca_u_T_LOAD CA_UNALIGNED; \
2644
+ typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
2463
2645
  T_OUT acc = (INIT); \
2464
2646
  int8_t __ssK = (st_in).slab_ndim; \
2465
2647
  int8_t __ssOuterK = __ssK - 1; \
@@ -2485,8 +2667,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2485
2667
  char *__ssQo = (char *) (p_out) + __ssOOff; \
2486
2668
  if ( (m_in) == NULL ) { \
2487
2669
  if ( __ssIContig && __ssOContig ) { \
2488
- const T_LOAD *__ssSi = (const T_LOAD *) __ssQi; \
2489
- T_OUT *__ssSo = (T_OUT *) __ssQo; \
2670
+ const __ca_u_T_LOAD *__ssSi = (const __ca_u_T_LOAD *) __ssQi; \
2671
+ __ca_u_T_OUT *__ssSo = (__ca_u_T_OUT *) __ssQo; \
2490
2672
  for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
2491
2673
  T_LOAD v = __ssSi[__sj]; \
2492
2674
  T_OUT r; \
@@ -2495,10 +2677,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2495
2677
  } \
2496
2678
  } else { \
2497
2679
  for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
2498
- T_LOAD v = *(const T_LOAD *)(__ssQi + __sj * __ssInS); \
2680
+ T_LOAD v = *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
2499
2681
  T_OUT r; \
2500
2682
  STEP; \
2501
- *(T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2683
+ *(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2502
2684
  } \
2503
2685
  } \
2504
2686
  } else { \
@@ -2507,16 +2689,16 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2507
2689
  T_OUT r; \
2508
2690
  if ( ! __ssMM[__ssMaskU ? __sj : __sj * __ssInMS] ) { \
2509
2691
  T_LOAD v = __ssIContig \
2510
- ? ((const T_LOAD *) __ssQi)[__sj] \
2511
- : *(const T_LOAD *)(__ssQi + __sj * __ssInS); \
2692
+ ? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
2693
+ : *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
2512
2694
  STEP; \
2513
2695
  } else { \
2514
2696
  r = acc; /* masked: write current running aggregate (legacy parity) */ \
2515
2697
  } \
2516
2698
  if ( __ssOContig ) \
2517
- ((T_OUT *) __ssQo)[__sj] = r; \
2699
+ ((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
2518
2700
  else \
2519
- *(T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2701
+ *(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2520
2702
  } \
2521
2703
  } \
2522
2704
  for ( int8_t __sk = __ssOuterK - 1; __sk >= 0; __sk-- ) { \
@@ -2553,6 +2735,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2553
2735
  (defensive: falls back to holding the sentinel unmasked). */
2554
2736
  #define CA_SLAB_SCAN_T_GATED(T_LOAD, T_OUT, st_in, p_in, m_in, \
2555
2737
  st_out, p_out, m_out, INIT, STEP) do { \
2738
+ typedef T_LOAD __ca_u_T_LOAD CA_UNALIGNED; \
2739
+ typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
2556
2740
  T_OUT acc = (INIT); \
2557
2741
  int8_t __ssK = (st_in).slab_ndim; \
2558
2742
  int8_t __ssOuterK = __ssK - 1; \
@@ -2581,8 +2765,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2581
2765
  int __ssSeen = 0; \
2582
2766
  if ( (m_in) == NULL ) { \
2583
2767
  if ( __ssIContig && __ssOContig ) { \
2584
- const T_LOAD *__ssSi = (const T_LOAD *) __ssQi; \
2585
- T_OUT *__ssSo = (T_OUT *) __ssQo; \
2768
+ const __ca_u_T_LOAD *__ssSi = (const __ca_u_T_LOAD *) __ssQi; \
2769
+ __ca_u_T_OUT *__ssSo = (__ca_u_T_OUT *) __ssQo; \
2586
2770
  for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
2587
2771
  T_LOAD v = __ssSi[__sj]; \
2588
2772
  T_OUT r; \
@@ -2591,10 +2775,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2591
2775
  } \
2592
2776
  } else { \
2593
2777
  for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
2594
- T_LOAD v = *(const T_LOAD *)(__ssQi + __sj * __ssInS); \
2778
+ T_LOAD v = *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
2595
2779
  T_OUT r; \
2596
2780
  STEP; \
2597
- *(T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2781
+ *(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2598
2782
  } \
2599
2783
  } \
2600
2784
  } else { \
@@ -2603,8 +2787,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2603
2787
  T_OUT r; \
2604
2788
  if ( ! __ssMM[__ssMaskU ? __sj : __sj * __ssInMS] ) { \
2605
2789
  T_LOAD v = __ssIContig \
2606
- ? ((const T_LOAD *) __ssQi)[__sj] \
2607
- : *(const T_LOAD *)(__ssQi + __sj * __ssInS); \
2790
+ ? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
2791
+ : *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
2608
2792
  STEP; \
2609
2793
  __ssSeen = 1; \
2610
2794
  } else if ( ! __ssSeen && (m_out) != NULL ) { \
@@ -2614,9 +2798,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2614
2798
  r = acc; /* masked after first present: hold running extremum */\
2615
2799
  } \
2616
2800
  if ( __ssOContig ) \
2617
- ((T_OUT *) __ssQo)[__sj] = r; \
2801
+ ((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
2618
2802
  else \
2619
- *(T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2803
+ *(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2620
2804
  } \
2621
2805
  } \
2622
2806
  for ( int8_t __sk = __ssOuterK - 1; __sk >= 0; __sk-- ) { \
@@ -2646,6 +2830,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2646
2830
  "last live value" trail. */
2647
2831
  #define CA_SLAB_SCAN_TA(T_LOAD, T_OUT, T_ACC, st_in, p_in, m_in, \
2648
2832
  st_out, p_out, INIT, STEP) do { \
2833
+ typedef T_LOAD __ca_u_T_LOAD CA_UNALIGNED; \
2834
+ typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
2649
2835
  T_ACC acc = (INIT); \
2650
2836
  int8_t __ssK = (st_in).slab_ndim; \
2651
2837
  int8_t __ssOuterK = __ssK - 1; \
@@ -2673,16 +2859,16 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2673
2859
  if ( (m_in) == NULL ) { \
2674
2860
  for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
2675
2861
  T_LOAD v = __ssIContig \
2676
- ? ((const T_LOAD *) __ssQi)[__sj] \
2677
- : *(const T_LOAD *)(__ssQi + __sj * __ssInS); \
2862
+ ? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
2863
+ : *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
2678
2864
  T_OUT r; \
2679
2865
  int first = __ssFirst; \
2680
2866
  STEP; \
2681
2867
  __ssFirst = 0; \
2682
2868
  if ( __ssOContig ) \
2683
- ((T_OUT *) __ssQo)[__sj] = r; \
2869
+ ((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
2684
2870
  else \
2685
- *(T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2871
+ *(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2686
2872
  } \
2687
2873
  } else { \
2688
2874
  const boolean8_t *__ssMM = (const boolean8_t *)(m_in) + __ssMoff; \
@@ -2690,8 +2876,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2690
2876
  T_OUT r; \
2691
2877
  if ( ! __ssMM[__ssMaskU ? __sj : __sj * __ssInMS] ) { \
2692
2878
  T_LOAD v = __ssIContig \
2693
- ? ((const T_LOAD *) __ssQi)[__sj] \
2694
- : *(const T_LOAD *)(__ssQi + __sj * __ssInS); \
2879
+ ? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
2880
+ : *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
2695
2881
  int first = __ssFirst; \
2696
2882
  STEP; \
2697
2883
  __ssFirst = 0; \
@@ -2699,9 +2885,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2699
2885
  r = (T_OUT) 0; /* masked: neutral output for downstream OR */ \
2700
2886
  } \
2701
2887
  if ( __ssOContig ) \
2702
- ((T_OUT *) __ssQo)[__sj] = r; \
2888
+ ((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
2703
2889
  else \
2704
- *(T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2890
+ *(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
2705
2891
  } \
2706
2892
  } \
2707
2893
  for ( int8_t __sk = __ssOuterK - 1; __sk >= 0; __sk-- ) { \
@@ -2712,10 +2898,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
2712
2898
  } while (0)
2713
2899
 
2714
2900
  /* ---- Ruby surface ---------------------------------------------------- */
2715
- /* Called from Init_carray_ext (ruby_carray.c). Creates the
2716
- Ractor-local key for the kept scratch buffer, and in a development
2717
- build (CARRAY_DEV_BUILD) registers the CArray.t1_* smoke entry points
2718
- the spec_ai kernel iterator tests use. */
2901
+ /* Called from Init_carray_ext (ruby_carray.c). In a development build
2902
+ (CARRAY_DEV_BUILD) it registers the CArray.t1_* smoke entry points the
2903
+ spec_ai kernel iterator tests use; in a release build it does nothing. */
2719
2904
  void Init_ca_kernel_iterator (void);
2720
2905
 
2721
2906
  #endif /* CA_KERNEL_ITERATOR_H */