carray 3.0.3 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +264 -0
  3. data/README.md +3 -2
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_descriptor.h +27 -24
  6. data/ext/ca_axis_dispatch.c +1 -6
  7. data/ext/ca_axis_group.c +27 -27
  8. data/ext/ca_binop_dispatch.c +1 -29
  9. data/ext/ca_binop_dispatch.h +0 -1
  10. data/ext/ca_categorical_iterator.c +205 -134
  11. data/ext/ca_compare.c +7 -18
  12. data/ext/ca_group_iter.c +16 -17
  13. data/ext/ca_kernel_iterator.c +1234 -1994
  14. data/ext/ca_kernel_iterator.h +391 -437
  15. data/ext/ca_monop_dispatch.c +82 -59
  16. data/ext/ca_monop_dispatch.h +1 -0
  17. data/ext/ca_obj_array.c +96 -39
  18. data/ext/ca_obj_bincmp.c +24 -21
  19. data/ext/ca_obj_binop.c +15 -26
  20. data/ext/ca_obj_bitarray.c +3 -2
  21. data/ext/ca_obj_bitfield.c +5 -6
  22. data/ext/ca_obj_block.c +18 -9
  23. data/ext/ca_obj_byte_swap.c +5 -6
  24. data/ext/ca_obj_const_string.c +173 -19
  25. data/ext/ca_obj_face.c +238 -57
  26. data/ext/ca_obj_face.h +19 -0
  27. data/ext/ca_obj_fake.c +15 -29
  28. data/ext/ca_obj_farray.c +6 -3
  29. data/ext/ca_obj_field.c +17 -5
  30. data/ext/ca_obj_fixlen_string.c +2 -3
  31. data/ext/ca_obj_grid.c +406 -132
  32. data/ext/ca_obj_meld.c +172 -34
  33. data/ext/ca_obj_moncmp.c +23 -8
  34. data/ext/ca_obj_monop.c +67 -57
  35. data/ext/ca_obj_object.c +157 -51
  36. data/ext/ca_obj_record.c +12 -7
  37. data/ext/ca_obj_reduce.c +13 -8
  38. data/ext/ca_obj_refer.c +25 -27
  39. data/ext/ca_obj_remap.c +30 -9
  40. data/ext/ca_obj_repeat.c +9 -10
  41. data/ext/ca_obj_roll.c +6 -5
  42. data/ext/ca_obj_select.c +14 -9
  43. data/ext/ca_obj_select_axis.c +15 -47
  44. data/ext/ca_obj_shift.c +11 -1
  45. data/ext/ca_obj_stack.c +179 -34
  46. data/ext/ca_obj_stride.c +162 -115
  47. data/ext/ca_obj_string.c +71 -4
  48. data/ext/ca_obj_tile.c +6 -5
  49. data/ext/ca_obj_time.c +22 -6
  50. data/ext/ca_obj_timedelta.c +22 -6
  51. data/ext/ca_obj_transpose.c +5 -3
  52. data/ext/ca_obj_triop.c +23 -32
  53. data/ext/ca_obj_window.c +190 -26
  54. data/ext/ca_sweep_engine.c +2 -2
  55. data/ext/caframe_csv_reader.c +80 -1
  56. data/ext/carray.h +181 -75
  57. data/ext/carray_access.c +184 -96
  58. data/ext/carray_address_basis.c +343 -87
  59. data/ext/carray_attribute.c +89 -6
  60. data/ext/carray_bincount.c +10 -8
  61. data/ext/carray_broadcast.c +13 -8
  62. data/ext/carray_build_flags.h +3 -0
  63. data/ext/carray_call_cfunc.c +108 -0
  64. data/ext/carray_cast.c +142 -24
  65. data/ext/carray_cast_func.rb +46 -6
  66. data/ext/carray_class.c +4 -3
  67. data/ext/carray_config.h +28 -0
  68. data/ext/carray_conversion.c +16 -7
  69. data/ext/carray_copy.c +51 -14
  70. data/ext/carray_core.c +147 -55
  71. data/ext/carray_count.c +50 -20
  72. data/ext/carray_element.c +76 -33
  73. data/ext/carray_factorize.c +118 -64
  74. data/ext/carray_generate.c +191 -50
  75. data/ext/carray_histogram.c +62 -17
  76. data/ext/carray_hold.c +12 -15
  77. data/ext/carray_index_classifier.c +388 -176
  78. data/ext/carray_internal.h +336 -29
  79. data/ext/carray_kernels_bincmp.c +4453 -0
  80. data/ext/carray_kernels_binop.c +11368 -0
  81. data/ext/carray_kernels_init.c +1734 -0
  82. data/ext/carray_kernels_moncmp.c +2104 -0
  83. data/ext/carray_kernels_monop.c +18369 -0
  84. data/ext/carray_kernels_reduce_aggregate.c +23818 -0
  85. data/ext/carray_kernels_reduce_boolean.c +406 -0
  86. data/ext/carray_kernels_reduce_cumulative.c +10086 -0
  87. data/ext/carray_kernels_reduce_extreme.c +17634 -0
  88. data/ext/carray_kernels_reduce_variance.c +3849 -0
  89. data/ext/carray_kernels_scan.c +4302 -0
  90. data/ext/carray_kernels_search.c +33829 -0
  91. data/ext/carray_kernels_sort.c +10906 -0
  92. data/ext/carray_kernels_triop.c +1864 -0
  93. data/ext/carray_key_windows.c +216 -0
  94. data/ext/carray_lazy.c +170 -111
  95. data/ext/carray_mask.c +292 -72
  96. data/ext/carray_median_percentile.c +82 -28
  97. data/ext/carray_memory_view.c +669 -561
  98. data/ext/carray_operator.c +124 -47
  99. data/ext/carray_order.c +384 -76
  100. data/ext/carray_partition.c +28 -27
  101. data/ext/carray_random.c +67 -13
  102. data/ext/carray_round.c +273 -0
  103. data/ext/carray_scatter.c +63 -38
  104. data/ext/carray_segment.c +3 -3
  105. data/ext/carray_slab.c +91 -28
  106. data/ext/carray_sort.c +35 -35
  107. data/ext/carray_struct.c +19 -18
  108. data/ext/carray_test.c +26 -5
  109. data/ext/carray_undef.c +4 -0
  110. data/ext/carray_utils.c +246 -45
  111. data/ext/mk_call_cfunc.rb +19 -2
  112. data/ext/mkkernel.rb +198 -240
  113. data/ext/ruby_carray.c +10 -4
  114. data/ext/version.h +4 -4
  115. data/lib/carray/attribute.rb +126 -22
  116. data/lib/carray/autoload_carray.rb +8 -31
  117. data/lib/carray/autoload_method_extension.rb +14 -5
  118. data/lib/carray/axis_group.rb +255 -106
  119. data/lib/carray/basics.rb +78 -10
  120. data/lib/carray/bincount_nd.rb +12 -2
  121. data/lib/carray/block_iterator.rb +126 -23
  122. data/lib/carray/boolean_reduce.rb +17 -17
  123. data/lib/carray/categorical.rb +165 -43
  124. data/lib/carray/categorical_iterator.rb +130 -27
  125. data/lib/carray/complex.rb +5 -5
  126. data/lib/carray/conditional.rb +67 -18
  127. data/lib/carray/const_string.rb +14 -5
  128. data/lib/carray/construct.rb +15 -11
  129. data/lib/carray/data_type_extension.rb +1 -0
  130. data/lib/carray/distance_limit.rb +77 -0
  131. data/lib/carray/fixlen_string.rb +7 -1
  132. data/lib/carray/frame/column_verbs.rb +70 -0
  133. data/lib/carray/frame/convert.rb +69 -0
  134. data/lib/carray/frame/csv_parser.rb +111 -17
  135. data/lib/carray/frame/describe.rb +3 -2
  136. data/lib/carray/frame/frame.rb +246 -21
  137. data/lib/carray/frame/group.rb +510 -95
  138. data/lib/carray/frame/io.rb +115 -28
  139. data/lib/carray/frame/join.rb +312 -51
  140. data/lib/carray/frame/records.rb +83 -7
  141. data/lib/carray/frame/reshape.rb +42 -35
  142. data/lib/carray/frame/sort.rb +7 -19
  143. data/lib/carray/frame/verbs.rb +188 -48
  144. data/lib/carray/frame/windows.rb +117 -0
  145. data/lib/carray/frame.rb +2 -0
  146. data/lib/carray/fuse_source.rb +11 -4
  147. data/lib/carray/fusion.rb +40 -16
  148. data/lib/carray/histogram.rb +6 -1
  149. data/lib/carray/inspect.rb +31 -8
  150. data/lib/carray/iterator.rb +85 -7
  151. data/lib/carray/key_windows.rb +620 -0
  152. data/lib/carray/lazy.rb +182 -183
  153. data/lib/carray/mask_gap_fill.rb +63 -57
  154. data/lib/carray/meld_reduce.rb +12 -14
  155. data/lib/carray/methods/bin.rb +26 -18
  156. data/lib/carray/methods/bincount.rb +16 -0
  157. data/lib/carray/methods/bit_string.rb +1 -1
  158. data/lib/carray/methods/choose.rb +16 -3
  159. data/lib/carray/methods/composition.rb +3 -1
  160. data/lib/carray/methods/gather_nd.rb +14 -5
  161. data/lib/carray/methods/insert_block.rb +2 -2
  162. data/lib/carray/methods/is_between.rb +59 -0
  163. data/lib/carray/methods/is_in.rb +53 -36
  164. data/lib/carray/methods/join.rb +1 -6
  165. data/lib/carray/methods/lag.rb +49 -0
  166. data/lib/carray/methods/locate_addr.rb +47 -51
  167. data/lib/carray/methods/mode.rb +1 -1
  168. data/lib/carray/methods/nunique.rb +11 -6
  169. data/lib/carray/methods/resize.rb +3 -3
  170. data/lib/carray/methods/value_counts.rb +4 -11
  171. data/lib/carray/record_order.rb +3 -3
  172. data/lib/carray/rng.rb +3 -0
  173. data/lib/carray/runtime.rb +53 -8
  174. data/lib/carray/segment_iterator.rb +313 -82
  175. data/lib/carray/serialize.rb +24 -10
  176. data/lib/carray/slab_iterator.rb +50 -0
  177. data/lib/carray/string.rb +105 -5
  178. data/lib/carray/string_operation_extension.rb +8 -1
  179. data/lib/carray/struct.rb +9 -21
  180. data/lib/carray/struct_builder.rb +19 -8
  181. data/lib/carray/time.rb +726 -104
  182. data/lib/carray/window_iterator.rb +122 -16
  183. data/lib/carray.rb +7 -0
  184. data/yard-stubs/ca_obj_bitfield.rb +1 -1
  185. data/yard-stubs/ca_obj_field.rb +4 -0
  186. data/yard-stubs/ca_obj_grid.rb +21 -10
  187. data/yard-stubs/ca_obj_refer.rb +2 -1
  188. data/yard-stubs/ca_obj_roll.rb +2 -2
  189. data/yard-stubs/ca_obj_shift.rb +5 -1
  190. data/yard-stubs/ca_obj_tile.rb +1 -1
  191. data/yard-stubs/ca_obj_window.rb +8 -3
  192. data/yard-stubs/carray_access.rb +6 -1
  193. data/yard-stubs/carray_attribute.rb +0 -5
  194. data/yard-stubs/carray_cast.rb +10 -0
  195. data/yard-stubs/carray_conversion.rb +2 -1
  196. data/yard-stubs/carray_copy.rb +4 -2
  197. data/yard-stubs/carray_count.rb +4 -4
  198. data/yard-stubs/carray_element.rb +10 -4
  199. data/yard-stubs/carray_generate.rb +21 -1
  200. data/yard-stubs/carray_mask.rb +26 -2
  201. data/yard-stubs/carray_math.rb +27 -1
  202. data/yard-stubs/carray_median_percentile.rb +5 -2
  203. data/yard-stubs/carray_memory_view.rb +40 -23
  204. data/yard-stubs/carray_order.rb +29 -18
  205. data/yard-stubs/carray_random.rb +13 -4
  206. data/yard-stubs/carray_scatter.rb +14 -9
  207. data/yard-stubs/carray_slab.rb +9 -2
  208. data/yard-stubs/carray_sort.rb +6 -5
  209. data/yard-stubs/carray_test.rb +5 -3
  210. metadata +25 -3
  211. data/lib/carray/arrow_tensor.rb +0 -401
  212. data/lib/carray/table.rb +0 -108
@@ -1,32 +1,45 @@
1
1
  /* ---------------------------------------------------------------------------
2
2
 
3
- T1 kernel_iterator MVP — Phase 1 step 1 + step 2 + step 3 implementation.
4
-
5
- Scope after step 3 (PROPOSAL_T1_KERNEL_ITERATOR.md §10.3):
6
- - ca_iter_state struct + init / next_slab / next_slab_strided /
7
- finish state machine
8
- - CA_SLAB_WHOLE policy only (step 5+ adds AXES / FREE)
9
- - level == 1 (L1, contig kernel):
10
- - entity / CAStride contig: alias path (single slab, alias_ptr
11
- = parent->ptr, stride implicit = bytes)
12
- - CAStride family non-contig: scratch path (ca_copy_data
13
- compose-fold gather into a malloc'd buffer)
14
- - other sources: CA_ITER_ERR_NOT_CHEAP
15
- - level == 2 (L2, strided kernel):
16
- - entity / CAStride contig: alias path (single slab,
17
- stride_bytes = bytes)
18
- - CAStride family non-contig: alias_strided path — no scratch,
19
- per-outer-prefix yield with native inner stride_bytes
20
- - other sources: CA_ITER_ERR_NOT_CHEAP
21
- - READ-only (flags == 0). WRITE = step 4, NO_MASK = step 7.
3
+ ca_kernel_iterator.c -- the kernel iterator engine
4
+
5
+ A C kernel asks for the cells of an array, one slab at a time, and gets
6
+ them whatever the array is: an entity, a view of a view, a selection, a
7
+ lazy expression. This file decides, for each source, how to deliver
8
+ them (borrow its memory, gather a copy, walk a stack's parents) and, on
9
+ a write, how the cells the kernel changed get back to the array.
10
+
11
+ Where to read first:
12
+ - writing a kernel: guides/devel/11_kernel_iterator.md and
13
+ docs/authoring/HOW_TO_WRITE_KERNEL.md; the surface itself is
14
+ ca_kernel_iterator.h
15
+ - how the engine works (its routes, the promises every route keeps,
16
+ and which part keeps each): docs/internal/KernelIteratorEngine.md
17
+
18
+ Map of this file:
19
+ - scratch buffers (ki_data_alloc / ki_data_free) and helpers
20
+ - classification: ca_iter_classify_source, ca_iter_route_source
21
+ - the promise parts every route calls: ki_take_mask_copy,
22
+ ki_gather_view, ki_scratch_push_changed, ki_attach_stack_parents,
23
+ ki_fiber_contig_data / _mask
24
+ - opening a walk: init_l1, and for L2 ki_choose_l2_route (decides)
25
+ then ki_open_l2 (builds the state, with ki_lay_out_slabs)
26
+ - handing out slabs: next_slab, next_slab_axes
27
+ - writing back: sync_slab
28
+ - closing: finish, close, and the raise-safe entry points
29
+ (ca_iter_protect, ca_iter_ensure)
30
+ - development-build test surface (CARRAY_DEV_BUILD), at the end
31
+
32
+ A route added for speed is a shortcut, tagged next to its code with
33
+ "Shortcut NAME" and a fixed set of fields; each one is switched off by
34
+ ca_fast_paths_enabled().
22
35
 
23
36
  ---------------------------------------------------------------------------- */
24
37
 
25
38
  #include "carray.h"
26
39
  #include "carray_internal.h" /* per-obj_type view constructors */
27
40
  #include "ca_kernel_iterator.h"
28
- #include "ca_monop_dispatch.h" /* P.6.2.d: ca_monop_view_is_single_cast for F.6.2 gate */
29
- #include "ca_obj_face.h" /* PROPOSAL_CAFACE_PHASE_2 F.2.6 — ca_strip_face for SRC_* entry */
41
+ #include "ca_monop_dispatch.h" /* ca_monop_view_is_single_cast */
42
+ #include "ca_obj_face.h" /* ca_strip_face, to walk what a Face wraps */
30
43
 
31
44
  #include <assert.h>
32
45
  #include <string.h>
@@ -47,7 +60,24 @@
47
60
  reduction itself. The kept buffer is owned by nobody, and a walk that
48
61
  takes it owns it alone until it frees it again. Each Ractor keeps its
49
62
  own (ractor-local storage), since walks in different Ractors run at
50
- the same time. */
63
+ the same time.
64
+
65
+ Shortcut KEEP_SCRATCH (name tag; docs/internal/KernelIteratorEngine.md
66
+ section 3.4 has the table of all shortcuts).
67
+ Layer data: where a scratch comes from. The walk is unchanged.
68
+ Speeds up walks with a scratch of 1 MiB or more, run one after
69
+ another.
70
+ Why no difference measured on macOS (a per-axis sum of a
71
+ 2000x2000 fake view); the page faults it
72
+ avoids are expected to cost more on Linux (unmeasured).
73
+ Enters ki_data_alloc: bytes >= KI_KEEP_MIN and a kept buffer
74
+ that fits; ki_data_free: KI_KEEP_MIN <= bytes <=
75
+ KI_KEEP_MAX.
76
+ Promises 4 (a kept buffer has no owner, so no exit can leave it
77
+ held); 7 (object cells are cleared and held either way).
78
+ Switch ca_fast_paths_enabled(): off, every scratch is fresh.
79
+ Checked by test_iter_ractor_scratch.rb; the buffer count in
80
+ test_iter_delivery_detectors.rb. */
51
81
 
52
82
  #define KI_KEEP_MIN ((ca_size_t) 1 << 20)
53
83
  #define KI_KEEP_MAX ((ca_size_t) 64 << 20)
@@ -86,11 +116,24 @@ ki_kept_slot (void)
86
116
  return k;
87
117
  }
88
118
 
119
+ /* Development build: the number of buffers ki_data_alloc has handed out
120
+ that ki_data_free has not taken back (CArray.__iter_buffers__). A walk
121
+ that ends leaves the count where it found it; the spec suite compares it
122
+ around each walk. */
123
+ #ifdef CARRAY_DEV_BUILD
124
+ static long ki_buffers_out = 0;
125
+ /* atomic: walks run in several Ractors at once */
126
+ #define KI_BUFFERS_OUT(d) ( (void) __atomic_add_fetch(&ki_buffers_out, (d), __ATOMIC_RELAXED) )
127
+ #else
128
+ #define KI_BUFFERS_OUT(d) ((void) 0)
129
+ #endif
130
+
89
131
  static void *
90
132
  ki_data_alloc (int8_t data_type, ca_size_t bytes)
91
133
  {
92
134
  void *p = NULL;
93
- if ( bytes >= KI_KEEP_MIN ) {
135
+ KI_BUFFERS_OUT(1);
136
+ if ( bytes >= KI_KEEP_MIN && ca_fast_paths_enabled() ) {
94
137
  ki_kept_t *k = ki_kept_slot();
95
138
  if ( k->p && bytes <= k->bytes ) {
96
139
  p = k->p;
@@ -102,10 +145,8 @@ ki_data_alloc (int8_t data_type, ca_size_t bytes)
102
145
  p = xmalloc(bytes > 0 ? bytes : 1);
103
146
  }
104
147
  if ( data_type == CA_OBJECT ) {
105
- VALUE *v = (VALUE *) p;
106
148
  ca_size_t n = bytes / (ca_size_t) sizeof(VALUE);
107
- ca_size_t i;
108
- for ( i = 0; i < n; i++ ) v[i] = Qnil;
149
+ ca_object_cells_clear(p, n);
109
150
  ca_gc_hold_buffer(p, n);
110
151
  }
111
152
  return p;
@@ -115,8 +156,9 @@ ki_data_alloc (int8_t data_type, ca_size_t bytes)
115
156
  static void
116
157
  ki_data_free (void *p, ca_size_t bytes)
117
158
  {
159
+ KI_BUFFERS_OUT(-1);
118
160
  ca_gc_release_buffer(p);
119
- if ( bytes >= KI_KEEP_MIN && bytes <= KI_KEEP_MAX ) {
161
+ if ( bytes >= KI_KEEP_MIN && bytes <= KI_KEEP_MAX && ca_fast_paths_enabled() ) {
120
162
  ki_kept_t *k = ki_kept_slot();
121
163
  if ( bytes > k->bytes ) {
122
164
  void *old = k->p;
@@ -136,26 +178,28 @@ ki_data_free (void *p, ca_size_t bytes)
136
178
  extern int ca_attach_is_alias (void *ap);
137
179
  extern int ca_root_lends_no_memory (void *ap);
138
180
 
139
- /* CAStride family attach-fn marker (step 1-4). */
181
+ /* CAStride family attach-fn marker. */
140
182
  extern ca_operation_function_t ca_stride_func;
141
183
 
142
- /* Descriptor framework view ops (step 5+). Each view's describe_axes
143
- emits the per-axis descriptor + parent dim snapshot consumed by
144
- the P3 ca_axis_dispatch_* substrate. CAShift uses ca_window_func
145
- (Phase G typedef pattern, ca_shift_func is a copy with only free /
146
- clone / create_mask overridden) so it routes via attach pointer
147
- match against ca_window_func.attach. */
184
+ /* Descriptor framework view ops. Each view's describe_axes emits the
185
+ per-axis descriptor + parent dim snapshot consumed by the
186
+ ca_axis_dispatch_* engine. CAShift shares CAWindow's operation table
187
+ (ca_shift_func is a copy with only free / clone / create_mask
188
+ overridden), so it is recognised by its attach pointer matching
189
+ ca_window_func.attach. */
148
190
  extern ca_operation_function_t ca_select_axis_func;
149
191
  extern ca_operation_function_t ca_grid_func;
150
192
  extern ca_operation_function_t ca_select_func;
151
193
  extern ca_operation_function_t ca_window_func;
152
- /* ca_mapping_func retired in R.3 (PROPOSAL_CAMAPPING_REMOVAL). */
153
194
 
154
- /* SRC_ATTACH 5 view (step 9). Each defines its own attach that
155
- materialises src->ptr via view-specific transform (cast / swap /
156
- bit unpack / reduction). kernel_iterator treats them uniformly:
157
- ca_attach(src) → kernel sees src->ptr as contig → on WRITE,
158
- ca_sync(src) lets the view's sync_data scatter back. */
195
+ /* Operation tables the classifier compares a source's attach slot
196
+ against. Every view below is delivered as CA_ITER_SRC_ATTACH: the
197
+ engine gathers its cells into a scratch buffer of its own through the
198
+ view's transfer (ki_gather_view = ca_xfer_all GET) and, on a WRITE
199
+ walk, sends back the cells the kernel changed (ki_scratch_push_changed
200
+ = ca_xfer_addrs, or ca_xfer_all PUT when every cell changed). The
201
+ view's attach / sync slots are not called. Cast / byte swap / bit
202
+ unpack / reduction views: */
159
203
  extern ca_operation_function_t ca_fake_func;
160
204
  extern ca_operation_function_t ca_byte_swap_func;
161
205
  extern ca_operation_function_t ca_bitfield_func;
@@ -163,42 +207,24 @@ extern ca_operation_function_t ca_bitarray_func;
163
207
  extern ca_operation_function_t ca_reduce_func;
164
208
  extern ca_operation_function_t ca_object_func;
165
209
 
166
- /* F-2 follow-up (2026-05-26): connect CATile / CARoll to kernel_iterator
167
- via the SRC_ATTACH pattern. Per-cell modulo wrap does not fit the
168
- descriptor framework's kind enum {STRIDE, INDEX, SHIFT}, so the
169
- innermost-STRIDE L2 alias path is not used; but the view-specific
170
- func_attach (embed-region gather) materialises src->ptr and func_sync
171
- scatters back — structurally identical to SRC_ATTACH. Fills the gap
172
- in the "deliver" principle (= these were previously rejected via
173
- SRC_NONE). */
210
+ /* CATile / CARoll: the per-cell modulo wrap has no STRIDE / INDEX /
211
+ SHIFT descriptor, so they are delivered as SRC_ATTACH (above). */
174
212
  extern ca_operation_function_t ca_tile_func;
175
213
  extern ca_operation_function_t ca_roll_func;
176
214
 
177
- /* PROPOSAL_CASTACK.md Phase 3 (2026-06-18): CAStack via SRC_ATTACH.
178
- func_attach materialises src->ptr via per-parent ca_xfer_all GET
179
- (= K * parent.bytes alloc, caller responsibility per MEMO §3.4);
180
- func_sync scatters back via xfer_all PUT. kernel sees a flat
181
- contig slab over the stacked output of shape (K, *parent_shape). */
215
+ /* CAStack / CAMeld: SRC_ATTACH (above). A stack can also be walked
216
+ through its parents directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K,
217
+ see init_l2). */
182
218
  extern ca_operation_function_t ca_stack_func;
183
219
  extern ca_operation_function_t ca_meld_func;
184
220
 
185
- /* M.6 (PROPOSAL_CAREMAP_INTERNAL.md §5.2): CARemap is internal-only
186
- (no Ruby class constant) but participates in kernel_iterator as an
187
- SRC_ATTACH source. Per-element gather has no STRIDE structure to
188
- preserve, so SRC_ATTACH (scratch materialise via ca_remap_func_attach)
189
- is the natural acceptance path. func_attach allocates a fresh
190
- buffer and runs xfer_all(GET); kernel sees a flat contig slab.
191
- On WRITE, ca_sync routes through xfer_all(PUT). */
221
+ /* CARemap (internal, no Ruby class constant): a per-element gather with
222
+ no STRIDE structure, delivered as SRC_ATTACH (above). */
192
223
  extern ca_operation_function_t ca_remap_func;
193
224
 
194
- /* PROPOSAL_LAZY_ELEMENTWISE_VIEW Phase 4.5 P.4.5.1 (2026-06-07): lazy
195
- element-wise view family. CAMonOp (Phase 1) / CABinOp (Phase 2) /
196
- CABinCmp + CAMonCmp (Phase 4) all expose a func_attach that pulls
197
- the lazy tree's materialise into a fresh contig buffer; kernel sees
198
- a flat slab. Same SRC_ATTACH structural pattern as CAFake et al,
199
- so a 4-line addition to the classify_source list opens all 22
200
- mkkernel-generated reduction ops (sum / count / mean / variance /
201
- argmin / ...) to lazy operands — `(a.lazy + b).sum` now works. */
225
+ /* Lazy element-wise views (CAMonOp / CABinOp / CABinCmp / CAMonCmp /
226
+ CATriOp): SRC_ATTACH (above); gathering evaluates the expression into
227
+ the scratch, so every generated reduction accepts a lazy operand. */
202
228
  extern ca_operation_function_t ca_monop_func;
203
229
  extern ca_operation_function_t ca_binop_func;
204
230
  extern ca_operation_function_t ca_bincmp_func;
@@ -255,16 +281,27 @@ ca_iter_register_source_kind (int obj_type, uint8_t kind)
255
281
  ca_iter_registered_source_kind[obj_type] = kind;
256
282
  }
257
283
 
258
- /* Classify the source by routing kind (proposal §1 strategy table).
259
- Returns CA_ITER_SRC_NONE for sources not yet supported. */
284
+ /* Classify the source by how its cells are delivered. Returns
285
+ CA_ITER_SRC_NONE only for a NULL source or one with no attach slot;
286
+ every other view is delivered, as SRC_ATTACH if nothing more specific
287
+ applies. */
260
288
  static uint8_t
261
289
  ca_iter_classify_source (CArray *src)
262
290
  {
263
291
  if ( src == NULL ) return CA_ITER_SRC_NONE;
292
+
293
+ /* Development build, CARRAY_GENERAL_PATH=2: every source takes the
294
+ transfer walk (see ki_open_by_transfer), the reference the other
295
+ routes are checked against. */
296
+ if ( ca_iter_by_transfer_only() ) {
297
+ return ca_func[src->obj_type].attach != NULL ? CA_ITER_SRC_ATTACH
298
+ : CA_ITER_SRC_NONE;
299
+ }
300
+
264
301
  if ( ca_is_entity(src) ) return CA_ITER_SRC_CASTRIDE;
265
302
  if ( ca_iter_is_castride_family(src) ) return CA_ITER_SRC_CASTRIDE;
266
303
 
267
- /* Externally installed obj_types declare their routing (2026-08-07).
304
+ /* Externally installed obj_types declare their routing.
268
305
  Placed after the two structural cases and before the built-in list:
269
306
  entity and CAStride-family sources are recognised from the struct
270
307
  itself and are read directly, so a registration must not be able to
@@ -276,95 +313,73 @@ ca_iter_classify_source (CArray *src)
276
313
  if ( kind != CA_ITER_SRC_NONE ) return kind;
277
314
  }
278
315
 
279
- /* Descriptor framework views (step 5.1: CSA + CAGrid; 5.2: + CASelect /
280
- CAMapping / CAWindow / CAShift). CAShift uses ca_window_func
281
- for attach (Phase G typedef pattern) so it matches the CAWindow
282
- check below. */
316
+ /* Descriptor views: CASelectAxis / CAGrid / CASelect / CAWindow.
317
+ CAShift shares ca_window_func's attach, so it matches the CAWindow
318
+ check. */
283
319
  void *attach = ca_func[src->obj_type].attach;
284
320
  if ( attach == ca_select_axis_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
285
321
  if ( attach == ca_grid_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
286
322
  if ( attach == ca_select_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
287
323
  if ( attach == ca_window_func.attach ) return CA_ITER_SRC_DESCRIPTOR; /* + CAShift */
288
324
 
289
- /* Step 9: SRC_ATTACH 5 view. Each view-specific attach materialises
290
- src->ptr via per-element transform; kernel sees a flat contig slab. */
325
+ /* The views below are SRC_ATTACH: gathered into scratch through their
326
+ own transfer (ki_gather_view) and, on WRITE, sent back the changed
327
+ cells (ki_scratch_push_changed). Their attach / sync slots are not
328
+ called by the engine. */
291
329
  if ( attach == ca_fake_func.attach ) return CA_ITER_SRC_ATTACH;
292
330
  if ( attach == ca_byte_swap_func.attach ) return CA_ITER_SRC_ATTACH;
293
331
  if ( attach == ca_bitfield_func.attach ) return CA_ITER_SRC_ATTACH;
294
332
  if ( attach == ca_bitarray_func.attach ) return CA_ITER_SRC_ATTACH;
295
333
  if ( attach == ca_reduce_func.attach ) return CA_ITER_SRC_ATTACH;
296
- /* Step 11: CAObject — Ruby callback per-element bridge. Same
297
- SRC_ATTACH structural pattern (func_attach materialises via Ruby
298
- copy_data, func_sync scatters back via Ruby sync_data, CA_FLAG_
299
- READ_ONLY auto-rejects WRITE via ca_is_readonly). Bench gate
300
- n/a (Ruby callback overhead structurally dominant). */
334
+ /* CAObject: each transfer calls the Ruby callback. A read-only
335
+ CAObject is refused for WRITE by the ca_is_readonly check. */
301
336
  if ( attach == ca_object_func.attach ) return CA_ITER_SRC_ATTACH;
302
337
 
303
- /* F-2 follow-up: CATile / CARoll via SRC_ATTACH (embed-region gather
304
- materialises src->ptr; ca_sync scatters back per view-specific
305
- semantics — CATile tile decomposition, CARoll cyclic permutation). */
306
338
  if ( attach == ca_tile_func.attach ) return CA_ITER_SRC_ATTACH;
307
339
  if ( attach == ca_roll_func.attach ) return CA_ITER_SRC_ATTACH;
308
340
 
309
- /* Phase 3 (PROPOSAL_CASTACK.md): CAStack via SRC_ATTACH. func_attach
310
- materialises src->ptr via per-parent ca_xfer_all GET (= K * parent
311
- bytes alloc, caller responsibility per MEMO §3.4); func_sync
312
- scatters back via xfer_all PUT. kernel sees a flat contig slab
313
- over the stacked output. Routine ndim mismatch is irrelevant
314
- here (= attach delivers a self-owned buffer of shape (K, *parent
315
- shape), kernel reads/writes that buffer). */
341
+ /* CAStack: SRC_ATTACH here; init_l2 may instead walk its parents
342
+ directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K). */
316
343
  if ( attach == ca_stack_func.attach ) return CA_ITER_SRC_ATTACH;
317
344
 
318
- /* CAMeld — ragged concatenate along an existing axis. func_attach
319
- materialises via K per-parent xfer_all GET into a contig buffer at
320
- seg_offset[k] * tail_bytes offsets; func_sync scatters back. Same
321
- SRC_ATTACH structural pattern as CAStack; reduce hot paths bypass
322
- this via the per-parent decompose in lib/carray/meld_reduce.rb. */
345
+ /* CAMeld: reduce hot paths avoid the gather through the per-parent
346
+ decompose in lib/carray/meld_reduce.rb. */
323
347
  if ( attach == ca_meld_func.attach ) return CA_ITER_SRC_ATTACH;
324
348
 
325
- /* M.6: CARemap — per-element gather, internal-only. Same SRC_ATTACH
326
- structural pattern (func_attach materialises via xfer_all GET,
327
- func_sync via xfer_all PUT). */
328
349
  if ( attach == ca_remap_func.attach ) return CA_ITER_SRC_ATTACH;
329
350
 
330
- /* Phase 4.5 P.4.5.1: lazy view family — CAMonOp / CABinOp / CABinCmp /
331
- CAMonCmp. func_attach pulls the lazy chain materialise (arena-pooled
332
- scratches under the hood from Phase 3); kernel sees a flat contig
333
- slab. Read-only (= CA_FLAG_READ_ONLY) so WRITE auto-rejects at
334
- the ca_is_readonly check. Opens 22 mkkernel-generated reduction
335
- ops to lazy operands. */
351
+ /* Lazy views (CAMonOp / CABinOp / CABinCmp / CAMonCmp / CATriOp):
352
+ gathering evaluates the expression. They are read-only, so WRITE is
353
+ refused by the ca_is_readonly check. */
336
354
  if ( attach == ca_monop_func.attach ) return CA_ITER_SRC_ATTACH;
337
355
  if ( attach == ca_binop_func.attach ) return CA_ITER_SRC_ATTACH;
338
356
  if ( attach == ca_bincmp_func.attach ) return CA_ITER_SRC_ATTACH;
339
357
  if ( attach == ca_moncmp_func.attach ) return CA_ITER_SRC_ATTACH;
340
358
  if ( attach == ca_triop_func.attach ) return CA_ITER_SRC_ATTACH;
341
359
 
342
- /* Anything else is a view class installed from outside that did not
343
- register. Attaching is the one thing every view can do, and it is
344
- what a registration would declare, so the array is delivered that way
345
- rather than refused. */
360
+ /* Anything else -- a view class installed from outside that did not
361
+ register, or an in-tree class with no case above (such as the mask
362
+ views CAMaskOfOperands and CAObjectMask). It is classified as a
363
+ registration would declare it, SRC_ATTACH, and delivered by the
364
+ transfer walk through its xfer slots rather than refused. */
346
365
  if ( attach != NULL ) return CA_ITER_SRC_ATTACH;
347
366
 
348
367
  return CA_ITER_SRC_NONE;
349
368
  }
350
369
 
351
- /* F-2 (PROPOSAL_F2_KERNEL_ITERATOR_ALIAS rev6): route a source by
352
- running classify_source and, for descriptor-routed views, also calling
353
- describe_axes to inspect the innermost axis kind. Returns the refined
354
- src_kind (= SRC_DESCRIPTOR_L2_ALIASABLE iff innermost axis is STRIDE,
355
- else SRC_DESCRIPTOR). out_descs / out_parent_dims / out_ndim are
356
- populated for descriptor sources so the caller (init_l1 / init_l2)
357
- does not have to re-call describe_axes. For non-descriptor sources
358
- (CASTRIDE / ATTACH / NONE) the out_* arguments are not touched and
359
- classify_source's verdict is returned as-is.
360
-
361
- Cost analysis: classify_source is O(1) pointer compares. For
362
- SRC_DESCRIPTOR candidates we add one describe_axes call (O(ndim) with
363
- ndim ≤ CA_RANK_MAX = 16) plus one innermost-axis kind compare.
364
- Per-walk overhead = 1 describe_axes call (descriptor sources only),
365
- which init was going to do anyway -- so the routing is net zero cost
366
- compared to the pre-rev6 path where classify_source + init both
367
- computed describe_axes redundantly. See prep doc rev2 §4.1.1. */
370
+ /* Routes a source by running classify_source and, for descriptor-routed views, also calling
371
+ describe_axes to inspect the innermost axis kind. A descriptor view
372
+ comes back as SRC_DESCRIPTOR_L2_ALIASABLE (innermost axis STRIDE) or
373
+ SRC_DESCRIPTOR, and then out_descs / out_parent_dims / out_ndim are
374
+ filled so the caller does not re-call describe_axes. A grid with holes
375
+ and a view whose parent lends no memory come back as SRC_ATTACH, and
376
+ the out_* arguments are not touched; nor are they for any other kind,
377
+ whose classify_source verdict is returned as-is.
378
+
379
+ Cost: classify_source is O(1) pointer compares; a descriptor view
380
+ adds one describe_axes call (O(ndim)), which init would make anyway,
381
+ and init reads the descriptors from here instead of describing the
382
+ axes again. */
368
383
  static uint8_t ca_iter_classify_source (CArray *src);
369
384
  static void ca_iter_describe_axes (CArray *src, ca_axis_desc_t *,
370
385
  ca_size_t *, int8_t *);
@@ -378,6 +393,13 @@ ca_iter_route_source (CArray *src,
378
393
  uint8_t kind = ca_iter_classify_source(src);
379
394
  if ( kind != CA_ITER_SRC_DESCRIPTOR ) return kind;
380
395
 
396
+ /* A grid with holes (masked index cells) is not described by the engine
397
+ either: its holes are delivered by its own transfers. */
398
+ if ( ca_func[src->obj_type].attach == ca_grid_func.attach
399
+ && ca_grid_has_holes(src) ) {
400
+ return CA_ITER_SRC_ATTACH;
401
+ }
402
+
381
403
  /* The descriptor walk reads the parent's memory, which such a parent
382
404
  only has after a whole materialise -- however few cells the view
383
405
  selects. Deliver through the view's own transfer instead: SRC_ATTACH
@@ -398,9 +420,8 @@ ca_iter_route_source (CArray *src,
398
420
  return CA_ITER_SRC_DESCRIPTOR;
399
421
  }
400
422
 
401
- /* Dispatch to the view's describe_axes. Routing keyed on the
402
- shared attach pointer (CAShift shares with CAWindow per Phase G
403
- typedef). */
423
+ /* Dispatch to the view's describe_axes, keyed on the attach pointer
424
+ (CAShift shares CAWindow's). */
404
425
  static void
405
426
  ca_iter_describe_axes (CArray *src,
406
427
  ca_axis_desc_t *out_descs,
@@ -435,7 +456,8 @@ ca_iter_describe_axes (CArray *src,
435
456
  *out_ndim = 0;
436
457
  }
437
458
 
438
- /* Public alias eligibility predicate (proposal §11.3 case (a)). */
459
+ /* Public alias eligibility predicate: whether a walk at `level` could
460
+ read the source's own memory in place. */
439
461
  int
440
462
  ca_iter_can_alias (void *ap, int level)
441
463
  {
@@ -451,9 +473,8 @@ ca_iter_can_alias (void *ap, int level)
451
473
  return ca_iter_is_castride_family(ca);
452
474
 
453
475
  default:
454
- /* L3 (and any future level) — not implemented in Phase 1; L1
455
- fallback so the predicate stays well-defined for callers that
456
- probe ahead. */
476
+ /* L3 (and any future level) is not implemented; answer as L1 so
477
+ the predicate stays well-defined for callers that probe ahead. */
457
478
  return ca_attach_is_alias(ca);
458
479
  }
459
480
  }
@@ -473,24 +494,24 @@ ca_iter_build_rowmajor_strides (ca_size_t *strides,
473
494
  }
474
495
  }
475
496
 
476
- /* ---- PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 dispatch predicate -------- */
497
+ /* ---- PER_FIBER_FUSED: which views take it ----------------------------- */
477
498
 
478
- /* Hybrid (rev2 Q2): coarse src_kind branch + view-specific func pointer
479
- probe + fiber-axis effective stride check (Q3).
480
- F.6.1 substrate: returns 0 (= disabled) for all sources. F.6.2+
481
- (CAFake / CAByteSwap / CAShift / CAWindow / etc.) progressively enable
482
- specific source kinds with bench-driven justification.
499
+ /* A src_kind branch, then a check of the view's operation table, then of
500
+ the fiber axis's stride. Returns nonzero for
501
+ CAFake / CAByteSwap / a single-cast CAMonOp under SRC_ATTACH when the
502
+ fiber is contiguous in the parent, and for CAWindow / CAShift under
503
+ SRC_DESCRIPTOR; zero otherwise.
483
504
 
484
505
  The fiber_axis_stride argument is the effective byte stride of the
485
506
  fiber axis on the view (= what next_slab_axes would yield as
486
- slab_strides[0]). Q3-equivalence: fiber_axis_stride == src->bytes
487
- means the fiber is parent-memory contig, which is the precondition
488
- for X.1/X.4 per-region fused paths to deliver a 1-pass result.
489
-
490
- Called from init_l2 SRC_ATTACH branch right before the whole-view
491
- ca_xfer_all GET. When returning 1, caller skips xfer_all GET and
492
- sets alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED with fiber dispatch
493
- state populated. */
507
+ slab_strides[0]). fiber_axis_stride == src->bytes means the fiber is
508
+ a contiguous run of the parent's memory, which is what lets the view's
509
+ region transfer deliver it in one pass.
510
+
511
+ Called from two places in init_l2: the CA_SLAB_AXES descriptor branch
512
+ and the SRC_ATTACH branch, each before it would materialise. When it
513
+ returns nonzero the caller skips the materialise and sets alias_mode =
514
+ CA_ITER_ALIAS_PER_FIBER_FUSED with the fiber dispatch state filled. */
494
515
  static int
495
516
  ca_iter_should_per_fiber_fused (CArray *src,
496
517
  int src_kind,
@@ -499,27 +520,22 @@ ca_iter_should_per_fiber_fused (CArray *src,
499
520
  uint32_t flags)
500
521
  {
501
522
  (void) fiber_axis;
502
- (void) flags;
503
-
504
- /* F.6.2: CAFake / CAByteSwap (transform-fused, X.4 per-region).
505
- Enable when fiber-axis effective stride == view cell bytes
506
- (= Q3: fiber is parent-memory contig run, so ca_xfer_stride
507
- routes into ca_xfer_stride_transform_fused inner-contig fast
508
- path for 1-pass per-fiber delivery).
509
-
510
- F.5 bench: innermost-axis fiber 30-35% faster than whole-view
511
- materialise (CAFake) / 16-18% (CAByteSwap). Non-innermost (=
512
- fiber_axis_stride != bytes) was a loser, so the stride gate
513
- keeps current path for that case.
514
-
515
- Phase 6 P.6.2.d (Q13 α): single-cast CAMonOp (= post-migration
516
- successor of CAFake numeric path) is recognised via
517
- ca_monop_view_is_single_cast and routed through the same F.6.2
518
- fast path. Chain CAMonOp (depth ≥ 2) is excluded — its
519
- materialise path (= arena-pooled scratches from Phase 3) is
520
- structurally different from the X.4 inner-contig fast path. */
523
+
524
+ /* CAFake / CAByteSwap: when the fiber is a contiguous run of the
525
+ parent's memory, ca_xfer_stride takes the view's converting fast
526
+ path (ca_xfer_stride_transform_fused) and delivers the fiber in one
527
+ pass. A fiber along another axis was measured slower this way, so
528
+ the stride gate keeps it on the gathered route. A single-cast
529
+ CAMonOp (ca_monop_view_is_single_cast) is the lazy form of the same
530
+ conversion and takes the same route; a chain of lazy operations
531
+ gathers through the lazy arena instead, which this does not fit. */
521
532
  void *attach = ca_func[src->obj_type].attach;
522
- if ( src_kind == CA_ITER_SRC_ATTACH ) {
533
+ /* A conversion view is read-only on this path. Its put-back sends the
534
+ whole fiber, and a cell the kernel did not write does not come back
535
+ unchanged through a conversion that loses information (an int64
536
+ above 2^53 seen as float64), so a write takes the transfer walk,
537
+ which sends back only the changed cells (promise 3). */
538
+ if ( src_kind == CA_ITER_SRC_ATTACH && !(flags & CA_KERNEL_WRITE) ) {
523
539
  if ( attach == ca_fake_func.attach
524
540
  || attach == ca_byte_swap_func.attach
525
541
  || ca_monop_view_is_single_cast(src) ) {
@@ -527,64 +543,26 @@ ca_iter_should_per_fiber_fused (CArray *src,
527
543
  }
528
544
  }
529
545
 
530
- /* F.6.3: CAShift / CAWindow with OOB-fused materialise path (X.1).
531
- Per-fiber ca_xfer_stride wins decisively when the view would
532
- otherwise fall into a per-slab gather / whole-view materialise
533
- path (= SRC_DESCRIPTOR, e.g. CAShift always, CAWindow with SHIFT
534
- axes). But when the view qualifies for the L2 alias fast path
535
- (SRC_DESCRIPTOR_L2_ALIASABLE, = interior-only CAWindow with F-1
536
- STRIDE promotion), the existing alias path is already optimal
537
- (parent.ptr + composed offsets, zero materialise). Per-fiber
538
- dispatch in that case adds ca_xfer_stride per-call overhead with
539
- no payoff -- F.6.3 bench measured 0.35-0.82x (regression). Gate
540
- on SRC_DESCRIPTOR only. */
546
+ /* CAShift / CAWindow: the view's region transfer fills out-of-bounds
547
+ cells itself, so a fiber can be fetched in one call instead of a
548
+ per-slab or whole-view gather (SRC_DESCRIPTOR). A view that can
549
+ lend the parent's memory (SRC_DESCRIPTOR_L2_ALIASABLE, an interior
550
+ CAWindow) is already read in place; fetching it fiber by fiber was
551
+ measured slower (0.35-0.82x), so only SRC_DESCRIPTOR takes it. */
541
552
  if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
542
- /* Both CAWindow and CAShift match ca_window_func.attach (Phase G
543
- typedef pattern: ca_shift_func = ca_window_func). */
553
+ /* CAWindow and CAShift share ca_window_func.attach. */
544
554
  if ( attach == ca_window_func.attach ) {
545
555
  return 1;
546
556
  }
547
557
  }
548
558
 
549
- /* F.6.4 audit (devel/bench_f6_4_audit.rb): CSA (CASelectAxis) /
550
- CAGrid / CASelect / CAMapping all kept on the whole-view
551
- materialise path. Per-fiber ca_xfer_stride is a LOSER for
552
- these views (1.25-4.7x slower across all axis positions
553
- measured) because the descriptor framework's
554
- ca_axis_dispatch_gather is already a single batched per-axis
555
- kind gather; splitting into N per-fiber calls loses the
556
- batched throughput. Structurally different from
557
- CAFake/CAByteSwap (transform-fused at byte level, X.4) and
558
- CAShift/CAWindow OOB (per-region fused with bound check, X.1):
559
- descriptor framework views do not have a per-region fused fast
560
- path designed for per-fiber entry.
561
-
562
- Note (rev4): bench split fiber-axis kind shows STRIDE fiber
563
- 1.25x slower vs INDEX fiber 4-5x slower. STRIDE-fiber case
564
- is close to parity and could conceivably reach win with finer
565
- gating (= bypass per-axis kind classification when the outer
566
- prefix is homogeneous STRIDE). Not pursued at current 1.25x
567
- deficit; recorded as a future possible refinement should an
568
- application motivate it. See proposal §3.1 SRC_DESCRIPTOR
569
- (CSA, CAGrid, ...) inline comment.
570
-
571
- F.6.6 audit (devel/bench_f6_6_bit_audit.rb): CABitarray /
572
- CABitfield kept on current path. Bench measured via
573
- bits.fake(:float64) (= production CAFake-wrap hot path) shows
574
- per-fiber 1.23-5.86x slower on the F.6.2-gate-off axes.
575
-
576
- The CAFake wrap overhead applies equally to both per-fiber and
577
- whole-view paths (same total cell-cast cost), so the 5x ratio
578
- is essentially the CABitarray whole-view-vs-per-fiber ratio
579
- itself. Phase A bench note "xfer_stride 0.98x parity" is a
580
- single-call comparison: N per-fiber calls cumulatively cost
581
- N x parity > 1 x xfer_all (+ N dispatch overhead + per-call
582
- bit alignment math). This reasoning generalises to direct
583
- CABitarray / CABitfield source (= when a non-f64 fiber smoke
584
- is added in a future phase) -- no separate audit needed for
585
- that case. Predicate stays unchanged.
586
-
587
- F.6.7 will audit CAReduce. */
559
+ /* Not taken by the other views, each measured slower fiber by fiber:
560
+ CASelectAxis / CAGrid / CASelect (1.25-4.7x), whose descriptor
561
+ engine already gathers the whole view in one batched pass and has no
562
+ per-region fast path to enter; CABitarray / CABitfield (1.2-5.9x,
563
+ through a fake over them), where N per-fiber calls each pay the bit
564
+ alignment that one whole-view transfer pays once. CAReduce is not
565
+ routed here either. */
588
566
  return 0;
589
567
  }
590
568
 
@@ -600,8 +578,8 @@ ca_iter_validate_inputs (ca_iter_state *st,
600
578
  uint32_t flags)
601
579
  {
602
580
  if ( st == NULL || src == NULL ) return CA_ITER_ERR_FLAGS;
603
- /* Phase A capstone: CA_SLAB_AXES accepted (init_l2 path only — init_l1
604
- does not implement CA_SLAB_AXES yet, see init_l1's policy gate). */
581
+ /* CA_SLAB_AXES is accepted here and refused by init_l1 itself; init_l2
582
+ refuses CA_SLAB_WHOLE (see each init's policy gate). */
605
583
  if ( policy != CA_SLAB_WHOLE && policy != CA_SLAB_AXES )
606
584
  return CA_ITER_ERR_POLICY;
607
585
 
@@ -724,6 +702,45 @@ ca_iter_check_init (int rc)
724
702
  return rc; /* not reached */
725
703
  }
726
704
 
705
+ /* Attaches a CAStack's parents and keeps their buffers. stack_n_parents
706
+ counts the parents attached so far, so an attach that raises leaves
707
+ finish with exactly those to detach (promise 4). The mask of a stack
708
+ reads its parts' masks, some perhaps through stand-ins: those give way
709
+ first, as the stack's own slots would let them. */
710
+ static void
711
+ ki_attach_stack_parents (ca_iter_state *st, CAStack *stack)
712
+ {
713
+ if ( stack->mask_owner != NULL ) {
714
+ CAStack *o = (CAStack *) stack->mask_owner;
715
+ ca_multi_parent_mask_refresh((CArray *) stack, o->parents,
716
+ o->mask_standins, o->n_parents);
717
+ }
718
+ st->stack_n_parents = 0;
719
+ st->stack_parent_ptrs =
720
+ (char **) xmalloc(stack->n_parents * sizeof(char *));
721
+ for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
722
+ ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
723
+ st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
724
+ st->stack_n_parents = kk + 1;
725
+ }
726
+ }
727
+
728
+ /* Promise 2: the mask cursor a walk hands out is the walk's own copy of
729
+ the source's mask, taken when the walk opens -- one flag per cell, in
730
+ the view's row-major order -- never the live mask, so a write through
731
+ it reaches nothing. A source without a mask leaves alias_mask NULL.
732
+ finish frees the copy. */
733
+ static void
734
+ ki_take_mask_copy (ca_iter_state *st, CArray *src)
735
+ {
736
+ if ( ! ca_has_mask(src) ) return;
737
+ st->scratch_mask = (boolean8_t *) xmalloc(src->elements > 0 ? src->elements : 1);
738
+ if ( src->elements > 0 ) {
739
+ ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
740
+ }
741
+ st->alias_mask = st->scratch_mask;
742
+ }
743
+
727
744
  /* Whole-view scratch. The view is gathered into the walk's scratch,
728
745
  by the view's own transfer (by_copy = 0) or by ca_copy_data (by_copy =
729
746
  1, the CAStride compose-fold). A WRITE walk sends back only the cells
@@ -764,7 +781,7 @@ ki_scratch_push_changed (ca_iter_state *st)
764
781
  ca_size_t view_bytes = (ca_size_t) src->elements * bytes;
765
782
  char *now = st->scratch_ptr;
766
783
  char *was = st->scratch_ptr + view_bytes;
767
- int whole = ! ( st->policy == CA_SLAB_AXES && st->level == 2 );
784
+ int whole = ( st->level == 1 ); /* L1 is the whole array */
768
785
  ca_size_t n = whole ? src->elements : st->slab_elements;
769
786
  ca_size_t idx[CA_RANK_MAX];
770
787
  ca_size_t *addrs;
@@ -800,7 +817,16 @@ ki_scratch_push_changed (ca_iter_state *st)
800
817
  }
801
818
  }
802
819
 
803
- if ( whole && n_changed == n ) {
820
+ /* Shortcut PUSH_ALL_CHANGED.
821
+ Layer write-back.
822
+ Speeds up a whole-view write walk that changed every cell: one
823
+ ca_xfer_all PUT instead of a list of all the addresses.
824
+ Why about 5% on a whole write of a fake view of 1M cells.
825
+ Enters whole && n_changed == n.
826
+ Promises 3 (the same cells go back; the list would hold them all).
827
+ Switch ca_fast_paths_enabled(): off, the address list.
828
+ Checked by the write rows of the delivery matrix (L1 / L2 whole). */
829
+ if ( whole && n_changed == n && ca_fast_paths_enabled() ) {
804
830
  ca_xfer_all(src, now, CA_XFER_PUT);
805
831
  }
806
832
  else if ( n_changed > 0 ) {
@@ -810,6 +836,36 @@ ki_scratch_push_changed (ca_iter_state *st)
810
836
  ALLOCV_END(h_addrs);
811
837
  }
812
838
 
839
+ /* The transfer walk: the whole view gathered into scratch through the
840
+ view's own transfer, its mask likewise, and yielded as one contiguous
841
+ slab; on WRITE, sync_slab sends the changed cells back through the same
842
+ transfer (ki_scratch_push_changed).
843
+
844
+ Every promise the iterator makes is kept here by the view itself --
845
+ what a cell is, where its mask flag lives, which root cell a write
846
+ lands on -- so this walk is the reference the other routes are checked
847
+ against (CARRAY_GENERAL_PATH=2 sends every source through it). It is
848
+ also the route for a view the classifier knows nothing more about.
849
+
850
+ The caller has filled src / src_kind / level / policy / ndim / flags /
851
+ bytes / naxes. */
852
+ static void
853
+ ki_open_by_transfer (ca_iter_state *st, CArray *src, uint32_t flags)
854
+ {
855
+ ki_gather_view(st, src, flags, 0);
856
+ st->alias_mode = CA_ITER_ALIAS_NONE;
857
+ st->alias_ptr = st->scratch_ptr;
858
+ st->composed_strides[0] = src->bytes;
859
+ st->composed_base = 0;
860
+ st->slab_n = src->elements;
861
+ st->total_slabs = 1;
862
+ st->slabs_emitted = 0;
863
+ st->chunk_size = st->slab_n;
864
+ st->outer_idx = NULL;
865
+
866
+ ki_take_mask_copy(st, src);
867
+ }
868
+
813
869
  static int
814
870
  ca_iter_state_init_l1_unprotected (ca_iter_state *st,
815
871
  struct _CArray *src,
@@ -826,9 +882,9 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
826
882
  memset(st, 0, sizeof(*st));
827
883
  if ( src == NULL ) return CA_ITER_ERR_FLAGS;
828
884
 
829
- /* PROPOSAL_CAFACE_PHASE_2 F.2.6 + PROPOSAL_LAZY_MARKER_LIFT Phase 0:
830
- storage-identical wrapper strip at entry (= same rationale as init_l2
831
- below). Strip before validate_inputs.
885
+ /* A Face or a lazy marker wraps storage identical to its parent, so it
886
+ is stripped at entry (the same reason as in init_l2 below), before
887
+ validate_inputs.
832
888
 
833
889
  The WRITE rejection is taken against the wrapper, not what it wraps: a
834
890
  CALazyMarker is read-only while its parent is not, and stripping first
@@ -844,12 +900,32 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
844
900
  int rc = ca_iter_validate_inputs(st, src, policy, flags);
845
901
  if ( rc != CA_ITER_OK ) return rc;
846
902
 
847
- /* Phase A capstone: CA_SLAB_AXES is L2-only (kernels needing K-D slab
848
- yield use init_l2). init_l1 rejects so callers don't get a silent
849
- misdispatch. */
903
+ /* CA_SLAB_AXES is L2 only (a kernel that wants a K-D slab uses
904
+ init_l2); init_l1 refuses it rather than walking the whole array. */
850
905
  if ( policy == CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
851
906
 
852
- uint8_t src_kind = ca_iter_classify_source(src);
907
+ /* The same routing as init_l2: a descriptor view whose parent lends no
908
+ memory, or a grid with holes, takes the transfer walk. An L1 walk
909
+ yields the whole view as one slab, so a descriptor view the L2 walk
910
+ could borrow is gathered here like any other descriptor view. */
911
+ ca_axis_desc_t route_descs[CA_RANK_MAX];
912
+ ca_size_t route_pdims[CA_RANK_MAX];
913
+ int8_t route_ndim = 0;
914
+ uint8_t src_kind = ca_iter_route_source(src, route_descs, route_pdims,
915
+ &route_ndim);
916
+ if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
917
+ src_kind = CA_ITER_SRC_DESCRIPTOR;
918
+ }
919
+ /* A write over a descriptor view takes the transfer walk, as an L2
920
+ write over an INDEX or SHIFT slab axis does. Gathering with the
921
+ descriptor engine would leave nothing to send back but the whole
922
+ view, and a view that shows a root cell twice (a grid with a
923
+ repeated index) would then let the copy the kernel did not write
924
+ overwrite the one it did (promise 3: only the changed cells go
925
+ back). */
926
+ if ( src_kind == CA_ITER_SRC_DESCRIPTOR && (flags & CA_KERNEL_WRITE) ) {
927
+ src_kind = CA_ITER_SRC_ATTACH;
928
+ }
853
929
 
854
930
  st->src = src;
855
931
  st->src_kind = src_kind;
@@ -866,41 +942,18 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
866
942
  st->chunk_size = st->slab_n;
867
943
 
868
944
  if ( src_kind == CA_ITER_SRC_ATTACH ) {
869
- /* === SRC_ATTACH path (step 9 + 2026-05-31 refactor): CAFake /
870
- CAByteSwap / CABitfield / CABitarray / CAReduce =================
871
- View's xfer_all delivers materialised data through the unified
872
- dispatch surface. Previously used `ca_attach(src) + src->ptr`
873
- (= view's attach slot) and `ca_sync(src)` for WRITE; now uses
874
- `ca_xfer_all(src, scratch, GET/PUT)` with iterator-owned scratch.
875
- Decouples kernel_iterator from per-view attach/sync lifecycle
876
- and inherits xfer reform improvements (transform-fused, etc.)
877
- automatically. */
878
- ki_gather_view(st, src, flags, 0);
879
- st->alias_mode = CA_ITER_ALIAS_NONE; /* scratch-owned, no src.detach */
880
- st->alias_ptr = st->scratch_ptr;
881
- st->composed_strides[0] = src->bytes;
882
- st->composed_base = 0;
883
- st->outer_idx = NULL;
884
- /* Step 6: mask gather (xfer_all on mask -- ca_copy_data is a thin
885
- wrapper, this just removes the extra hop). */
886
- if ( ca_has_mask(src) ) {
887
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
888
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
889
- if ( src->elements > 0 ) {
890
- ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
891
- }
892
- st->alias_mask = st->scratch_mask;
893
- }
945
+ ki_open_by_transfer(st, src, flags);
894
946
  return CA_ITER_OK;
895
947
  }
896
948
 
897
949
  if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
898
- /* === descriptor path (step 5.1: CSA + CAGrid) ======================
899
- Reuse P3 substrate: describe_axes → prepare → layout. For now
900
- always materialise via ca_axis_dispatch_attach into scratch and
901
- yield one contig slab (alias_mode = NONE). The view-transparency
902
- principle (proposal §0): kernel sees a flat slab regardless of
903
- per-axis kind; INDEX / SHIFT mix is handled inside the engine. */
950
+ /* === descriptor path (CASelectAxis / CAGrid / CASelect / CAWindow /
951
+ CAShift) ==========================================================
952
+ describe_axes → prepare → layout, then gather the view into
953
+ scratch with ca_axis_dispatch_gather and yield one contig slab
954
+ (alias_mode = NONE). The kernel sees a flat slab whatever the
955
+ per-axis kinds are; an INDEX / SHIFT mix is handled inside the
956
+ engine. */
904
957
  ca_axis_desc_t raw_descs[CA_RANK_MAX];
905
958
  int8_t raw_ndim = 0;
906
959
  ca_iter_describe_axes(src, raw_descs, st->parent_axis_dims, &raw_ndim);
@@ -918,20 +971,17 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
918
971
  }
919
972
  st->total_elements = src->elements;
920
973
 
921
- /* Materialise via the engine. ca_axis_dispatch_attach handles
922
- parent attach + scratch alloc + gather (INDEX axes included);
923
- we own the resulting buffer. Descriptor views inherit from
924
- CAView so parent is at the CAVIEW prefix slot. */
974
+ /* Materialise: attach the parent here, allocate the scratch, and
975
+ gather into it with ca_axis_dispatch_gather (INDEX axes included);
976
+ the iterator owns the buffer. Descriptor views inherit from CAView
977
+ so parent is at the CAVIEW prefix slot. */
925
978
  CArray *parent = CAVIEW(src)->parent;
926
979
  ca_attach(parent); /* engine reads parent->ptr */
927
980
  st->root = parent; /* finish() detaches */
928
981
  st->scratch_cap = src->elements * src->bytes;
929
- /* Engine ndim = descriptor ndim from describe_axes (not view.ndim).
930
- Critical for CAMapping where view.ndim > 1 but the descriptor is
931
- 1-D INDEX gather.
932
- bound_fill = CAWindow/CAShift fill value (Tier 2.B SHIFT-axis
933
- OOB cell), NULL for non-window views (CSA / CAGrid / CASelect /
934
- CAMapping don't have OOB semantics — engine sees no SHIFT axes). */
982
+ /* bound_fill = the CAWindow / CAShift fill value for an out-of-bounds
983
+ SHIFT cell; NULL for the other descriptor views, which have no
984
+ SHIFT axes. */
935
985
  const void *bound_fill = NULL;
936
986
  if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
937
987
  bound_fill = ((CAWindow *) src)->fill;
@@ -944,22 +994,11 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
944
994
  st->alias_mode = CA_ITER_ALIAS_NONE;
945
995
  st->alias_ptr = st->scratch_ptr;
946
996
 
947
- /* Step 6: gather mask via ca_copy_data on src->mask (works for
948
- all descriptor views — mask propagates through their
949
- func_copy_data via the descriptor framework's own mask
950
- handling). */
951
- if ( ca_has_mask(src) ) {
952
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
953
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
954
- if ( src->elements > 0 ) {
955
- ca_copy_data(src->mask, (char *) st->scratch_mask);
956
- }
957
- st->alias_mask = st->scratch_mask;
958
- }
997
+ ki_take_mask_copy(st, src);
959
998
  return CA_ITER_OK;
960
999
  }
961
1000
 
962
- /* === CAStride path (step 1-4) ============================== */
1001
+ /* === CAStride path ========================================= */
963
1002
  if ( ca_iter_can_alias(src, 1) ) {
964
1003
  /* alias path: entity or CAStride contig */
965
1004
  st->alias_mode = CA_ITER_ALIAS_CONTIG;
@@ -975,1440 +1014,687 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
975
1014
  st->alias_ptr = st->scratch_ptr;
976
1015
  }
977
1016
 
978
- /* Step 6: gather mask into scratch_mask if source carries one.
979
- We always gather into a contig boolean8_t buffer for uniformity
980
- across alias / scratch / descriptor paths — the kernel sees a
981
- simple `boolean8_t *` aligned with the value slab. Future
982
- optimisation: alias mask directly for CAStride contig. Mask is
983
- informational; kernel uses CA_FOR_EACH_UNMASKED macros to skip
984
- masked cells. */
985
- if ( ca_has_mask(src) ) {
986
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
987
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
988
- if ( src->elements > 0 ) {
989
- ca_copy_data(src->mask, (char *) st->scratch_mask);
990
- }
991
- st->alias_mask = st->scratch_mask;
992
- }
1017
+ ki_take_mask_copy(st, src);
993
1018
  return CA_ITER_OK;
994
1019
  }
995
1020
 
996
- static int
997
- ca_iter_state_init_l2_unprotected (ca_iter_state *st,
998
- struct _CArray *src,
999
- ca_slab_policy_t policy,
1000
- int8_t *axes,
1001
- int8_t naxes,
1002
- uint32_t flags)
1003
- {
1004
- /* Leave the state defined before anything can return. Every error
1005
- return below hands control back to a caller that may not look at the
1006
- code, so what it is left holding has to be a state that iterates zero
1007
- times rather than whatever its stack frame happened to contain. */
1008
- if ( st == NULL ) return CA_ITER_ERR_FLAGS;
1009
- memset(st, 0, sizeof(*st));
1010
- if ( src == NULL ) return CA_ITER_ERR_FLAGS;
1011
-
1012
- /* PROPOSAL_CAFACE_PHASE_2 F.2.6 (= MEMO §3.5 kernel_iterator entry strip)
1013
- + PROPOSAL_LAZY_MARKER_LIFT Phase 0.
1014
-
1015
- A Face only layers a semantic identifier and a CALazyMarker only layers
1016
- "leaf of a lazy chain"; storage is identical to the parent either way, so
1017
- strip at the kernel-compute entry and descend. Routing and alias
1018
- decisions belong to the parent. Strip before validate_inputs —
1019
- classify_source rejects both as knows-no.
1021
+ /* ---- L2 walks: choosing a route, then building the state for it --------
1022
+
1023
+ init_l2 checks the request, classifies the source and chooses one of the
1024
+ routes below (ki_choose_l2_route, which only decides). ki_open_l2 then
1025
+ builds the state for that route: the part only that route needs, the
1026
+ axis layout every route shares (ki_lay_out_slabs) and the mask. So the
1027
+ state is assembled in one place and init_l2 has one way out. */
1028
+
1029
+ typedef enum {
1030
+ KI_L2_FIBER_FUSED, /* PER_FIBER_FUSED: a CAWindow / CAShift fiber, or a
1031
+ read of a conversion view's innermost fiber */
1032
+ KI_L2_PER_SLAB, /* descriptor view, read, a slab axis INDEX or SHIFT:
1033
+ one slab gathered at a time (PER_SLAB / _HOIST) */
1034
+ KI_L2_DESC_WHOLE, /* descriptor view, read: the whole view gathered by
1035
+ the descriptor engine (outer SHIFT, SHIFT_WHOLE_VIEW) */
1036
+ KI_L2_DESC_STRIDED, /* descriptor view whose slab axes are all STRIDE:
1037
+ the parent's memory is borrowed */
1038
+ KI_L2_STACK, /* CAStack read along its stack axis */
1039
+ KI_L2_STACK_OUTER_K, /* CAStack read with the stack axis outside the slab */
1040
+ KI_L2_TRANSFER, /* the whole view gathered by its own transfer */
1041
+ KI_L2_ROOT_STRIDED, /* CAStride: the root walked through composed strides */
1042
+ KI_L2_VIEW_SCRATCH, /* CAStride over a root that lends no memory */
1043
+ KI_L2_CONTIG /* entity, or a contiguous CAStride */
1044
+ } ki_l2_route;
1020
1045
 
1021
- The Face identifier is re-wrapped onto the result by the caller's lift
1022
- hook (= primary operators / reductions / etc.). The marker is not:
1023
- a reduction over a lazy marker yields a plain entity.
1046
+ typedef struct {
1047
+ ki_l2_route route;
1048
+ uint8_t src_kind; /* what st->src_kind records */
1049
+ int8_t in_slab[CA_RANK_MAX]; /* 1 for each slab axis */
1050
+ ca_axis_desc_t descs[CA_RANK_MAX]; /* descriptor routes: the view's */
1051
+ ca_size_t pdims[CA_RANK_MAX]; /* axes and the parent's dims */
1052
+ int8_t ndesc;
1053
+ CArray *root; /* KI_L2_ROOT_STRIDED */
1054
+ ca_size_t composed[CA_RANK_MAX];
1055
+ ca_size_t base;
1056
+ } ki_l2_plan;
1057
+
1058
+ /* Per view axis: its extent, its byte stride when it is a slab axis, its
1059
+ byte stride when it is an outer axis, and its mask element stride. */
1060
+ typedef struct {
1061
+ ca_size_t dims[CA_RANK_MAX];
1062
+ ca_size_t slab_strides[CA_RANK_MAX];
1063
+ ca_size_t outer_strides[CA_RANK_MAX];
1064
+ ca_size_t mask_strides[CA_RANK_MAX];
1065
+ } ki_axis_layout;
1024
1066
 
1025
- WRITE is rejected against the wrapper, before the strip — a marker is
1026
- read-only while its parent is not. validate_inputs re-checks the
1027
- stripped source, which is harmless. */
1028
- if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
1029
- return CA_ITER_ERR_READONLY;
1030
- }
1031
- if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
1032
- src = ca_iter_strip_storage_wrapper(src);
1067
+ /* Row-major byte and element strides over the view's own shape. */
1068
+ static void
1069
+ ki_view_row_major (CArray *src, ca_size_t *bytes, ca_size_t *elems)
1070
+ {
1071
+ ca_size_t b = src->bytes, e = 1;
1072
+ for ( int8_t k = src->ndim - 1; k >= 0; k-- ) {
1073
+ if ( bytes ) bytes[k] = b;
1074
+ if ( elems ) elems[k] = e;
1075
+ b *= src->dim[k];
1076
+ e *= src->dim[k];
1033
1077
  }
1078
+ }
1034
1079
 
1035
- int rc = ca_iter_validate_inputs(st, src, policy, flags);
1036
- if ( rc != CA_ITER_OK ) return rc;
1037
-
1038
- /* F-2 (rev6): route_source returns the refined src_kind and (for
1039
- descriptor sources) populates raw_descs / raw_pdims / raw_ndim so we
1040
- don't re-call describe_axes when SRC_DESCRIPTOR_L2_ALIASABLE upgrades
1041
- into the alias branch. Non-descriptor sources leave the out_*
1042
- buffers untouched. */
1043
- ca_axis_desc_t raw_descs[CA_RANK_MAX];
1044
- ca_size_t raw_pdims[CA_RANK_MAX];
1045
- int8_t raw_ndim = 0;
1046
- uint8_t src_kind = ca_iter_route_source(src, raw_descs, raw_pdims, &raw_ndim);
1047
-
1048
- /* Phase B capstone (T2): CA_SLAB_AXES + descriptor view (= SRC_DESCRIPTOR
1049
- and SRC_DESCRIPTOR_L2_ALIASABLE), accept when slab axes are all-STRIDE
1050
- kind and no outer SHIFT. Other combinations:
1051
- - slab has INDEX/SHIFT kind → reject (Phase C T3 scope)
1052
- - outer has SHIFT axis → reject (B.1.5 materialise downgrade scope)
1053
- - SRC_ATTACH → reject (overlay views; no kind structure)
1054
- SRC_CASTRIDE still goes through the Phase A branch below. */
1055
- if ( policy == CA_SLAB_AXES ) {
1056
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR
1057
- || src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
1058
- /* Validate axes early (range / duplicate). */
1059
- if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
1060
- return CA_ITER_ERR_POLICY;
1061
- }
1062
- int8_t in_slab[CA_RANK_MAX];
1063
- int8_t k;
1064
- for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
1065
- for ( k = 0; k < naxes; k++ ) {
1066
- int8_t ax = axes[k];
1067
- if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
1068
- if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
1069
- in_slab[ax] = 1;
1070
- }
1071
-
1072
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.3 hook (= parallel to
1073
- F.6.2 SRC_ATTACH hook): for descriptor views CAWindow /
1074
- CAShift (predicate returns 1 unconditionally per F.5 always-
1075
- win bench), skip materialise (alias / C.1 PER_SLAB / B.1.5)
1076
- and use per-fiber ca_xfer_stride which routes to the view's
1077
- X.1 OOB-fused per-region fast path. */
1078
- if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
1079
- && raw_ndim == src->ndim && ca_fast_paths_enabled() ) {
1080
- int8_t fiber_ax = axes[0];
1081
- ca_size_t row_byte_strides[CA_RANK_MAX];
1082
- {
1083
- ca_size_t b = src->bytes;
1084
- for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
1085
- row_byte_strides[kk] = b;
1086
- b *= src->dim[kk];
1087
- }
1088
- }
1089
- if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
1090
- row_byte_strides[fiber_ax],
1091
- flags) ) {
1092
- st->src = src;
1093
- st->src_kind = src_kind;
1094
- st->level = 2;
1095
- st->policy = policy;
1096
- st->ndim = src->ndim;
1097
- st->flags = flags;
1098
- st->bytes = src->bytes;
1099
- st->naxes = naxes;
1100
-
1101
- st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
1102
- st->alias_ptr = NULL;
1103
- st->fiber_axis = fiber_ax;
1104
- for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
1105
- st->fiber_native_strides[kk] = row_byte_strides[kk];
1106
- }
1107
-
1108
- /* Row-major element strides for mask (= identity element
1109
- index strides over src->dim). */
1110
- ca_size_t row_elem_strides[CA_RANK_MAX];
1111
- {
1112
- ca_size_t e = 1;
1113
- for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
1114
- row_elem_strides[kk] = e;
1115
- e *= src->dim[kk];
1116
- }
1117
- }
1118
-
1119
- int8_t sp = 0, op = 0;
1120
- st->slab_elements = 1;
1121
- for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
1122
- if ( in_slab[kk] ) {
1123
- st->slab_axes_buf[sp] = kk;
1124
- st->slab_dims[sp] = src->dim[kk];
1125
- st->slab_strides[sp] = row_byte_strides[kk];
1126
- st->slab_mask_strides[sp] = row_elem_strides[kk];
1127
- st->slab_elements *= src->dim[kk];
1128
- sp++;
1129
- } else {
1130
- st->outer_axes[op] = kk;
1131
- st->outer_dims[op] = src->dim[kk];
1132
- st->outer_strides[op] = row_byte_strides[kk];
1133
- st->outer_mask_strides[op] = row_elem_strides[kk];
1134
- op++;
1135
- }
1136
- }
1137
- st->slab_ndim = sp;
1138
- st->outer_ndim = op;
1139
- st->desc_ndim = 0;
1140
-
1141
- ca_size_t total = 1;
1142
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1143
- st->total_slabs = total;
1144
- st->slab_n = st->slab_elements;
1145
- st->slabs_emitted = 0;
1146
- st->chunk_size = st->slab_n;
1147
-
1148
- if ( st->outer_ndim > 0 ) {
1149
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1150
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1151
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1152
- } else {
1153
- st->outer_idx = NULL;
1154
- }
1155
-
1156
- /* Mask: per-fiber gather via next_slab_axes; no whole-view
1157
- materialise (rev2 §3.3, (data,mask) pair travels together). */
1158
- st->alias_mask = NULL;
1159
- return CA_ITER_OK;
1160
- }
1161
- }
1162
-
1163
- /* Phase C T3 (C.1): slab axis has INDEX or SHIFT kind → per-slab
1164
- materialise fallback path (D1.1 (B) + D1.2 (A)). CAMapping-style
1165
- views (raw_ndim != src->ndim) are out of C.1 scope — descriptor
1166
- exposes a flat 1-D INDEX gather that doesn't expose the view-
1167
- axis partition the user gave via slab_axes. Reject explicitly. */
1168
- if ( raw_ndim != src->ndim ) {
1169
- return CA_ITER_ERR_POLICY;
1170
- }
1171
- int slab_has_non_stride = 0;
1172
- int slab_has_index = 0;
1173
- for ( k = 0; k < src->ndim; k++ ) {
1174
- if ( in_slab[k] && raw_descs[k].kind != CA_AXIS_KIND_STRIDE ) {
1175
- slab_has_non_stride = 1;
1176
- if ( raw_descs[k].kind == CA_AXIS_KIND_INDEX ) {
1177
- slab_has_index = 1;
1178
- }
1179
- }
1180
- }
1181
- /* A slab with a SHIFT axis that leaves out the innermost view axis
1182
- would be gathered one strided cell at a time, once per slab. Read
1183
- walks take the whole-view path below instead: the view is gathered
1184
- once, row by row, and the slabs are walked in the copy. */
1185
- int whole_view = slab_has_non_stride && ! slab_has_index
1186
- && ! in_slab[src->ndim - 1]
1187
- && ! ( flags & CA_KERNEL_WRITE )
1188
- && ca_fast_paths_enabled();
1189
- if ( slab_has_non_stride && ! whole_view ) {
1190
- /* C.1 scope: READ-only. WRITE = future sub-step (C.1c). */
1191
- if ( flags & CA_KERNEL_WRITE ) {
1192
- return CA_ITER_ERR_FLAGS;
1193
- }
1194
-
1195
- st->src = src;
1196
- st->src_kind = CA_ITER_SRC_DESCRIPTOR;
1197
- st->level = 2;
1198
- st->policy = policy;
1199
- st->ndim = src->ndim;
1200
- st->flags = flags;
1201
- st->bytes = src->bytes;
1202
- st->naxes = naxes;
1203
-
1204
- CArray *parent = CAVIEW(src)->parent;
1205
- ca_attach(parent);
1206
- st->root = parent;
1207
-
1208
- /* Persist raw_descs / pdims / pstrides for per-slab subset
1209
- construction (PER_SLAB fallback) or for the hoisted manual
1210
- gather (PER_SLAB_HOIST specialised path). desc_ndim =
1211
- src->ndim so has_descs branch in next_slab_axes can read
1212
- st->descs (though T3 path takes its own branch before that
1213
- check). */
1214
- memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
1215
- memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
1216
- {
1217
- ca_size_t s = src->bytes;
1218
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1219
- st->pstrides[k] = s;
1220
- s *= raw_pdims[k];
1221
- }
1222
- }
1223
- st->desc_ndim = src->ndim;
1224
-
1225
- /* View-row-major element strides (= mask scratch layout, since
1226
- mask is gathered whole-view once at init). */
1227
- ca_size_t row_elem_strides[CA_RANK_MAX];
1228
- {
1229
- ca_size_t e = 1;
1230
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1231
- row_elem_strides[k] = e;
1232
- e *= src->dim[k];
1233
- }
1234
- }
1235
-
1236
- /* Row-major byte strides over SLAB axes only (= scratch_ptr
1237
- layout, since scratch_ptr is sized slab_elements × bytes
1238
- and refilled per outer iter as a contig row-major slab). */
1239
- ca_size_t slab_data_strides[CA_RANK_MAX];
1240
- {
1241
- ca_size_t b = src->bytes;
1242
- ca_size_t e_slab = 1;
1243
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1244
- if ( in_slab[k] ) {
1245
- slab_data_strides[k] = b;
1246
- b *= raw_descs[k].count;
1247
- e_slab *= raw_descs[k].count;
1248
- } else {
1249
- slab_data_strides[k] = 0; /* unused for outer axes */
1250
- }
1251
- }
1252
- st->slab_elements = e_slab;
1253
- }
1254
-
1255
- int8_t sp = 0, op = 0;
1256
- for ( k = 0; k < src->ndim; k++ ) {
1257
- if ( in_slab[k] ) {
1258
- st->slab_axes_buf[sp] = k;
1259
- st->slab_dims[sp] = raw_descs[k].count;
1260
- st->slab_strides[sp] = slab_data_strides[k];
1261
- /* Slab mask stride is view-row-major elem stride along
1262
- this view axis (mask scratch is whole-view layout). */
1263
- st->slab_mask_strides[sp] = row_elem_strides[k];
1264
- sp++;
1265
- } else {
1266
- st->outer_axes[op] = k;
1267
- st->outer_dims[op] = raw_descs[k].count;
1268
- /* T3 path: outer data offset is always 0 (scratch refilled
1269
- per slab, ptr = scratch_ptr). */
1270
- st->outer_strides[op] = 0;
1271
- /* Outer mask offset uses view-row-major elem stride into
1272
- whole-view mask scratch. */
1273
- st->outer_mask_strides[op] = row_elem_strides[k];
1274
- op++;
1275
- }
1276
- }
1277
- st->slab_ndim = sp;
1278
- st->outer_ndim = op;
1279
- st->composed_base = 0; /* unused for T3 fallback */
1280
-
1281
- ca_size_t total = 1;
1282
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1283
- st->total_slabs = total;
1284
- st->slab_n = st->slab_elements;
1285
- st->slabs_emitted = 0;
1286
- st->chunk_size = st->slab_n;
1287
-
1288
- if ( st->outer_ndim > 0 ) {
1289
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1290
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1291
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1292
- } else {
1293
- st->outer_idx = NULL;
1294
- }
1295
-
1296
- /* Alloc per-slab data scratch (D1.1 (B): one max-slab buffer
1297
- refilled per outer iter, instead of per-iter alloc/free or
1298
- full-view materialise). */
1299
- ca_size_t slab_bytes = st->slab_elements * src->bytes;
1300
- st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
1301
- st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1302
- st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
1303
-
1304
- /* Specialisation (B-1b, C.1b): if innermost slab axis is STRIDE
1305
- and no SHIFT axes anywhere in the view, use the hoisted
1306
- manual gather path (= outer + non-innermost-slab hoist,
1307
- inner pure STRIDE linear memcpy). Else (innermost INDEX /
1308
- SHIFT-anywhere): use the engine fallback (A). */
1309
- int specialised_eligible = (sp > 0) && ca_fast_paths_enabled();
1310
- if ( specialised_eligible ) {
1311
- int8_t inner_view_ax = st->slab_axes_buf[sp - 1];
1312
- if ( raw_descs[inner_view_ax].kind != CA_AXIS_KIND_STRIDE ) {
1313
- specialised_eligible = 0;
1314
- }
1315
- }
1316
- if ( specialised_eligible ) {
1317
- /* Any SHIFT axis (slab or outer) → fallback to engine for
1318
- clean bound_fill semantics in C.1b scope. */
1319
- for ( k = 0; k < src->ndim; k++ ) {
1320
- if ( raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
1321
- specialised_eligible = 0;
1322
- break;
1323
- }
1324
- }
1325
- }
1326
- st->alias_mode = specialised_eligible
1327
- ? CA_ITER_ALIAS_PER_SLAB_HOIST
1328
- : CA_ITER_ALIAS_PER_SLAB;
1329
-
1330
- /* Mask: gather whole view mask once into scratch_mask (= same
1331
- strategy as B.1.5). Per-slab mask offset is computed via
1332
- outer_idx × outer_mask_strides in next_slab_axes. */
1333
- if ( ca_has_mask(src) ) {
1334
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1335
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1336
- if ( src->elements > 0 ) {
1337
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1338
- }
1339
- st->alias_mask = st->scratch_mask;
1340
- }
1341
-
1342
- return CA_ITER_OK;
1343
- }
1344
- /* slab is all-STRIDE: fall through to B.1.5 / Phase B paths. */
1345
- /* Phase B.1.5: outer SHIFT axis → materialise downgrade. Alias
1346
- can't deliver OOB cells (= need fill_value), and the view-family
1347
- surface prioritises delivering the cells over avoiding a copy, so
1348
- we materialise the entire view into a row-major scratch
1349
- buffer via ca_axis_dispatch_attach (= same engine as the
1350
- existing SRC_DESCRIPTOR L2 NONE path), then walk it with
1351
- Phase A-style row-major K-D strides. Slab / outer partition
1352
- applies to view-axes; the scratch IS the flat row-major view. */
1353
- int outer_has_shift = 0;
1354
- for ( k = 0; k < src->ndim; k++ ) {
1355
- if ( !in_slab[k] && raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
1356
- outer_has_shift = 1;
1357
- break;
1358
- }
1359
- }
1360
-
1361
- if ( outer_has_shift || whole_view ) {
1362
- st->src = src;
1363
- st->src_kind = CA_ITER_SRC_DESCRIPTOR;
1364
- st->level = 2;
1365
- st->policy = policy;
1366
- st->ndim = src->ndim;
1367
- st->flags = flags;
1368
- st->bytes = src->bytes;
1369
- st->naxes = naxes;
1370
-
1371
- CArray *parent = CAVIEW(src)->parent;
1372
- ca_attach(parent);
1373
- st->root = parent;
1374
-
1375
- ca_axis_desc_t local_descs[CA_RANK_MAX];
1376
- memcpy(local_descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
1377
- const void *bound_fill = NULL;
1378
- if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
1379
- bound_fill = ((CAWindow *) src)->fill;
1380
- }
1381
- st->scratch_cap = src->elements * src->bytes;
1382
- st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1383
- ca_axis_dispatch_gather(parent, raw_pdims, local_descs, src->ndim,
1384
- src->bytes, src->elements, bound_fill,
1385
- st->scratch_ptr);
1386
- /* sync_slab scatters the whole scratch back by this count. */
1387
- st->total_elements = src->elements;
1388
- st->alias_mode = CA_ITER_ALIAS_NONE;
1389
- st->alias_ptr = st->scratch_ptr;
1390
-
1391
- /* Row-major byte / element strides on the scratch (= view layout). */
1392
- ca_size_t row_byte_strides[CA_RANK_MAX];
1393
- ca_size_t row_elem_strides[CA_RANK_MAX];
1394
- {
1395
- ca_size_t b = src->bytes, e = 1;
1396
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1397
- row_byte_strides[k] = b;
1398
- row_elem_strides[k] = e;
1399
- b *= src->dim[k];
1400
- e *= src->dim[k];
1401
- }
1402
- }
1403
-
1404
- int8_t sp = 0, op = 0;
1405
- st->slab_elements = 1;
1406
- for ( k = 0; k < src->ndim; k++ ) {
1407
- if ( in_slab[k] ) {
1408
- st->slab_axes_buf[sp] = k;
1409
- st->slab_dims[sp] = src->dim[k];
1410
- st->slab_strides[sp] = row_byte_strides[k];
1411
- st->slab_mask_strides[sp] = row_elem_strides[k];
1412
- st->slab_elements *= src->dim[k];
1413
- sp++;
1414
- } else {
1415
- st->outer_axes[op] = k;
1416
- st->outer_dims[op] = src->dim[k];
1417
- st->outer_strides[op] = row_byte_strides[k];
1418
- st->outer_mask_strides[op] = row_elem_strides[k];
1419
- op++;
1420
- }
1421
- }
1422
- st->slab_ndim = sp;
1423
- st->outer_ndim = op;
1424
- st->composed_base = 0;
1425
- /* desc_ndim = 0: next_slab_axes treats outer like Phase A
1426
- (no INDEX lookup), correct because scratch is row-major
1427
- contig (= STRIDE everywhere). */
1428
- st->desc_ndim = 0;
1429
-
1430
- ca_size_t total = 1;
1431
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1432
- st->total_slabs = total;
1433
- st->slab_n = st->slab_elements;
1434
- st->slabs_emitted = 0;
1435
- st->chunk_size = st->slab_n;
1436
-
1437
- if ( st->outer_ndim > 0 ) {
1438
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1439
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1440
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1441
- } else {
1442
- st->outer_idx = NULL;
1443
- }
1444
-
1445
- if ( ca_has_mask(src) ) {
1446
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1447
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1448
- if ( src->elements > 0 ) {
1449
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1450
- }
1451
- st->alias_mask = st->scratch_mask;
1452
- }
1453
- return CA_ITER_OK;
1454
- }
1080
+ /* Splits the view's axes into slab and outer axes (both in ascending axis
1081
+ order, so the user's order does not matter), counts the slabs and opens
1082
+ the outer cursor. */
1083
+ static void
1084
+ ki_lay_out_slabs (ca_iter_state *st, CArray *src, const int8_t *in_slab,
1085
+ const ki_axis_layout *lay)
1086
+ {
1087
+ int8_t sp = 0, op = 0;
1088
+ st->slab_elements = 1;
1089
+ for ( int8_t k = 0; k < src->ndim; k++ ) {
1090
+ if ( in_slab[k] ) {
1091
+ st->slab_axes_buf[sp] = k;
1092
+ st->slab_dims[sp] = lay->dims[k];
1093
+ st->slab_strides[sp] = lay->slab_strides[k];
1094
+ st->slab_mask_strides[sp] = lay->mask_strides[k];
1095
+ st->slab_elements *= lay->dims[k];
1096
+ sp++;
1097
+ } else {
1098
+ st->outer_axes[op] = k;
1099
+ st->outer_dims[op] = lay->dims[k];
1100
+ st->outer_strides[op] = lay->outer_strides[k];
1101
+ st->outer_mask_strides[op] = lay->mask_strides[k];
1102
+ op++;
1103
+ }
1104
+ }
1105
+ st->slab_ndim = sp;
1106
+ st->outer_ndim = op;
1455
1107
 
1456
- st->src = src;
1457
- st->src_kind = CA_ITER_SRC_DESCRIPTOR; /* Phase B alias path */
1458
- st->level = 2;
1459
- st->policy = policy;
1460
- st->ndim = src->ndim;
1461
- st->flags = flags;
1462
- st->bytes = src->bytes;
1463
- st->naxes = naxes;
1464
-
1465
- /* Parent row-major byte strides for offset computation. */
1466
- ca_size_t pstrides[CA_RANK_MAX];
1467
- {
1468
- ca_size_t s = src->bytes;
1469
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1470
- pstrides[k] = s;
1471
- s *= raw_pdims[k];
1472
- }
1473
- }
1474
- /* View row-major element strides (= mask scratch layout). */
1475
- ca_size_t row_elem_strides[CA_RANK_MAX];
1476
- {
1477
- ca_size_t s = 1;
1478
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1479
- row_elem_strides[k] = s;
1480
- s *= src->dim[k];
1481
- }
1482
- }
1108
+ ca_size_t total = 1;
1109
+ for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1110
+ st->total_slabs = total;
1111
+ st->slab_n = st->slab_elements;
1112
+ st->slabs_emitted = 0;
1113
+ st->chunk_size = st->slab_n;
1483
1114
 
1484
- /* Partition + populate slab / outer metadata. Slab axes contribute
1485
- to a constant slab_base (since they're all STRIDE: start +
1486
- step*idx, where the base is start*pstride summed). Outer axes
1487
- drive per-slab offset via direct (start + step*idx)*pstride or
1488
- indices[idx]*pstride (= no classify_prefix engine, since outer
1489
- axes here are not a prefix of raw_descs[] — they're a
1490
- complement of slab_axes). */
1491
- int8_t sp = 0, op = 0;
1492
- ca_size_t slab_base = 0;
1493
- st->slab_elements = 1;
1494
- for ( k = 0; k < src->ndim; k++ ) {
1495
- if ( in_slab[k] ) {
1496
- /* STRIDE-kind slab axis: start*pstride goes to slab_base,
1497
- step*pstride is the per-cell byte stride. */
1498
- slab_base += raw_descs[k].start * pstrides[k];
1499
- st->slab_axes_buf[sp] = k;
1500
- st->slab_dims[sp] = raw_descs[k].count;
1501
- st->slab_strides[sp] = raw_descs[k].step * pstrides[k];
1502
- st->slab_mask_strides[sp] = row_elem_strides[k];
1503
- st->slab_elements *= raw_descs[k].count;
1504
- sp++;
1505
- } else {
1506
- /* Outer axis: STRIDE or INDEX (SHIFT was rejected above).
1507
- Store axis index + view's outer dim/stride for descriptor
1508
- walk in next_slab_axes. We reuse outer_strides/outer_mask
1509
- _strides as the OUTER walk metadata; outer-axis kind is
1510
- carried implicitly via raw_descs[outer_axes[m]] in the
1511
- state struct's descs[] field (populated below). */
1512
- st->outer_axes[op] = k;
1513
- st->outer_dims[op] = raw_descs[k].count;
1514
- /* For STRIDE: per-axis stride = step * pstride; INDEX axes
1515
- use indices[] from raw_descs and need a 0 stride here
1516
- (next_slab_axes branches on descs[].kind to compute the
1517
- real offset). We persist the full raw_descs in st->descs
1518
- so the next_slab_axes implementation can dispatch. */
1519
- if ( raw_descs[k].kind == CA_AXIS_KIND_STRIDE ) {
1520
- st->outer_strides[op] = raw_descs[k].step * pstrides[k];
1521
- /* Bake in start*pstride into slab_base for STRIDE outer too
1522
- so per-cell offset is purely step*idx. */
1523
- slab_base += raw_descs[k].start * pstrides[k];
1524
- } else {
1525
- /* INDEX kind: outer_strides[m] = pstrides[k] (element-unit
1526
- from indices[]). We DON'T pre-bake into slab_base for
1527
- INDEX axes; per-iter offset = indices[idx[m]] * pstrides[k]
1528
- is computed in next_slab_axes. */
1529
- st->outer_strides[op] = pstrides[k];
1530
- }
1531
- st->outer_mask_strides[op] = row_elem_strides[k];
1532
- op++;
1533
- }
1534
- }
1535
- st->slab_ndim = sp;
1536
- st->outer_ndim = op;
1537
- st->composed_base = slab_base;
1538
-
1539
- /* Persist raw_descs / pstrides in state so next_slab_axes can
1540
- distinguish STRIDE vs INDEX outer per axis. */
1541
- memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
1542
- memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
1543
- memcpy(st->pstrides, pstrides, src->ndim * sizeof(ca_size_t));
1544
- st->desc_ndim = src->ndim;
1545
-
1546
- ca_size_t total = 1;
1547
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1548
- st->total_slabs = total;
1549
- st->slab_n = st->slab_elements;
1550
- st->slabs_emitted = 0;
1551
- st->chunk_size = st->slab_n;
1552
-
1553
- if ( st->outer_ndim > 0 ) {
1554
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1555
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1556
- } else {
1557
- st->outer_idx = NULL;
1558
- }
1115
+ if ( st->outer_ndim > 0 ) {
1116
+ CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1117
+ st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1118
+ for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1119
+ } else {
1120
+ st->outer_idx = NULL;
1121
+ }
1122
+ }
1559
1123
 
1560
- /* Attach parent for alias path. */
1561
- CArray *parent = CAVIEW(src)->parent;
1562
- ca_attach(parent);
1563
- st->root = parent;
1564
- st->alias_mode = CA_ITER_ALIAS_STRIDED;
1565
- st->alias_ptr = (char *) parent->ptr;
1566
-
1567
- /* Mask gather (= view row-major boolean8_t). */
1568
- if ( ca_has_mask(src) ) {
1569
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1570
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1571
- if ( src->elements > 0 ) {
1572
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1573
- }
1574
- st->alias_mask = st->scratch_mask;
1575
- }
1124
+ /* Chooses the route. Decides only: nothing is attached or allocated, so
1125
+ a refusal leaves nothing to close. */
1126
+ static int
1127
+ ki_choose_l2_route (CArray *src, int8_t *axes, int8_t naxes, uint32_t flags,
1128
+ ki_l2_plan *plan)
1129
+ {
1130
+ const int8_t *in_slab = plan->in_slab;
1131
+ ca_size_t row_bytes[CA_RANK_MAX];
1132
+ int8_t k;
1133
+
1134
+ ki_view_row_major(src, row_bytes, NULL);
1135
+
1136
+ if ( plan->src_kind == CA_ITER_SRC_DESCRIPTOR
1137
+ || plan->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
1138
+ /* Shortcut PER_FIBER_FUSED (CAWindow / CAShift).
1139
+ Layer all three, per fiber: each fiber is fetched with the
1140
+ view's ca_xfer_stride (which fills out-of-bounds
1141
+ cells itself) and put back the same way.
1142
+ Speeds up a fiber walk of a CAWindow / CAShift whose innermost
1143
+ axis is SHIFT.
1144
+ Why none measured: about 8% slower than the descriptor
1145
+ route on a sort of a 2000x2000 shift view.
1146
+ Enters CA_KERNEL_FIBER_CONTIG, one slab axis, the view's
1147
+ own rank, and ca_iter_should_per_fiber_fused. An
1148
+ outer-only SHIFT arrives as L2_ALIASABLE and never
1149
+ reaches this hook.
1150
+ Promises 1 (the fiber scratch is contiguous), 2 (the mask is
1151
+ fetched into the fiber mask scratch), 3 (ca_xfer_stride
1152
+ PUT of a fiber that neither converts nor repeats a
1153
+ cell), 4 (CA_ITER_FLAG_WALK_MAY_RAISE).
1154
+ Switch ca_fast_paths_enabled(): off, the descriptor route.
1155
+ Checked by the fiber rows of the "shift" view in the delivery
1156
+ matrix. */
1157
+ if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
1158
+ && plan->ndesc == src->ndim && ca_fast_paths_enabled()
1159
+ && ca_iter_should_per_fiber_fused(src, plan->src_kind, axes[0],
1160
+ row_bytes[axes[0]], flags) ) {
1161
+ plan->route = KI_L2_FIBER_FUSED;
1576
1162
  return CA_ITER_OK;
1577
1163
  }
1578
- if ( src_kind == CA_ITER_SRC_ATTACH ) {
1579
- /* Phase B.5: CA_SLAB_AXES + SRC_ATTACH (= CAFake / CAByteSwap /
1580
- CABitfield / CABitarray / CAReduce). These overlay views
1581
- materialise via their own ca_attach (= no descriptor framework
1582
- path), so we let ca_attach(src) populate src->ptr in view
1583
- row-major layout and walk it with Phase A-style K-D strides.
1584
-
1585
- The most common entry path here is wrap_readonly(int_src,
1586
- CA_FLOAT64) → CAFake, enabling sum_ki etc. to accept any
1587
- numeric data_type source via auto-cast (= D2.2 confirmed in
1588
- rev2 sparring). */
1589
- if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
1590
- return CA_ITER_ERR_POLICY;
1591
- }
1592
- int8_t in_slab[CA_RANK_MAX];
1593
- int8_t k;
1594
- for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
1595
- for ( k = 0; k < naxes; k++ ) {
1596
- int8_t ax = axes[k];
1597
- if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
1598
- if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
1599
- in_slab[ax] = 1;
1600
- }
1601
1164
 
1602
- st->src = src;
1603
- st->src_kind = CA_ITER_SRC_ATTACH;
1604
- st->level = 2;
1605
- st->policy = policy;
1606
- st->ndim = src->ndim;
1607
- st->flags = flags;
1608
- st->bytes = src->bytes;
1609
- st->naxes = naxes;
1610
-
1611
- /* Row-major byte / element strides on the attached view buffer.
1612
- Computed first so the F.6.1 predicate can inspect fiber-axis
1613
- effective stride before deciding whether to materialise. */
1614
- ca_size_t row_byte_strides[CA_RANK_MAX];
1615
- ca_size_t row_elem_strides[CA_RANK_MAX];
1616
- {
1617
- ca_size_t b = src->bytes, e = 1;
1618
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1619
- row_byte_strides[k] = b;
1620
- row_elem_strides[k] = e;
1621
- b *= src->dim[k];
1622
- e *= src->dim[k];
1623
- }
1165
+ /* The rank check guards a descriptor whose rank differs from the
1166
+ view's (it would not expose the axis partition slab_axes names).
1167
+ Every describe_axes in the engine reports the view's rank, so the
1168
+ check does not fire today; it stays as the contract's guard. */
1169
+ if ( plan->ndesc != src->ndim ) return CA_ITER_ERR_POLICY;
1170
+
1171
+ int slab_has_non_stride = 0;
1172
+ int slab_has_index = 0;
1173
+ int outer_has_shift = 0;
1174
+ for ( k = 0; k < src->ndim; k++ ) {
1175
+ if ( in_slab[k] && plan->descs[k].kind != CA_AXIS_KIND_STRIDE ) {
1176
+ slab_has_non_stride = 1;
1177
+ if ( plan->descs[k].kind == CA_AXIS_KIND_INDEX ) slab_has_index = 1;
1624
1178
  }
1625
-
1626
- /* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev4 (direct per-
1627
- parent ptr access path + tile cache). Engages when:
1628
- - source.attach is ca_stack_func.attach (= CAStack identity)
1629
- - no mask (= horizontal mask propagation per-slab gather is
1630
- out of scope; falls back to SRC_ATTACH whole-view path)
1631
- - naxes == 1 && axes[0] == k_axis (= K-axis-only slab, the
1632
- demand-driving case = reduce along the stacked axis like
1633
- view.mean(axis: k_axis))
1634
- init attaches K parents up front (O(1) per entity parent),
1635
- caches parent->ptr aliases + uniform parent-native byte strides,
1636
- allocates a slab-sized scratch (= K * bytes) plus a tile cache
1637
- sized to fit the L1d budget. next_slab_axes refills TILE fibers
1638
- per K contig parent reads (rev4 2026-06-19) and serves the next
1639
- TILE-1 calls from the L1d-resident cache, beating SRC_ATTACH
1640
- across all measured sizes. No size-threshold gate: rev3's gate
1641
- was a perf trade-off justification that the tile cache erased.
1642
-
1643
- K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): engage predicate
1644
- generalised from `axes[0] == 0` to `axes[0] == k_axis`. The
1645
- tile cache mechanism + outer_idx -> parent axis mapping are
1646
- already k_axis-agnostic: outer_idx[m] maps to parent axis m
1647
- regardless of k_axis position (= for k_axis = 0 outer is stack
1648
- axes 1..N-1 = parent axes 0..N-2; for k_axis > 0 outer is stack
1649
- axes [0..k_axis-1, k_axis+1..N-1] = parent axes [0..k_axis-1,
1650
- k_axis..N-2]; either way outer_idx[m] = parent axis m). */
1651
- if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1652
- && !ca_has_mask(src)
1653
- && !(flags & CA_KERNEL_WRITE)
1654
- && naxes == 1 && axes[0] == ((CAStack *) src)->k_axis
1655
- && ca_fast_paths_enabled() ) {
1656
- CAStack *stack = (CAStack *) src;
1657
- int8_t parent_ndim = src->ndim - 1;
1658
- int8_t sp = 0, op = 0;
1659
-
1660
- st->slab_elements = 1;
1661
- for ( k = 0; k < src->ndim; k++ ) {
1662
- if ( in_slab[k] ) {
1663
- st->slab_axes_buf[sp] = k;
1664
- st->slab_dims[sp] = src->dim[k];
1665
- st->slab_elements *= src->dim[k];
1666
- sp++;
1667
- } else {
1668
- st->outer_axes[op] = k;
1669
- st->outer_dims[op] = src->dim[k];
1670
- st->outer_strides[op] = row_byte_strides[k]; /* unused on STACK path */
1671
- st->outer_mask_strides[op] = row_elem_strides[k]; /* unused on STACK path */
1672
- op++;
1673
- }
1674
- }
1675
- st->slab_ndim = sp;
1676
- st->outer_ndim = op;
1677
- st->composed_base = 0;
1678
- st->desc_ndim = 0;
1679
- /* slab_strides on STACK path: slab_axes == [0], scratch is
1680
- packed contig over K elements (= K * bytes), so single-axis
1681
- slab walk uses bytes stride. */
1682
- st->slab_strides[0] = src->bytes;
1683
- st->slab_mask_strides[0] = 1; /* harmless on no-mask path */
1684
-
1685
- ca_size_t total = 1;
1686
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1687
- st->total_slabs = total;
1688
- st->slab_n = st->slab_elements;
1689
- st->slabs_emitted = 0;
1690
- st->chunk_size = st->slab_n;
1691
-
1692
- if ( st->outer_ndim > 0 ) {
1693
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1694
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1695
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1696
- } else {
1697
- st->outer_idx = NULL;
1698
- }
1699
-
1700
- /* Attach K parents and cache their ptr aliases. CAStack
1701
- guarantees uniform shape (MEMO §3.2) so one set of parent
1702
- native byte strides covers all K (= parents[0]->dim is
1703
- canonical). */
1704
- /* stack_n_parents counts the parents attached so far: an attach
1705
- that raises leaves finish with exactly those to detach. */
1706
- st->stack_n_parents = 0;
1707
- st->stack_parent_ptrs =
1708
- (char **) xmalloc(stack->n_parents * sizeof(char *));
1709
- for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1710
- ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
1711
- st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
1712
- st->stack_n_parents = kk + 1;
1713
- }
1714
- {
1715
- ca_size_t s = src->bytes;
1716
- for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
1717
- st->stack_parent_strides[kk] = s;
1718
- s *= stack->parents[0]->dim[kk];
1719
- }
1720
- }
1721
-
1722
- /* Slab-sized scratch (= K * bytes for slab_axes == [0]). */
1723
- st->scratch_cap = (ca_size_t) st->slab_elements * src->bytes;
1724
- st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1725
-
1726
- /* Tile cache (pilot/castack-axis0-loop-interchange): refill TILE
1727
- fibers per K parent reads, serve next TILE-1 fibers from L1d.
1728
- TILE budget = ~32 KB (half of M2 L1d). Empirically tested
1729
- values; clamped to [8, 64] to keep the refill loop tight. */
1730
- {
1731
- ca_size_t K = (ca_size_t) stack->n_parents;
1732
- ca_size_t bytes = (ca_size_t) src->bytes;
1733
- ca_size_t tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
1734
- if ( tile < 8 ) tile = 8;
1735
- if ( tile > 64 ) tile = 64;
1736
- st->stack_tile_cap = tile;
1737
- st->stack_tile_pos = 0;
1738
- st->stack_tile_have = 0; /* force refill on first call */
1739
- st->stack_tile_cache = (char *) ki_data_alloc(src->data_type, K * tile * bytes);
1740
- }
1741
-
1742
- st->alias_mode = CA_ITER_ALIAS_STACK;
1743
- st->alias_ptr = NULL; /* no whole-view buffer */
1744
- st->alias_mask = NULL;
1745
- return CA_ITER_OK;
1746
- }
1747
-
1748
- /* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18):
1749
- CAStack source + slab_axes excludes axis 0 (= K-axis stays in
1750
- outer iter, e.g. view.mean(axis: 1), view.mean(axis: 2),
1751
- view.mean(axis: 1, 2)). Each slab corresponds to a region
1752
- inside ONE parent selected by outer_idx[K_outer_pos]. Engine
1753
- pre-attaches K parents (+ K parent masks if mask present),
1754
- caches their ptrs, and next_slab_axes aliases parents[k]->ptr +
1755
- parent_off directly -- no scratch, no materialise, parent
1756
- entity case = eager-equivalent memory bandwidth.
1757
-
1758
- Layering: engine detects CAStack identity by function-pointer
1759
- comparison and accesses parents[] / mask via ca_func[STACK]
1760
- operation table where possible. CAStack downcast occurs only
1761
- for the n_parents / parents[] read in init_l2 (= same scope as
1762
- rev3 STACK; AC3 forbids parents[] walk in next_slab_axes only).
1763
- Q1 disposition: option (i) minimal-diff inline in SLAB_AXES
1764
- branch -- evaluated as smallest diff with acceptable layering
1765
- (proposal §3.1). */
1766
- /* STACK_OUTER_K: K-axis stays in outer iter (slab carved out of
1767
- one parent at a time). K.3 (2026-06-20): engage predicate +
1768
- stack-axis -> parent-axis mapping generalised for arbitrary
1769
- k_axis. stack axis s maps to parent axis (s if s < k_axis else
1770
- s - 1); only the K stack axis (s == k_axis) has no parent
1771
- counterpart. */
1772
- if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1773
- && !(flags & CA_KERNEL_WRITE)
1774
- && naxes >= 1 && !in_slab[((CAStack *) src)->k_axis]
1775
- && ca_fast_paths_enabled() ) {
1776
- CAStack *stack = (CAStack *) src;
1777
- int8_t k_axis = stack->k_axis;
1778
- int8_t parent_ndim = src->ndim - 1;
1779
- int8_t sp = 0, op = 0;
1780
-
1781
- st->slab_elements = 1;
1782
- for ( k = 0; k < src->ndim; k++ ) {
1783
- if ( in_slab[k] ) {
1784
- st->slab_axes_buf[sp] = k;
1785
- st->slab_dims[sp] = src->dim[k];
1786
- st->slab_elements *= src->dim[k];
1787
- /* slab_strides[sp] = parent native byte stride at parent
1788
- axis (k if k < k_axis else k - 1). k == k_axis cannot
1789
- appear here -- the engage predicate above excludes it. */
1790
- sp++;
1791
- } else {
1792
- st->outer_axes[op] = k;
1793
- st->outer_dims[op] = src->dim[k];
1794
- st->outer_strides[op] = row_byte_strides[k];
1795
- st->outer_mask_strides[op] = row_elem_strides[k];
1796
- op++;
1797
- }
1798
- }
1799
- st->slab_ndim = sp;
1800
- st->outer_ndim = op;
1801
- st->composed_base = 0;
1802
- st->desc_ndim = 0;
1803
-
1804
- /* Locate the K-axis (= stack axis k_axis) within outer_axes. */
1805
- st->stack_k_outer_pos = -1;
1806
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
1807
- if ( st->outer_axes[m] == k_axis ) {
1808
- st->stack_k_outer_pos = m;
1809
- break;
1810
- }
1811
- }
1812
-
1813
- /* Uniform parent native byte strides + parent element strides
1814
- for mask (= MEMO §3.2 uniform shape across parents,
1815
- parents[0]->dim is canonical). Mask strides cached so
1816
- next_slab_axes can compute mask_off without downcasting
1817
- (= AC3 layering goal). */
1818
- {
1819
- ca_size_t sb = src->bytes;
1820
- ca_size_t se = 1;
1821
- for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
1822
- st->stack_parent_strides[kk] = sb;
1823
- st->stack_parent_mask_strides[kk] = se;
1824
- sb *= stack->parents[0]->dim[kk];
1825
- se *= stack->parents[0]->dim[kk];
1826
- }
1827
- }
1828
-
1829
- /* Fill slab_strides + slab_mask_strides for slab axes (all of
1830
- which are parent inner axes since K is in outer). stack ax
1831
- -> parent ax: s if s < k_axis else s - 1. */
1832
- for ( int8_t s_i = 0; s_i < st->slab_ndim; s_i++ ) {
1833
- int8_t stack_ax = st->slab_axes_buf[s_i];
1834
- int8_t parent_ax = (stack_ax < k_axis) ? stack_ax : (stack_ax - 1);
1835
- st->slab_strides[s_i] = st->stack_parent_strides[parent_ax];
1836
- st->slab_mask_strides[s_i] = st->stack_parent_mask_strides[parent_ax];
1837
- }
1838
-
1839
- ca_size_t total = 1;
1840
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1841
- st->total_slabs = total;
1842
- st->slab_n = st->slab_elements;
1843
- st->slabs_emitted = 0;
1844
- st->chunk_size = st->slab_n;
1845
-
1846
- if ( st->outer_ndim > 0 ) {
1847
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1848
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1849
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1850
- } else {
1851
- st->outer_idx = NULL;
1852
- }
1853
-
1854
- /* Attach K parents + cache ptr aliases. Symmetric with rev3
1855
- Case B; for entity parents attach is O(1). */
1856
- /* stack_n_parents counts the parents attached so far: an attach
1857
- that raises leaves finish with exactly those to detach. */
1858
- st->stack_n_parents = 0;
1859
- st->stack_parent_ptrs =
1860
- (char **) xmalloc(stack->n_parents * sizeof(char *));
1861
- for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1862
- ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
1863
- st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
1864
- st->stack_n_parents = kk + 1;
1865
- }
1866
-
1867
- /* If CAStack carries mask (= horizontal propagation already
1868
- applied at create_mask), copy the K parent masks as they are
1869
- at the start of the walk; next_slab_axes points into the copy
1870
- of parent k at mask_off. A copy, as on every other path: the
1871
- cursor is the walk's own, and a write to it reaches nothing. */
1872
- if ( ca_has_mask(src) ) {
1873
- /* The stack's mask may still read stand-ins for parents that
1874
- have gained a mask since; read through to those. */
1875
- ca_multi_parent_mask_refresh(src->mask, stack->parents,
1876
- stack->mask_standins,
1877
- stack->n_parents, 0);
1878
- /* Zero-filled: a slot is set once its copy is made, and finish
1879
- frees the copies whose slot is set. */
1880
- st->stack_parent_mask_ptrs =
1881
- (boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
1882
- memset(st->stack_parent_mask_ptrs, 0,
1883
- stack->n_parents * sizeof(boolean8_t *));
1884
- for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1885
- CArray *pm = ((CAStack *) src->mask)->parents[kk];
1886
- ca_size_t n = pm->elements;
1887
- boolean8_t *copy = (boolean8_t *) xmalloc(n > 0 ? n : 1);
1888
- st->stack_parent_mask_ptrs[kk] = copy;
1889
- ca_attach(pm); /* window: only a memcpy before the detach */
1890
- memcpy(copy, pm->ptr, n);
1891
- ca_detach(pm);
1892
- }
1893
- } else {
1894
- st->stack_parent_mask_ptrs = NULL;
1895
- }
1896
-
1897
- st->scratch_ptr = NULL; /* no scratch: aliasing parent->ptr */
1898
- st->scratch_cap = 0;
1899
- st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
1900
- st->alias_ptr = NULL;
1901
- st->alias_mask = NULL;
1902
- return CA_ITER_OK;
1903
- }
1904
-
1905
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 hook: try per-fiber fused
1906
- dispatch before whole-view materialise. When predicate fires,
1907
- skip scratch alloc + xfer_all GET (= per-fiber ca_xfer_stride
1908
- is called on demand in next_slab_axes). Predicate is stubbed
1909
- in F.6.1 (returns 0); F.6.2+ enables specific source kinds. */
1910
- int per_fiber_fused = 0;
1911
- if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
1912
- && ca_fast_paths_enabled() ) {
1913
- int8_t fiber_ax = axes[0];
1914
- ca_size_t fiber_ax_stride = row_byte_strides[fiber_ax];
1915
- if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
1916
- fiber_ax_stride, flags) ) {
1917
- per_fiber_fused = 1;
1918
- st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
1919
- st->alias_ptr = NULL; /* no whole-view buffer */
1920
- st->fiber_axis = fiber_ax;
1921
- for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
1922
- st->fiber_native_strides[kk] = row_byte_strides[kk];
1923
- }
1924
- }
1925
- }
1926
-
1927
- if ( !per_fiber_fused ) {
1928
- /* 2026-05-31 refactor: iter-owns scratch via ca_xfer_all (was
1929
- ca_attach(src) + alias src->ptr). */
1930
- ki_gather_view(st, src, flags, 0);
1931
- st->alias_mode = CA_ITER_ALIAS_NONE;
1932
- st->alias_ptr = st->scratch_ptr;
1179
+ if ( !in_slab[k] && plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) {
1180
+ outer_has_shift = 1;
1933
1181
  }
1182
+ }
1934
1183
 
1935
- int8_t sp = 0, op = 0;
1936
- st->slab_elements = 1;
1937
- for ( k = 0; k < src->ndim; k++ ) {
1938
- if ( in_slab[k] ) {
1939
- st->slab_axes_buf[sp] = k;
1940
- st->slab_dims[sp] = src->dim[k];
1941
- st->slab_strides[sp] = row_byte_strides[k];
1942
- st->slab_mask_strides[sp] = row_elem_strides[k];
1943
- st->slab_elements *= src->dim[k];
1944
- sp++;
1945
- } else {
1946
- st->outer_axes[op] = k;
1947
- st->outer_dims[op] = src->dim[k];
1948
- st->outer_strides[op] = row_byte_strides[k];
1949
- st->outer_mask_strides[op] = row_elem_strides[k];
1950
- op++;
1951
- }
1952
- }
1953
- st->slab_ndim = sp;
1954
- st->outer_ndim = op;
1955
- st->composed_base = 0;
1956
- st->desc_ndim = 0; /* row-major STRIDE everywhere */
1957
-
1958
- ca_size_t total = 1;
1959
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1960
- st->total_slabs = total;
1961
- st->slab_n = st->slab_elements;
1962
- st->slabs_emitted = 0;
1963
- st->chunk_size = st->slab_n;
1964
-
1965
- if ( st->outer_ndim > 0 ) {
1966
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1967
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1968
- } else {
1969
- st->outer_idx = NULL;
1184
+ /* A slab with a SHIFT axis that leaves out the innermost view axis
1185
+ would be gathered one strided cell at a time, once per slab. Read
1186
+ walks take the whole-view route instead: the view is gathered
1187
+ once, row by row, and the slabs are walked in the copy.
1188
+
1189
+ Shortcut SHIFT_WHOLE_VIEW.
1190
+ Layer data; the fiber layer (ki_fiber_contig_*) sits on top.
1191
+ Speeds up read walks of a CAWindow / CAShift whose slab has a
1192
+ SHIFT axis and not the innermost axis.
1193
+ Why about 1.5x on a median along axis 0 of a 2000x2000
1194
+ shift view.
1195
+ Enters a non-stride slab axis, no INDEX slab axis, the
1196
+ innermost axis outside the slab, a read walk.
1197
+ Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
1198
+ only when the view has one).
1199
+ Switch ca_fast_paths_enabled(): off, the per-slab gather.
1200
+ Checked by the slab-axis rows of the "shift" and "window" views
1201
+ in the delivery matrix. */
1202
+ int whole_view = slab_has_non_stride && ! slab_has_index
1203
+ && ! in_slab[src->ndim - 1]
1204
+ && ! ( flags & CA_KERNEL_WRITE )
1205
+ && ca_fast_paths_enabled();
1206
+
1207
+ /* The descriptor engine's gathers (per slab and whole view) read
1208
+ only. A write takes the transfer walk instead: the view is
1209
+ gathered whole and the changed cells go back through its own
1210
+ transfer (promise 3). */
1211
+ if ( (slab_has_non_stride && ! whole_view) || outer_has_shift ) {
1212
+ if ( flags & CA_KERNEL_WRITE ) {
1213
+ plan->src_kind = CA_ITER_SRC_ATTACH;
1214
+ goto by_transfer;
1970
1215
  }
1216
+ }
1217
+ plan->src_kind = CA_ITER_SRC_DESCRIPTOR;
1218
+ if ( slab_has_non_stride && ! whole_view ) {
1219
+ plan->route = KI_L2_PER_SLAB;
1220
+ }
1221
+ else if ( outer_has_shift || whole_view ) {
1222
+ plan->route = KI_L2_DESC_WHOLE;
1223
+ }
1224
+ else {
1225
+ plan->route = KI_L2_DESC_STRIDED;
1226
+ }
1227
+ return CA_ITER_OK;
1228
+ }
1971
1229
 
1972
- if ( ca_has_mask(src) ) {
1973
- if ( per_fiber_fused ) {
1974
- /* F.6.1 rev2 §3.3: mask travels with data per-fiber. Mask
1975
- buffer is delivered through fiber_mask_scratch by
1976
- next_slab_axes; no whole-view materialise. */
1977
- st->alias_mask = NULL;
1978
- } else {
1979
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1980
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1981
- if ( src->elements > 0 ) {
1982
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1983
- }
1984
- st->alias_mask = st->scratch_mask;
1985
- }
1986
- }
1230
+ if ( plan->src_kind == CA_ITER_SRC_ATTACH ) {
1231
+ by_transfer:
1232
+ /* Shortcut STACK.
1233
+ Layer data and fiber: the slab is cut contiguous from the K
1234
+ parents through a tile cache.
1235
+ Speeds up read walks of an unmasked CAStack whose one slab axis
1236
+ is the stack axis.
1237
+ Why about 2x on a mean along the stack axis of 4 parents
1238
+ of 500x2000.
1239
+ Enters a CAStack, no mask, a read walk, one slab axis that is
1240
+ k_axis.
1241
+ Promises 1 (the slab is contiguous), 6 (no mask exists to touch).
1242
+ Switch ca_fast_paths_enabled(): off, the transfer walk.
1243
+ Checked by the slab-axis and fiber rows of the stack parent along
1244
+ axis 0 in the delivery matrix. */
1245
+ if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1246
+ && !ca_has_mask(src)
1247
+ && !(flags & CA_KERNEL_WRITE)
1248
+ && naxes == 1 && axes[0] == ((CAStack *) src)->k_axis
1249
+ && ca_fast_paths_enabled() ) {
1250
+ plan->route = KI_L2_STACK;
1251
+ return CA_ITER_OK;
1252
+ }
1253
+ /* Shortcut STACK_OUTER_K.
1254
+ Layer data; the fiber layer sits on top.
1255
+ Speeds up read walks of a CAStack whose stack axis is an outer
1256
+ axis: each slab is cut from the one parent the outer
1257
+ index picks, without gathering the stack.
1258
+ Why about 11x on a mean along axis 1 of 4 parents of
1259
+ 500x2000.
1260
+ Enters a CAStack, a read walk, the stack axis outside the slab.
1261
+ Promises 2 (the parents' masks are copied), 6 (masks are read
1262
+ only when a parent has one), 8 (alias_ptr stays NULL,
1263
+ so no slab is lent as a window).
1264
+ Switch ca_fast_paths_enabled(): off, the transfer walk.
1265
+ Checked by the slab-axis, fiber and two-slab-axes rows of the
1266
+ stack parent along axes 1 and 2 in the delivery matrix. */
1267
+ if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1268
+ && !(flags & CA_KERNEL_WRITE)
1269
+ && naxes >= 1 && !in_slab[((CAStack *) src)->k_axis]
1270
+ && ca_fast_paths_enabled() ) {
1271
+ plan->route = KI_L2_STACK_OUTER_K;
1987
1272
  return CA_ITER_OK;
1988
1273
  }
1989
- if ( src_kind != CA_ITER_SRC_CASTRIDE ) {
1990
- /* Any other unclassified kind. */
1991
- return CA_ITER_ERR_POLICY;
1274
+ /* Shortcut PER_FIBER_FUSED (conversion views).
1275
+ Layer all three, per fiber: each fiber is fetched with the
1276
+ view's ca_xfer_stride into the fiber scratch, without
1277
+ gathering the whole view.
1278
+ Speeds up a read of a fiber of a fake or byte-swap view, or of a
1279
+ lazy monop that only casts, along its innermost axis.
1280
+ Why no difference measured on a sort along the last axis
1281
+ of a 2000x2000 fake view.
1282
+ Enters CA_KERNEL_FIBER_CONTIG, one slab axis, a read walk, and
1283
+ ca_iter_should_per_fiber_fused.
1284
+ Promises 1, 2 and 4, as the CAWindow / CAShift hook above. A
1285
+ write takes the transfer walk (promise 3).
1286
+ Switch ca_fast_paths_enabled(): off, the transfer walk.
1287
+ Checked by the fiber rows of the fake parent in the delivery
1288
+ matrix. */
1289
+ if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
1290
+ && ca_fast_paths_enabled()
1291
+ && ca_iter_should_per_fiber_fused(src, CA_ITER_SRC_ATTACH, axes[0],
1292
+ row_bytes[axes[0]], flags) ) {
1293
+ plan->route = KI_L2_FIBER_FUSED;
1294
+ return CA_ITER_OK;
1992
1295
  }
1296
+ plan->route = KI_L2_TRANSFER;
1297
+ return CA_ITER_OK;
1993
1298
  }
1994
1299
 
1995
- /* F-2 minimal scope: SHIFT outer axes need OOB fill-slab support which
1996
- is not yet implemented; downgrade to materialise (= existing
1997
- SRC_DESCRIPTOR path) when present. Future work: allocate a 1-slab
1998
- fill scratch and yield it on OOB iterations (see rev6 §3.5 deferred). */
1999
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE
2000
- && ca_axis_dispatch_outer_has_shift(raw_descs, raw_ndim) ) {
2001
- src_kind = CA_ITER_SRC_DESCRIPTOR;
1300
+ if ( plan->src_kind != CA_ITER_SRC_CASTRIDE ) return CA_ITER_ERR_POLICY;
1301
+
1302
+ if ( ca_iter_can_alias(src, 1) ) {
1303
+ plan->route = KI_L2_CONTIG;
1304
+ return CA_ITER_OK;
2002
1305
  }
1306
+ /* A non-contiguous CAStride: compose the strides up to the root. A
1307
+ root that holds nothing to read through would cost one whole-root
1308
+ materialise no matter how few cells the view touches -- the cliff
1309
+ the xfer path avoids by asking for a region -- so that view is
1310
+ gathered by its own region protocol instead: ca_copy_data walks the
1311
+ leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
1312
+ CAObject asks for exactly that block. */
1313
+ ca_stride_compose_to_root((CAStride *) src, &plan->root, plan->composed,
1314
+ &plan->base);
1315
+ plan->route = ca_root_lends_no_memory(plan->root) ? KI_L2_VIEW_SCRATCH
1316
+ : KI_L2_ROOT_STRIDED;
1317
+ return CA_ITER_OK;
1318
+ }
2003
1319
 
2004
- /* Step 9 + 2026-05-31 refactor: L2 dispatch over SRC_ATTACH sources
2005
- (CAFake / CAByteSwap / CABitfield / CABitarray / CAReduce / CAObject
2006
- / CATile / CARoll). Same shape as L1 SRC_ATTACH: iterator-owns
2007
- scratch + xfer_all GET/PUT. Yield as one 1-D strided slab. */
2008
- if ( src_kind == CA_ITER_SRC_ATTACH ) {
2009
- st->src = src;
2010
- st->src_kind = CA_ITER_SRC_ATTACH;
2011
- st->level = 2;
2012
- st->policy = policy;
2013
- st->ndim = 1; /* logical 1-D L2 layout */
2014
- st->flags = flags;
2015
- st->bytes = src->bytes;
2016
- st->naxes = naxes;
1320
+ /* Builds the state for the chosen route. */
1321
+ static void
1322
+ ki_open_l2 (ca_iter_state *st, CArray *src, ca_slab_policy_t policy,
1323
+ int8_t *axes, int8_t naxes, uint32_t flags, ki_l2_plan *plan)
1324
+ {
1325
+ ki_axis_layout lay;
1326
+ int8_t k;
2017
1327
 
2018
- ki_gather_view(st, src, flags, 0);
2019
- st->alias_mode = CA_ITER_ALIAS_NONE;
2020
- st->alias_ptr = st->scratch_ptr;
2021
- st->composed_strides[0] = src->bytes;
2022
- st->composed_base = 0;
2023
- st->slab_n = src->elements;
2024
- st->total_slabs = 1;
2025
- st->slabs_emitted = 0;
2026
- st->chunk_size = st->slab_n;
2027
- st->outer_idx = NULL;
1328
+ st->src = src;
1329
+ st->src_kind = plan->src_kind;
1330
+ st->level = 2;
1331
+ st->policy = policy;
1332
+ st->ndim = src->ndim;
1333
+ st->flags = flags;
1334
+ st->bytes = src->bytes;
1335
+ st->naxes = naxes;
2028
1336
 
2029
- if ( ca_has_mask(src) ) {
2030
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
2031
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
2032
- if ( src->elements > 0 ) {
2033
- ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
2034
- }
2035
- st->alias_mask = st->scratch_mask;
2036
- }
2037
- return CA_ITER_OK;
2038
- }
1337
+ /* The default layout: the view's own row-major strides, which is how
1338
+ a scratch the walk gathered is laid out and how the mask copy is
1339
+ laid out on every route that takes one. */
1340
+ for ( k = 0; k < src->ndim; k++ ) lay.dims[k] = src->dim[k];
1341
+ ki_view_row_major(src, lay.slab_strides, lay.mask_strides);
1342
+ memcpy(lay.outer_strides, lay.slab_strides, src->ndim * sizeof(ca_size_t));
1343
+
1344
+ switch ( plan->route ) {
1345
+
1346
+ case KI_L2_FIBER_FUSED:
1347
+ st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
1348
+ st->alias_ptr = NULL; /* no whole-view buffer */
1349
+ st->fiber_axis = axes[0];
1350
+ memcpy(st->fiber_native_strides, lay.slab_strides,
1351
+ src->ndim * sizeof(ca_size_t));
1352
+ break;
2039
1353
 
2040
- /* F-2 (rev6, PROPOSAL_F2_KERNEL_ITERATOR_ALIAS): descriptor L2 alias
2041
- path. Eligibility (route_source verdict): innermost descriptor axis
2042
- is STRIDE kind, no outer SHIFT axis (downgraded above when present).
2043
- Setup: ca_attach(parent), alias_ptr = parent->ptr, no scratch alloc.
2044
- Each outer-prefix iteration yields a strided slab
2045
- slab_ptr = parent->ptr + inner_byte_start + outer_prefix_offset
2046
- slab_n = descs[ndim-1].count
2047
- slab_stride = descs[ndim-1].step * pstrides[ndim-1]
2048
- where outer_prefix_offset is computed via ca_axis_dispatch_prefix_offset
2049
- on the pre-classified prefix[]. next_slab_strided branches on
2050
- src_kind to dispatch to this offset formula. */
2051
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
2052
- int8_t nd = raw_ndim;
2053
- int8_t k;
2054
-
2055
- st->src = src;
2056
- st->src_kind = CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE;
2057
- st->level = 2;
2058
- st->policy = policy;
2059
- st->ndim = nd;
2060
- st->flags = flags;
2061
- st->bytes = src->bytes;
2062
- st->naxes = naxes;
2063
-
2064
- /* Persist descs / parent_axis_dims in state; build parent row-major
2065
- pstrides so prefix_offset / inner offset share the same byte space. */
2066
- memcpy(st->descs, raw_descs, nd * sizeof(ca_axis_desc_t));
2067
- memcpy(st->parent_axis_dims, raw_pdims, nd * sizeof(ca_size_t));
2068
- st->desc_ndim = nd;
1354
+ case KI_L2_PER_SLAB: {
1355
+ /* One slab is gathered into a slab-sized scratch per outer index, so
1356
+ a slab axis steps through that scratch (packed row-major over the
1357
+ slab axes) and an outer axis moves nothing in it. The descriptors
1358
+ are kept for next_slab_axes to build each slab's subset. */
1359
+ CArray *parent = CAVIEW(src)->parent;
1360
+ ca_size_t b = src->bytes;
1361
+ ca_attach(parent); /* window: finish detaches st->root */
1362
+ st->root = parent;
1363
+ memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
1364
+ memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
2069
1365
  {
2070
1366
  ca_size_t s = src->bytes;
2071
- for ( k = nd - 1; k >= 0; k-- ) {
1367
+ for ( k = src->ndim - 1; k >= 0; k-- ) {
2072
1368
  st->pstrides[k] = s;
2073
- s *= raw_pdims[k];
1369
+ s *= plan->pdims[k];
2074
1370
  }
2075
1371
  }
2076
-
2077
- /* Inner slab parameters from innermost STRIDE descriptor axis. */
2078
- ca_size_t inner_start = raw_descs[nd - 1].start * st->pstrides[nd - 1];
2079
- ca_size_t inner_stride = raw_descs[nd - 1].step * st->pstrides[nd - 1];
2080
- ca_size_t inner_n = raw_descs[nd - 1].count;
2081
-
2082
- /* Outer prefix classify (skipped for 1-D source since no outer). */
2083
- if ( nd > 1 ) {
2084
- ca_axis_dispatch_classify_prefix(st->descs, st->pstrides, nd - 1, st->prefix);
2085
- }
2086
-
2087
- /* Total slabs = Π descs[0..nd-2].count (= src->elements / inner_n). */
2088
- ca_size_t outer_total = 1;
2089
- for ( k = 0; k < nd - 1; k++ ) outer_total *= raw_descs[k].count;
2090
- st->total_slabs = outer_total;
2091
- st->slab_n = inner_n;
2092
- st->slabs_emitted = 0;
2093
- st->chunk_size = inner_n;
2094
- st->total_elements = src->elements;
2095
-
2096
- /* composed_base holds inner_byte_start; composed_strides[nd-1] holds
2097
- inner stride so next_slab_strided's existing inner_st extraction
2098
- (= composed_strides[nd-1]) works without per-iter recomputation.
2099
- Outer slots of composed_strides are unused by the descriptor
2100
- branch in next_slab_strided (= prefix[] drives the offset). */
2101
- st->composed_base = inner_start;
2102
- st->composed_strides[nd - 1] = inner_stride;
2103
-
2104
- if ( nd > 1 ) {
2105
- CA_ASSUME(nd <= CA_RANK_MAX); /* with nd > 1: nd-1 in [1, CA_RANK_MAX-1] */
2106
- st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
2107
- for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
2108
- } else {
2109
- st->outer_idx = NULL;
2110
- }
2111
-
2112
- CArray *parent = CAVIEW(src)->parent;
2113
- ca_attach(parent);
2114
- st->root = parent;
2115
- st->alias_mode = CA_ITER_ALIAS_STRIDED;
2116
- st->alias_ptr = (char *) parent->ptr;
2117
-
2118
- /* Mask: gather into scratch_mask (same as SRC_DESCRIPTOR path).
2119
- Data alias + mask materialise is per rev6 §4.4 deferred to F-4.c. */
2120
- if ( ca_has_mask(src) ) {
2121
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
2122
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
2123
- if ( src->elements > 0 ) {
2124
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1372
+ st->desc_ndim = src->ndim;
1373
+ for ( k = src->ndim - 1; k >= 0; k-- ) {
1374
+ lay.dims[k] = plan->descs[k].count;
1375
+ lay.outer_strides[k] = 0;
1376
+ if ( plan->in_slab[k] ) {
1377
+ lay.slab_strides[k] = b;
1378
+ b *= plan->descs[k].count;
2125
1379
  }
2126
- st->alias_mask = st->scratch_mask;
2127
1380
  }
2128
- return CA_ITER_OK;
1381
+ break;
2129
1382
  }
2130
1383
 
2131
- /* Sub-step 5.3: L2 dispatch over descriptor sources. The view-family
2132
- surface prioritises delivering the cells over avoiding a copy: always
2133
- materialise into a
2134
- scratch buffer via ca_axis_dispatch_attach and yield a single
2135
- strided slab (stride = bytes). CASelect/CAMapping always reach
2136
- here, CSA/CAGrid/CAWindow/CAShift when INDEX/SHIFT axes are
2137
- present. F-2 (rev6) routes innermost-STRIDE descriptor cases to
2138
- the L2 alias path above; this block now handles remaining mixed
2139
- cases (innermost INDEX, outer SHIFT temporarily, etc.). */
2140
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
2141
- /* raw_descs / raw_pdims / raw_ndim were populated by route_source
2142
- above so we skip the local describe_axes call. Kept locals
2143
- named raw_* to match the original code. */
2144
- st->src = src;
2145
- st->src_kind = CA_ITER_SRC_DESCRIPTOR;
2146
- st->level = 2;
2147
- st->policy = policy;
2148
- st->ndim = src->ndim;
2149
- st->flags = flags;
2150
- st->bytes = src->bytes;
2151
- st->naxes = naxes;
2152
-
2153
- memcpy(st->parent_axis_dims, raw_pdims, raw_ndim * sizeof(ca_size_t));
2154
- ca_axis_dispatch_prepare(st->parent_axis_dims, raw_descs, raw_ndim,
2155
- st->bytes, st->descs, st->pstrides,
2156
- st->mdim, &st->desc_ndim);
2157
- ca_axis_dispatch_layout(st->descs, st->pstrides, st->mdim,
2158
- st->desc_ndim, st->bytes,
2159
- &st->slab_start, &st->slab_bytes_desc,
2160
- &st->slab_base);
2161
- if ( st->slab_start > 0 ) {
2162
- ca_axis_dispatch_classify_prefix(st->descs, st->pstrides,
2163
- st->slab_start, st->prefix);
2164
- }
2165
- st->total_elements = src->elements;
2166
-
2167
- /* Materialise via the engine into a scratch buffer (contig
2168
- layout = single strided run, stride = bytes). */
2169
- CArray *parent = CAVIEW(src)->parent;
2170
- ca_attach(parent);
2171
- st->root = parent;
2172
- st->scratch_cap = src->elements * src->bytes;
2173
- const void *bound_fill = NULL;
1384
+ case KI_L2_DESC_WHOLE: {
1385
+ /* The view gathered whole into a row-major scratch by the
1386
+ descriptor engine (which fills out-of-bounds cells), then walked
1387
+ with the default layout. */
1388
+ CArray *parent = CAVIEW(src)->parent;
1389
+ ca_axis_desc_t local_descs[CA_RANK_MAX];
1390
+ const void *bound_fill = NULL;
1391
+ ca_attach(parent); /* window: finish detaches st->root */
1392
+ st->root = parent;
1393
+ memcpy(local_descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
2174
1394
  if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
2175
1395
  bound_fill = ((CAWindow *) src)->fill;
2176
1396
  }
2177
- st->scratch_cap = st->total_elements * src->bytes;
1397
+ st->scratch_cap = src->elements * src->bytes;
2178
1398
  st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
2179
- ca_axis_dispatch_gather(parent, st->parent_axis_dims, raw_descs, raw_ndim,
2180
- src->bytes, st->total_elements, bound_fill,
1399
+ ca_axis_dispatch_gather(parent, plan->pdims, local_descs, src->ndim,
1400
+ src->bytes, src->elements, bound_fill,
2181
1401
  st->scratch_ptr);
1402
+ st->total_elements = src->elements;
1403
+ st->alias_mode = CA_ITER_ALIAS_NONE;
1404
+ st->alias_ptr = st->scratch_ptr;
1405
+ break;
1406
+ }
2182
1407
 
2183
- /* Lay out as a single 1-D L2 strided slab: ptr = scratch,
2184
- n = total_elements, stride = bytes. next_slab_strided reads
2185
- inner_stride from composed_strides[ndim - 1] and skips the
2186
- outer loop when outer_idx == NULL, so a 1-D logical layout
2187
- (st->ndim = 1, composed_strides[0] = bytes) yields exactly one
2188
- contig run. Note st->ndim diverges from src->ndim here — the
2189
- iterator's logical ndim is 1, but the view itself can be N-D
2190
- (the materialised buffer is flat). */
2191
- st->ndim = 1;
2192
- st->alias_mode = CA_ITER_ALIAS_NONE;
2193
- st->alias_ptr = st->scratch_ptr;
2194
- st->composed_strides[0] = src->bytes;
2195
- st->composed_base = 0;
2196
- st->slab_n = src->elements;
2197
- st->total_slabs = 1;
2198
- st->slabs_emitted = 0;
2199
- st->chunk_size = st->slab_n;
2200
- st->outer_idx = NULL;
2201
-
2202
- /* Step 6: mask gather, same as L1 descriptor branch. */
2203
- if ( ca_has_mask(src) ) {
2204
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
2205
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
2206
- if ( src->elements > 0 ) {
2207
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1408
+ case KI_L2_DESC_STRIDED: {
1409
+ /* The parent's memory is walked in place. Each slab axis is a
1410
+ STRIDE axis; an outer axis is STRIDE or INDEX (an outer SHIFT went
1411
+ to KI_L2_DESC_WHOLE). The start of every STRIDE axis is folded
1412
+ into composed_base, so a slab or STRIDE outer offset is step *
1413
+ index; an INDEX outer axis is looked up in its indices[] by
1414
+ next_slab_axes, which needs the descriptors kept. */
1415
+ CArray *parent = CAVIEW(src)->parent;
1416
+ ca_size_t base = 0;
1417
+ {
1418
+ ca_size_t s = src->bytes;
1419
+ for ( k = src->ndim - 1; k >= 0; k-- ) {
1420
+ st->pstrides[k] = s;
1421
+ s *= plan->pdims[k];
2208
1422
  }
2209
- st->alias_mask = st->scratch_mask;
2210
1423
  }
2211
- return CA_ITER_OK;
2212
- }
2213
-
2214
- /* === CAStride / entity L2 path === */
2215
- st->src = src;
2216
- st->src_kind = CA_ITER_SRC_CASTRIDE;
2217
- st->level = 2;
2218
- st->policy = policy;
2219
- st->ndim = src->ndim;
2220
- st->flags = flags;
2221
- st->bytes = src->bytes;
2222
- st->naxes = naxes;
2223
-
2224
- /* Compute composed strides + base. For entity / CAStride contig we
2225
- synthesise row-major byte strides so next_slab_strided's offset
2226
- math is uniform across alias modes; for CAStride non-contig we
2227
- compose leaf strides up to the root entity via the substrate. */
2228
- int8_t nd = src->ndim;
2229
- int use_strided = 0;
2230
- int use_view_scratch = 0;
2231
- CArray *root = NULL;
2232
-
2233
- if ( ca_iter_can_alias(src, 1) ) {
2234
- /* Entity or CAStride contig: stride = row-major bytes, base = 0. */
2235
- ca_iter_build_rowmajor_strides(st->composed_strides,
2236
- src->dim, nd, src->bytes);
2237
- st->composed_base = 0;
2238
- } else {
2239
- /* CAStride non-contig: leaf->root compose. */
2240
- ca_size_t cs[CA_RANK_MAX];
2241
- ca_size_t base;
2242
- ca_stride_compose_to_root((CAStride *) src, &root, cs, &base);
2243
- if ( !ca_root_lends_no_memory(root) ) {
2244
- memcpy(st->composed_strides, cs, nd * sizeof(ca_size_t));
2245
- st->composed_base = base;
2246
- use_strided = 1;
2247
- } else {
2248
- /* Root holds nothing to read through: reaching root->ptr costs one
2249
- whole-root materialise no matter how few cells the view touches —
2250
- the same cliff the xfer path avoids by asking for a region. Take
2251
- the view's own region protocol instead: ca_copy_data walks the
2252
- leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
2253
- CAObject asks for exactly that block. The gathered buffer is view
2254
- row-major, so the slab arithmetic below is the contig-alias case
2255
- unchanged. */
2256
- root = NULL;
2257
- ca_iter_build_rowmajor_strides(st->composed_strides,
2258
- src->dim, nd, src->bytes);
2259
- st->composed_base = 0;
2260
- use_view_scratch = 1;
1424
+ for ( k = 0; k < src->ndim; k++ ) {
1425
+ const ca_axis_desc_t *d = &plan->descs[k];
1426
+ lay.dims[k] = d->count;
1427
+ if ( plan->in_slab[k] ) {
1428
+ base += d->start * st->pstrides[k];
1429
+ lay.slab_strides[k] = d->step * st->pstrides[k];
1430
+ }
1431
+ else if ( d->kind == CA_AXIS_KIND_STRIDE ) {
1432
+ base += d->start * st->pstrides[k];
1433
+ lay.outer_strides[k] = d->step * st->pstrides[k];
1434
+ }
1435
+ else {
1436
+ lay.outer_strides[k] = st->pstrides[k];
1437
+ }
2261
1438
  }
1439
+ st->composed_base = base;
1440
+ memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
1441
+ memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
1442
+ st->desc_ndim = src->ndim;
1443
+ ca_attach(parent); /* window: finish detaches st->root */
1444
+ st->root = parent;
1445
+ st->alias_mode = CA_ITER_ALIAS_STRIDED;
1446
+ st->alias_ptr = (char *) parent->ptr;
1447
+ break;
2262
1448
  }
2263
1449
 
2264
- /* Phase A: CA_SLAB_AXES branch. Partition axes into slab vs outer,
2265
- populate per-axis dims / strides (both data byte strides from
2266
- composed_strides[] and mask element strides from view row-major
2267
- dim products), allocate outer_idx if outer_ndim > 0, attach data
2268
- buffer (alias or strided), gather mask if present.
2269
-
2270
- Layout invariant: slab_axes_buf and outer_axes are both stored
2271
- sort-ascending (in source-axis order), so multi-axis CA_SLAB_AXES
2272
- is canonical regardless of the user's input order. */
2273
- if ( policy == CA_SLAB_AXES ) {
2274
- /* Validate axes input. */
2275
- if ( axes == NULL || naxes <= 0 || naxes > nd ) {
2276
- return CA_ITER_ERR_POLICY;
2277
- }
2278
- int8_t in_slab[CA_RANK_MAX];
2279
- int8_t k;
2280
- for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
2281
- for ( k = 0; k < naxes; k++ ) {
2282
- int8_t ax = axes[k];
2283
- if ( ax < 0 || ax >= nd ) return CA_ITER_ERR_POLICY;
2284
- if ( in_slab[ax] ) return CA_ITER_ERR_POLICY; /* duplicate */
2285
- in_slab[ax] = 1;
2286
- }
2287
-
2288
- /* View row-major element strides (= mask scratch layout strides). */
2289
- ca_size_t row_elem_strides[CA_RANK_MAX];
1450
+ case KI_L2_STACK: {
1451
+ /* The K parents are attached and read directly; the slab (the K cells
1452
+ along the stack axis) is cut into a slab-sized scratch, refilled
1453
+ from a tile cache of TILE fibers that the parents fill in one pass.
1454
+ CAStack guarantees one shape across its parents, so parents[0]
1455
+ gives the native strides of all of them. */
1456
+ CAStack *stack = (CAStack *) src;
1457
+ int8_t parent_ndim = src->ndim - 1;
1458
+ ca_size_t K = (ca_size_t) stack->n_parents;
1459
+ ca_size_t bytes = (ca_size_t) src->bytes;
1460
+ ca_size_t tile;
1461
+ lay.slab_strides[axes[0]] = src->bytes;
1462
+ lay.mask_strides[axes[0]] = 1;
1463
+ ki_attach_stack_parents(st, stack);
2290
1464
  {
2291
- ca_size_t s = 1;
2292
- for ( k = nd - 1; k >= 0; k-- ) {
2293
- row_elem_strides[k] = s;
2294
- s *= src->dim[k];
1465
+ ca_size_t s = src->bytes;
1466
+ for ( k = parent_ndim - 1; k >= 0; k-- ) {
1467
+ st->stack_parent_strides[k] = s;
1468
+ s *= stack->parents[0]->dim[k];
2295
1469
  }
2296
1470
  }
1471
+ st->scratch_cap = (ca_size_t) src->dim[axes[0]] * src->bytes;
1472
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1473
+ /* TILE budget = ~32 KB (half of M2 L1d), clamped to [8, 64] to keep
1474
+ the refill loop tight. */
1475
+ tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
1476
+ if ( tile < 8 ) tile = 8;
1477
+ if ( tile > 64 ) tile = 64;
1478
+ st->stack_tile_cap = tile;
1479
+ st->stack_tile_pos = 0;
1480
+ st->stack_tile_have = 0; /* force refill on first call */
1481
+ st->stack_tile_cache = (char *) ki_data_alloc(src->data_type, K * tile * bytes);
1482
+ st->alias_mode = CA_ITER_ALIAS_STACK;
1483
+ st->alias_ptr = NULL; /* no whole-view buffer */
1484
+ break;
1485
+ }
2297
1486
 
2298
- /* Partition axes (ascending order). */
2299
- int8_t sp = 0, op = 0;
2300
- st->slab_elements = 1;
2301
- for ( k = 0; k < nd; k++ ) {
2302
- if ( in_slab[k] ) {
2303
- st->slab_axes_buf[sp] = k;
2304
- st->slab_dims[sp] = src->dim[k];
2305
- st->slab_strides[sp] = st->composed_strides[k];
2306
- st->slab_mask_strides[sp] = row_elem_strides[k];
2307
- st->slab_elements *= src->dim[k];
2308
- sp++;
2309
- } else {
2310
- st->outer_axes[op] = k;
2311
- st->outer_dims[op] = src->dim[k];
2312
- st->outer_strides[op] = st->composed_strides[k];
2313
- st->outer_mask_strides[op] = row_elem_strides[k];
2314
- op++;
1487
+ case KI_L2_STACK_OUTER_K: {
1488
+ /* Each slab is a region inside ONE parent, picked by
1489
+ outer_idx[stack_k_outer_pos]; next_slab_axes points into
1490
+ parents[k]->ptr + parent_off. A slab axis s of the stack is parent
1491
+ axis s, or s - 1 past the stack axis. */
1492
+ CAStack *stack = (CAStack *) src;
1493
+ int8_t k_axis = stack->k_axis;
1494
+ int8_t parent_ndim = src->ndim - 1;
1495
+ {
1496
+ ca_size_t sb = src->bytes, se = 1;
1497
+ for ( k = parent_ndim - 1; k >= 0; k-- ) {
1498
+ st->stack_parent_strides[k] = sb;
1499
+ st->stack_parent_mask_strides[k] = se;
1500
+ sb *= stack->parents[0]->dim[k];
1501
+ se *= stack->parents[0]->dim[k];
2315
1502
  }
2316
1503
  }
2317
- st->slab_ndim = sp;
2318
- st->outer_ndim = op;
2319
-
2320
- /* total_slabs = Π outer_dims. Empty product (all-axes case) = 1
2321
- → single slab = whole array (D1.4 WHOLE-equivalent). */
2322
- ca_size_t total = 1;
2323
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
2324
- st->total_slabs = total;
2325
- st->slab_n = st->slab_elements; /* mirror to legacy field */
2326
- st->slabs_emitted = 0;
2327
- st->chunk_size = st->slab_n;
2328
-
2329
- if ( st->outer_ndim > 0 ) {
2330
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
2331
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
2332
- } else {
2333
- st->outer_idx = NULL;
2334
- }
2335
-
2336
- if ( use_strided ) {
2337
- ca_attach(root); /* window: finish detaches st->root */
2338
- st->root = root; /* recorded once attached */
2339
- st->alias_mode = CA_ITER_ALIAS_STRIDED;
2340
- st->alias_ptr = (char *) root->ptr;
2341
- } else if ( use_view_scratch ) {
2342
- st->root = NULL;
2343
- ki_gather_view(st, src, flags, 1);
2344
- st->alias_mode = CA_ITER_ALIAS_NONE;
2345
- st->alias_ptr = st->scratch_ptr;
2346
- } else {
2347
- st->root = NULL;
2348
- ca_attach(src);
2349
- st->alias_mode = CA_ITER_ALIAS_CONTIG;
2350
- st->alias_ptr = (char *) src->ptr;
1504
+ st->stack_k_outer_pos = 0;
1505
+ for ( k = 0; k < src->ndim; k++ ) {
1506
+ if ( plan->in_slab[k] ) {
1507
+ int8_t parent_ax = (k < k_axis) ? k : (k - 1);
1508
+ lay.slab_strides[k] = st->stack_parent_strides[parent_ax];
1509
+ lay.mask_strides[k] = st->stack_parent_mask_strides[parent_ax];
1510
+ }
1511
+ else if ( k < k_axis ) {
1512
+ st->stack_k_outer_pos++;
1513
+ }
2351
1514
  }
2352
-
2353
- /* Mask: gather to scratch_mask in view row-major order (= same as
2354
- L1 path). The mask layout matches view->dim row-major, which is
2355
- what slab_mask_strides / outer_mask_strides walk. */
1515
+ ki_attach_stack_parents(st, stack);
1516
+ /* A masked stack: copy the K parent masks as they are at the start
1517
+ of the walk, and next_slab_axes points into the copy of parent k
1518
+ (promise 2). The stack's mask may still read stand-ins for
1519
+ parents that have gained a mask since; read through to those. */
2356
1520
  if ( ca_has_mask(src) ) {
2357
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
2358
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
2359
- if ( src->elements > 0 ) {
2360
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1521
+ ca_multi_parent_mask_refresh(src->mask, stack->parents,
1522
+ stack->mask_standins,
1523
+ stack->n_parents);
1524
+ /* Zero-filled: a slot is set once its copy is made, and finish
1525
+ frees the copies whose slot is set. */
1526
+ st->stack_parent_mask_ptrs =
1527
+ (boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
1528
+ memset(st->stack_parent_mask_ptrs, 0,
1529
+ stack->n_parents * sizeof(boolean8_t *));
1530
+ for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1531
+ CArray *pm = ((CAStack *) src->mask)->parents[kk];
1532
+ ca_size_t n = pm->elements;
1533
+ boolean8_t *copy = (boolean8_t *) xmalloc(n > 0 ? n : 1);
1534
+ st->stack_parent_mask_ptrs[kk] = copy;
1535
+ ca_attach(pm); /* window: only a memcpy before the detach */
1536
+ memcpy(copy, pm->ptr, n);
1537
+ ca_detach(pm);
2361
1538
  }
2362
- st->alias_mask = st->scratch_mask;
2363
1539
  }
2364
- return CA_ITER_OK;
1540
+ st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
1541
+ st->alias_ptr = NULL;
1542
+ break;
2365
1543
  }
2366
1544
 
2367
- /* Outer prefix axes [0..ndim-2] drive total_slabs; innermost axis is
2368
- the slab. 0-d / 1-d sources collapse to a single yield. */
2369
- if ( nd <= 1 ) {
2370
- st->total_slabs = 1;
2371
- st->slab_n = src->elements;
2372
- st->outer_idx = NULL;
2373
- } else {
2374
- ca_size_t total = 1;
2375
- int8_t k;
2376
- for ( k = 0; k < nd - 1; k++ ) total *= src->dim[k];
2377
- st->total_slabs = total;
2378
- st->slab_n = src->dim[nd - 1];
2379
- st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
2380
- for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
2381
- }
2382
- st->slabs_emitted = 0;
2383
- st->chunk_size = st->slab_n;
1545
+ case KI_L2_TRANSFER:
1546
+ ki_gather_view(st, src, flags, 0);
1547
+ st->alias_mode = CA_ITER_ALIAS_NONE;
1548
+ st->alias_ptr = st->scratch_ptr;
1549
+ break;
2384
1550
 
2385
- if ( use_strided ) {
2386
- ca_attach(root); /* window: finish detaches st->root */
2387
- st->root = root; /* recorded once attached */
1551
+ case KI_L2_ROOT_STRIDED:
1552
+ memcpy(st->composed_strides, plan->composed, src->ndim * sizeof(ca_size_t));
1553
+ st->composed_base = plan->base;
1554
+ memcpy(lay.slab_strides, plan->composed, src->ndim * sizeof(ca_size_t));
1555
+ memcpy(lay.outer_strides, plan->composed, src->ndim * sizeof(ca_size_t));
1556
+ ca_attach(plan->root); /* window: finish detaches st->root */
1557
+ st->root = plan->root; /* recorded once attached */
2388
1558
  st->alias_mode = CA_ITER_ALIAS_STRIDED;
2389
- st->alias_ptr = (char *) root->ptr;
2390
- } else if ( use_view_scratch ) {
2391
- st->root = NULL;
1559
+ st->alias_ptr = (char *) plan->root->ptr;
1560
+ break;
1561
+
1562
+ case KI_L2_VIEW_SCRATCH:
1563
+ ca_iter_build_rowmajor_strides(st->composed_strides,
1564
+ src->dim, src->ndim, src->bytes);
2392
1565
  ki_gather_view(st, src, flags, 1);
2393
1566
  st->alias_mode = CA_ITER_ALIAS_NONE;
2394
1567
  st->alias_ptr = st->scratch_ptr;
2395
- } else {
2396
- st->root = NULL;
1568
+ break;
1569
+
1570
+ case KI_L2_CONTIG:
1571
+ ca_iter_build_rowmajor_strides(st->composed_strides,
1572
+ src->dim, src->ndim, src->bytes);
2397
1573
  ca_attach(src);
2398
1574
  st->alias_mode = CA_ITER_ALIAS_CONTIG;
2399
1575
  st->alias_ptr = (char *) src->ptr;
1576
+ break;
2400
1577
  }
2401
1578
 
2402
- /* Mask: gathered in view row-major order, as on the AXES path above;
2403
- next_slab_strided hands out alias_mask + slab * slab_n. */
2404
- if ( ca_has_mask(src) ) {
2405
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
2406
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
2407
- if ( src->elements > 0 ) {
2408
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1579
+ ki_lay_out_slabs(st, src, plan->in_slab, &lay);
1580
+
1581
+ if ( plan->route == KI_L2_PER_SLAB ) {
1582
+ /* One max-slab buffer refilled per outer index, sized now that the
1583
+ slab is known.
1584
+
1585
+ Shortcut PER_SLAB_HOIST.
1586
+ Layer data; the fiber layer sits on top.
1587
+ Speeds up per-slab gathers whose innermost slab axis is STRIDE
1588
+ and with no SHIFT axis in the view: the outer and
1589
+ non-innermost slab offsets are hoisted and the inner
1590
+ run is copied with memcpy.
1591
+ Why no difference measured on a row-selected 2-D read.
1592
+ Enters at least one slab axis, the innermost slab axis
1593
+ STRIDE, no SHIFT axis.
1594
+ Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
1595
+ only when the view has one).
1596
+ Switch ca_fast_paths_enabled(): off, the PER_SLAB gather.
1597
+ Checked by the two-slab-axes rows of the grid parent in the
1598
+ delivery matrix. */
1599
+ ca_size_t slab_bytes = st->slab_elements * src->bytes;
1600
+ int hoist = (st->slab_ndim > 0) && ca_fast_paths_enabled()
1601
+ && plan->descs[st->slab_axes_buf[st->slab_ndim - 1]].kind
1602
+ == CA_AXIS_KIND_STRIDE;
1603
+ for ( k = 0; hoist && k < src->ndim; k++ ) {
1604
+ if ( plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) hoist = 0;
2409
1605
  }
2410
- st->alias_mask = st->scratch_mask;
1606
+ st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
1607
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1608
+ st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
1609
+ st->alias_mode = hoist ? CA_ITER_ALIAS_PER_SLAB_HOIST
1610
+ : CA_ITER_ALIAS_PER_SLAB;
2411
1611
  }
1612
+
1613
+ /* The mask cursor (promise 2): fiber by fiber through fiber_mask_scratch
1614
+ on PER_FIBER_FUSED, the parents' copies on STACK_OUTER_K, none on
1615
+ STACK (an unmasked stack), and the walk's copy of the whole view's
1616
+ mask everywhere else. */
1617
+ if ( plan->route == KI_L2_FIBER_FUSED || plan->route == KI_L2_STACK
1618
+ || plan->route == KI_L2_STACK_OUTER_K ) {
1619
+ st->alias_mask = NULL;
1620
+ }
1621
+ else {
1622
+ ki_take_mask_copy(st, src);
1623
+ }
1624
+ }
1625
+
1626
+ static int
1627
+ ca_iter_state_init_l2_unprotected (ca_iter_state *st,
1628
+ struct _CArray *src,
1629
+ ca_slab_policy_t policy,
1630
+ int8_t *axes,
1631
+ int8_t naxes,
1632
+ uint32_t flags)
1633
+ {
1634
+ ki_l2_plan plan;
1635
+ int rc;
1636
+
1637
+ /* Leave the state defined before anything can return. Every error
1638
+ return below hands control back to a caller that may not look at the
1639
+ code, so what it is left holding has to be a state that iterates zero
1640
+ times rather than whatever its stack frame happened to contain. */
1641
+ if ( st == NULL ) return CA_ITER_ERR_FLAGS;
1642
+ memset(st, 0, sizeof(*st));
1643
+ if ( src == NULL ) return CA_ITER_ERR_FLAGS;
1644
+
1645
+ /* A Face only layers a semantic identifier and a CALazyMarker only
1646
+ layers "leaf of a lazy chain"; storage is identical to the parent
1647
+ either way, so strip at the kernel-compute entry and descend.
1648
+ Routing and alias decisions belong to the parent. Strip before
1649
+ validate_inputs: left on, both would classify as SRC_ATTACH (they
1650
+ have attach slots) and be materialised whole (see
1651
+ ca_iter_strip_storage_wrapper).
1652
+
1653
+ The Face identifier is re-wrapped onto the result by the caller's lift
1654
+ hook (= primary operators / reductions / etc.). The marker is not:
1655
+ a reduction over a lazy marker yields a plain entity.
1656
+
1657
+ WRITE is rejected against the wrapper, before the strip — a marker is
1658
+ read-only while its parent is not. validate_inputs re-checks the
1659
+ stripped source, which is harmless. */
1660
+ if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
1661
+ return CA_ITER_ERR_READONLY;
1662
+ }
1663
+ if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
1664
+ src = ca_iter_strip_storage_wrapper(src);
1665
+ }
1666
+
1667
+ rc = ca_iter_validate_inputs(st, src, policy, flags);
1668
+ if ( rc != CA_ITER_OK ) return rc;
1669
+ /* L2 walks a slab of named axes. The whole-array walk is L1
1670
+ (init_l1 with CA_SLAB_WHOLE); one strided run per innermost row is
1671
+ CA_SLAB_AXES with the last axis. */
1672
+ if ( policy != CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
1673
+
1674
+ /* The slab axes: in range, no repeats. */
1675
+ if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
1676
+ return CA_ITER_ERR_POLICY;
1677
+ }
1678
+ memset(plan.in_slab, 0, sizeof(plan.in_slab));
1679
+ for ( int8_t k = 0; k < naxes; k++ ) {
1680
+ int8_t ax = axes[k];
1681
+ if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
1682
+ if ( plan.in_slab[ax] ) return CA_ITER_ERR_POLICY;
1683
+ plan.in_slab[ax] = 1;
1684
+ }
1685
+
1686
+ /* route_source returns the refined src_kind and, for descriptor
1687
+ sources, the descriptors, so the choice below does not describe the
1688
+ axes again. */
1689
+ plan.ndesc = 0;
1690
+ plan.root = NULL;
1691
+ plan.base = 0;
1692
+ plan.src_kind = ca_iter_route_source(src, plan.descs, plan.pdims,
1693
+ &plan.ndesc);
1694
+ rc = ki_choose_l2_route(src, axes, naxes, flags, &plan);
1695
+ if ( rc != CA_ITER_OK ) return rc;
1696
+
1697
+ ki_open_l2(st, src, policy, axes, naxes, flags, &plan);
2412
1698
  return CA_ITER_OK;
2413
1699
  }
2414
1700
 
@@ -2435,74 +1721,6 @@ ca_iter_state_next_slab_unprotected (ca_iter_state *st,
2435
1721
  return 1;
2436
1722
  }
2437
1723
 
2438
- static int
2439
- ca_iter_state_next_slab_strided_unprotected (ca_iter_state *st,
2440
- char **out_ptr,
2441
- boolean8_t **out_mask,
2442
- ca_size_t *out_n,
2443
- ca_size_t *out_stride_bytes)
2444
- {
2445
- if ( st == NULL || st->level != 2
2446
- || st->slabs_emitted >= st->total_slabs ) {
2447
- if ( out_ptr ) *out_ptr = NULL;
2448
- if ( out_mask ) *out_mask = NULL;
2449
- if ( out_n ) *out_n = 0;
2450
- if ( out_stride_bytes ) *out_stride_bytes = 0;
2451
- return 0;
2452
- }
2453
-
2454
- int8_t nd = st->ndim;
2455
- ca_size_t inner_st = (nd > 0)
2456
- ? st->composed_strides[nd - 1]
2457
- : st->bytes;
2458
- ca_size_t off = st->composed_base;
2459
- int8_t k;
2460
-
2461
- /* Sum outer prefix offset. For F-2 descriptor L2 alias the prefix can
2462
- contain INDEX axes (and, once OOB fill_slab lands, SHIFT axes), so
2463
- we delegate to the pre-classified prefix[] engine. Otherwise (=
2464
- CAStride / entity outer = STRIDE-only by construction) use the
2465
- direct sum that has been the L2 inner loop since step 3. For
2466
- 0-d / 1-d sources, the loop / call is a no-op and off stays at
2467
- composed_base. */
2468
- if ( st->outer_idx != NULL ) {
2469
- if ( st->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
2470
- int oob = 0;
2471
- off += ca_axis_dispatch_prefix_offset(st->prefix, st->outer_idx,
2472
- nd - 1, &oob);
2473
- /* oob unreachable: outer SHIFT was downgraded in init_l2. */
2474
- } else {
2475
- for ( k = 0; k < nd - 1; k++ ) {
2476
- off += st->outer_idx[k] * st->composed_strides[k];
2477
- }
2478
- }
2479
- }
2480
-
2481
- if ( out_ptr ) *out_ptr = st->alias_ptr + off;
2482
- /* mask layout mirrors value layout, but mask is contig boolean8_t
2483
- when alias_mask is a scratch buffer (step 6 baseline = always
2484
- gather mask into scratch_mask for uniformity). Per-slab mask
2485
- offset = i in the outer cursor (= slabs_emitted at this point). */
2486
- if ( out_mask ) {
2487
- *out_mask = st->alias_mask
2488
- ? st->alias_mask + st->slabs_emitted * st->slab_n
2489
- : NULL;
2490
- }
2491
- if ( out_n ) *out_n = st->slab_n;
2492
- if ( out_stride_bytes ) *out_stride_bytes = inner_st;
2493
-
2494
- st->slabs_emitted += 1;
2495
-
2496
- /* Advance outer_idx row-major (least-significant axis innermost,
2497
- so we tick outer_idx[ndim-2] first). */
2498
- if ( st->outer_idx != NULL ) {
2499
- for ( k = nd - 2; k >= 0; k-- ) {
2500
- if ( ++st->outer_idx[k] < st->src->dim[k] ) break;
2501
- st->outer_idx[k] = 0;
2502
- }
2503
- }
2504
- return 1;
2505
- }
2506
1724
 
2507
1725
  /* CA_KERNEL_FIBER_CONTIG: a fiber (a walk with one slab axis) reaches the
2508
1726
  kernel as a contiguous run of cells and of mask flags. Each path of
@@ -2565,12 +1783,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2565
1783
  return 0;
2566
1784
  }
2567
1785
 
2568
- /* Phase C T3 specialised path (B-1b, C.1b, 2026-05-27): innermost
2569
- slab axis is STRIDE, no SHIFT axes anywhere. Hoist outer + non-
2570
- innermost-slab axes (= per-cell switch evaluated once per slab row),
2571
- inner = pure STRIDE linear memcpy (= SIMD-friendly contig run, no
2572
- engine per-cell dispatch). Target: 1.5-1.8x win for INDEX slab
2573
- with innermost STRIDE (= grid / select sparse projection use cases). */
1786
+ /* PER_SLAB_HOIST (see its name tag in ki_open_l2): the innermost slab
1787
+ axis is STRIDE and no axis is SHIFT. The outer and non-innermost
1788
+ slab offsets are computed once per slab row, and the inner run is a
1789
+ plain memcpy instead of the descriptor engine's per-cell walk. */
2574
1790
  if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB_HOIST ) {
2575
1791
  int8_t inner_view_ax = st->slab_axes_buf[st->slab_ndim - 1];
2576
1792
  ca_size_t inner_count = st->descs[inner_view_ax].count;
@@ -2666,17 +1882,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2666
1882
  return 1;
2667
1883
  }
2668
1884
 
2669
- /* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18): CAStack
2670
- source + K-axis (k_axis) in outer iter. Each slab aliases a region
2671
- inside parents[k]->ptr where k = outer_idx[K_outer_pos]. Parent
2672
- inner byte offset = Σ outer_idx[m] * stack_parent_strides[parent_ax]
2673
- over all outer axes except the K-axis itself. Mask: parallel alias
2674
- into parents[k]->mask->ptr + mask_off. Zero copy / zero scratch /
2675
- parent entity bandwidth.
2676
-
2677
- K.3 (2026-06-20): parent_ax derivation generalised for arbitrary
2678
- k_axis -- stack axis s != k_axis maps to parent axis s if
2679
- s < k_axis else s - 1. */
1885
+ /* CAStack source with the stack axis k_axis in the outer iteration.
1886
+ Each slab is a region inside parents[k]->ptr where k =
1887
+ outer_idx[K_outer_pos]. Parent inner byte offset = Σ outer_idx[m] *
1888
+ stack_parent_strides[parent_ax] over all outer axes except the K-axis
1889
+ itself. Mask: a pointer into the copy of parent k's mask that init
1890
+ took (stack_parent_mask_ptrs[k] + mask_off); a write through it
1891
+ reaches nothing. The data is yielded in place unless FIBER_CONTIG
1892
+ needs a strided fiber gathered (ki_fiber_contig_data). A stack axis
1893
+ s other than k_axis is parent axis s, or s - 1 past k_axis. */
2680
1894
  if ( st->alias_mode == CA_ITER_ALIAS_STACK_OUTER_K ) {
2681
1895
  int8_t kpos = st->stack_k_outer_pos;
2682
1896
  int32_t k = (int32_t) st->outer_idx[kpos];
@@ -2711,20 +1925,19 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2711
1925
  return 1;
2712
1926
  }
2713
1927
 
2714
- /* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2: CAStack direct
2715
- per-parent ptr access path. For each outer iter, compute the
1928
+ /* CAStack read along its stack axis, from the parents' memory. For
1929
+ each outer index, compute the
2716
1930
  parent inner byte offset (= same across all K parents, uniform
2717
- shape), then K-fold direct memcpy gather from cached parent ptrs
2718
- into scratch. No ca_xfer_stride dispatch; per-cell cost = pointer
2719
- arith + memcpy(bytes). For f64 the memcpy(8) compiles to a single
2720
- mov, so the inner loop is tight.
2721
-
2722
- Note: scope-narrowed to slab_axes == [0] at init, so outer_axes are
2723
- stack axes 1..N-1 = parent axes 0..parent_ndim-1. outer_idx[m]
2724
- maps directly to parent axis (m) (since the m-th outer axis is
2725
- stack axis m+1 = parent axis m). */
1931
+ shape), and yield the K cells of that fiber out of stack_tile_cache,
1932
+ which is refilled TILE fibers at a time from the cached parent ptrs
1933
+ (below). No ca_xfer_stride dispatch. The scratch_ptr init
1934
+ allocates is not used by this path.
1935
+
1936
+ init engages this path only for slab_axes == [k_axis], so the outer
1937
+ axes are the stack axes other than k_axis, which are the parent axes
1938
+ in order: outer_idx[m] maps to parent axis m. */
2726
1939
  if ( st->alias_mode == CA_ITER_ALIAS_STACK ) {
2727
- /* pilot/castack-axis0-loop-interchange: tile cache. Outer iter
1940
+ /* Tile cache. Outer iter
2728
1941
  walks parent storage in row-major order so parent_off increments
2729
1942
  by `bytes` per call monotonically. Refill TILE fibers at once
2730
1943
  via K contig parent streams (= TILE consecutive cells from each
@@ -2794,11 +2007,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2794
2007
  return 1;
2795
2008
  }
2796
2009
 
2797
- /* Phase C T3 fallback (C.1, 2026-05-27): per-slab materialise via
2798
- ca_axis_dispatch_gather with a subset descriptor built by pinning
2799
- outer axes at the current outer_idx position (D1.2 (A): caller-side
2800
- subset construction, engine API unchanged). scratch_ptr is sized
2801
- for one max slab and reused across iters (D1.1 (B)). */
2010
+ /* PER_SLAB: one slab gathered by ca_axis_dispatch_gather, from a
2011
+ subset of the descriptors with the outer axes pinned at the current
2012
+ outer_idx position. scratch_ptr is sized for one slab and reused for
2013
+ every slab. */
2802
2014
  if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB ) {
2803
2015
  ca_axis_desc_t subset_descs[CA_RANK_MAX];
2804
2016
  memcpy(subset_descs, st->descs, st->ndim * sizeof(ca_axis_desc_t));
@@ -2833,7 +2045,7 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2833
2045
  }
2834
2046
  }
2835
2047
 
2836
- /* CAWindow fill value capture (= same lookup as B.1.5 init). */
2048
+ /* CAWindow fill value (the same lookup as in ki_open_l2). */
2837
2049
  const void *bound_fill = NULL;
2838
2050
  if ( ca_func[st->src->obj_type].attach == ca_window_func.attach ) {
2839
2051
  bound_fill = ((CAWindow *) st->src)->fill;
@@ -2843,8 +2055,8 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2843
2055
  st->ndim, st->bytes, st->slab_elements,
2844
2056
  bound_fill, st->scratch_ptr);
2845
2057
 
2846
- /* Mask offset into whole-view scratch_mask (T3 path keeps mask
2847
- layout view-row-major to match slab_mask_strides). */
2058
+ /* Mask offset into the whole-view mask copy, which is view
2059
+ row-major to match slab_mask_strides. */
2848
2060
  ca_size_t mask_off = 0;
2849
2061
  for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
2850
2062
  mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
@@ -2871,10 +2083,9 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2871
2083
  (element units, view row-major). Both are zero when outer_ndim
2872
2084
  == 0 (= all-axes WHOLE-equivalent case).
2873
2085
 
2874
- STRIDE outer axes (Phase A SRC_CASTRIDE, and Phase B
2875
- SRC_DESCRIPTOR STRIDE-kind axes) use outer_idx[m] directly as the
2876
- multiplier. INDEX outer axes (Phase B SRC_DESCRIPTOR INDEX-kind)
2877
- need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
2086
+ STRIDE outer axes (SRC_CASTRIDE, and the STRIDE axes of a descriptor
2087
+ view) use outer_idx[m] directly as the multiplier. INDEX outer axes
2088
+ of a descriptor view need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
2878
2089
  Mask offset uses outer_idx[m] directly in both cases (= mask
2879
2090
  scratch is in view row-major order, outer_idx walks view-axis
2880
2091
  positions). */
@@ -2894,13 +2105,12 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2894
2105
  mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
2895
2106
  }
2896
2107
 
2897
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1: per-fiber fused dispatch.
2898
- When init_l2 selected CA_ITER_ALIAS_PER_FIBER_FUSED, there is no
2899
- whole-view buffer to alias from. Build the fiber region from
2900
- outer_idx + fiber_axis and call ca_xfer_stride(src, ..., GET)
2901
- directly into fiber_data_scratch. For fused-aware views (X.1
2902
- OOB-fused / X.4 transform-fused) this routes to a 1-pass per-
2903
- region path.
2108
+ /* PER_FIBER_FUSED: there is no whole-view buffer to alias from. Build
2109
+ the fiber region from outer_idx + fiber_axis and call
2110
+ ca_xfer_stride(src, ..., GET) directly into fiber_data_scratch; the
2111
+ view's region transfer (out-of-bounds filling for CAWindow / CAShift,
2112
+ the conversion for a fake or byte-swap view) delivers it in one
2113
+ pass.
2904
2114
 
2905
2115
  last_data_off carries the fiber region's outer footprint so
2906
2116
  sync_slab can reconstruct the same region for WRITE PUT. Encode
@@ -2924,8 +2134,8 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2924
2134
  /* Build fiber region: fiber_axis spans the full fiber, all other
2925
2135
  axes pinned to outer_idx position (count=1). Cache starts[] in
2926
2136
  state so sync_slab can rebuild the same region for WRITE PUT
2927
- (= captured BEFORE outer_idx advance below, same hazard pattern
2928
- as F.1a last_data_off). */
2137
+ (= captured BEFORE the outer_idx advance below, the same hazard as
2138
+ last_data_off; see sync_slab). */
2929
2139
  {
2930
2140
  int8_t op = 0;
2931
2141
  for ( int8_t k = 0; k < nd; k++ ) {
@@ -2985,16 +2195,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2985
2195
  return 1;
2986
2196
  }
2987
2197
 
2988
- /* PROPOSAL_FIBER_DELIVERY F.1a: per-axis fiber contig delivery.
2989
- For naxes==1 (= single slab axis = fiber) the catalog contract
2198
+ /* Contiguous fiber delivery (promise 1). For naxes==1 (= single slab axis = fiber) the catalog contract
2990
2199
  CA_FOR_EACH_FIBER promises contig data delivery. When the fiber
2991
2200
  is not innermost-contig (= slab_strides[0] != bytes), the engine
2992
2201
  gathers the fiber into fiber_data_scratch before yielding so the
2993
2202
  author can write p[i] without stride math.
2994
2203
 
2995
2204
  Capture last_data_off BEFORE the outer_idx advance below; sync_slab
2996
- consumes it for WRITE scatter. See header field doc + PROPOSAL
2997
- §4.3.2 hazard comment in sync_slab. */
2205
+ consumes it for WRITE scatter. See the header field and the hazard
2206
+ comment in sync_slab. */
2998
2207
  /* sync_slab finds the slab here: the fiber it scatters back, and the
2999
2208
  cells of a whole-view scratch it compares. */
3000
2209
  st->last_data_off = data_off;
@@ -3041,8 +2250,6 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
3041
2250
  /* ======================================================================
3042
2251
  * !!! CORRECTNESS HAZARD - DO NOT MOVE !!!
3043
2252
  *
3044
- * (PROPOSAL_FIBER_DELIVERY F.1a)
3045
- *
3046
2253
  * Per-fiber scratch reuse + scatter correctness depends on the strict
3047
2254
  * evaluation order of the CA_FOR_EACH_FIBER_* macro sandwich:
3048
2255
  *
@@ -3069,14 +2276,13 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
3069
2276
  * instead of reusing -- separate phase, separate design.
3070
2277
  * ====================================================================== */
3071
2278
 
3072
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1: per-fiber fused WRITE PUT.
3073
- When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, the author
2279
+ /* PER_FIBER_FUSED write: the author
3074
2280
  wrote into fiber_data_scratch and there is no whole-view buffer
3075
2281
  to xfer_all PUT. Rebuild the same fiber region from cached
3076
2282
  fiber_region_starts[] (= captured pre-advance in next_slab_axes,
3077
- same hazard pattern as F.1a last_data_off) and call
3078
- ca_xfer_stride(src, ..., PUT) which routes through the view's
3079
- fused PUT path (X.1 / X.4) for 1-pass scatter back. */
2283
+ the same hazard as last_data_off) and call ca_xfer_stride(src, ...,
2284
+ PUT), which puts the fiber back through the view's region transfer
2285
+ in one pass. */
3080
2286
  if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
3081
2287
  ca_size_t fiber_n = st->slab_dims[0];
3082
2288
  int8_t nd = st->ndim;
@@ -3091,7 +2297,7 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
3091
2297
  return;
3092
2298
  }
3093
2299
 
3094
- /* Per-fiber gather path scatter (PROPOSAL_FIBER_DELIVERY F.1a).
2300
+ /* Per-fiber gather path scatter.
3095
2301
  When next_slab_axes gathered the fiber into fiber_data_scratch
3096
2302
  (= naxes==1 + slab_strides[0] != bytes), scatter it back to the
3097
2303
  source layout via the strided write. When the fiber was the
@@ -3116,77 +2322,48 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
3116
2322
  it fires for any source whose fiber is not innermost-contig -- so
3117
2323
  for SRC_ATTACH this has just written the fiber back into the
3118
2324
  whole-view scratch, and the push-back below carries it to the
3119
- source. For the alias paths scratch_ptr is NULL and the switch
3120
- below early-returns. */
3121
- }
3122
-
3123
- /* SRC_ATTACH path: the kernel wrote into the iterator's whole-view
3124
- scratch. The changed cells go back through the view's own transfer,
3125
- which converts, packs or scatters them as the view requires.
3126
-
3127
- This has to stay below the two per-fiber blocks: consulting
3128
- src_kind first would push back the whole-view buffer while the
3129
- fiber the author actually wrote sat unscattered in
3130
- fiber_data_scratch, and would push back a NULL one for
3131
- PER_FIBER_FUSED, which owns no whole-view buffer at all. */
3132
- if ( st->src_kind == CA_ITER_SRC_ATTACH ) {
3133
- ki_scratch_push_changed(st);
3134
- st->write_dirty = 0;
3135
- return;
2325
+ source. For the alias paths scratch_ptr is NULL and the push-back
2326
+ below is skipped. */
3136
2327
  }
3137
2328
 
3138
- /* alias path: kernel wrote through alias_ptr into parent directly
3139
- (case A semantics, PROPOSAL_T1_WRITE_SEMANTICS.md §(a)). No
3140
- scatter needed. Applies to all CAStride alias and to
3141
- STRIDE-only descriptor alias (= descriptor L2 alias future
3142
- optimisation, not yet enabled — but if it lands, scratch_ptr
3143
- stays NULL and we no-op correctly). */
2329
+ /* alias path: the kernel wrote through alias_ptr into the memory the
2330
+ walk borrowed, so nothing is scattered. Applies to all CAStride alias and to
2331
+ STRIDE-only descriptor alias (SRC_DESCRIPTOR_L2_ALIASABLE), whose
2332
+ scratch_ptr is NULL too. An alias walk over a view root is written
2333
+ back once, by finish (ca_sync_detach on the root). */
3144
2334
  if ( st->scratch_ptr == NULL ) return;
3145
2335
 
3146
- /* scratch path: scatter back the materialised buffer into the
3147
- source view. src_kind chooses the engine: */
3148
- switch ( st->src_kind ) {
3149
- case CA_ITER_SRC_CASTRIDE:
3150
- /* Two producers of CAStride + scratch: L1 non-contig, and the L2
3151
- non-entity-root path (init_l2's use_view_scratch). Both gathered
3152
- with ca_copy_data into a view row-major buffer; the changed
3153
- cells go back through the view's own transfer, which asks the
3154
- root for the region it owns. */
3155
- ki_scratch_push_changed(st);
3156
- break;
3157
-
3158
- case CA_ITER_SRC_DESCRIPTOR: {
3159
- /* Sub-step 5.4: descriptor framework scatter back.
3160
- ca_axis_dispatch_scatter is the P3-landed engine entry that
3161
- handles per-axis kind (STRIDE / INDEX / SHIFT) gather direction
3162
- in reverse — INDEX duplicates yield last-write-wins (R5 spec),
3163
- SHIFT OOB cells in CAWindow FILL policy are skipped (no parent
3164
- destination), CAShift WRAP/REFLECT bounds map back to interior
3165
- and are written normally. Iteration order is engine-defined;
3166
- user kernels must not rely on it.
3167
-
3168
- scatter wants the pre-merge raw descriptors (the engine re-runs
3169
- _prepare internally with whatever we give it). init cached
3170
- the post-merge axes in st->descs for next_slab_strided's
3171
- offset math; we re-call describe_axes here for the scatter
3172
- call rather than caching a second copy in the state struct. */
3173
- ca_axis_desc_t raw_descs[CA_RANK_MAX];
3174
- ca_size_t raw_pdims[CA_RANK_MAX];
3175
- int8_t raw_ndim = 0;
3176
- ca_iter_describe_axes(st->src, raw_descs, raw_pdims, &raw_ndim);
3177
- ca_axis_dispatch_scatter(st->root /* parent */,
3178
- raw_pdims,
3179
- raw_descs, raw_ndim,
3180
- st->bytes, st->total_elements,
3181
- st->scratch_ptr);
3182
- break;
3183
- }
3184
- }
2336
+ /* scratch path: the kernel wrote into a whole-view scratch, and the
2337
+ changed cells go back through the view's own transfer, which
2338
+ converts, packs or scatters them as the view requires (promise 3).
2339
+ Every write walk that reads a scratch gathered it with
2340
+ ki_gather_view, which keeps the copy the comparison needs: the
2341
+ transfer walk (SRC_ATTACH) and the CAStride paths that read a
2342
+ gathered copy. The descriptor engine's own gathers serve reads
2343
+ only; a write over a descriptor view is sent to the transfer walk
2344
+ at init.
2345
+
2346
+ This has to stay below the two per-fiber blocks: pushing back first
2347
+ would send the whole-view buffer while the fiber the author actually
2348
+ wrote sat unscattered in fiber_data_scratch. */
2349
+ if ( st->src_kind != CA_ITER_SRC_ATTACH
2350
+ && st->src_kind != CA_ITER_SRC_CASTRIDE ) {
2351
+ rb_bug("kernel iterator: a write walk over a scratch with nothing to "
2352
+ "compare it against (src_kind %d)", (int) st->src_kind);
2353
+ }
2354
+ ki_scratch_push_changed(st);
3185
2355
  st->write_dirty = 0;
3186
2356
  }
3187
2357
 
3188
2358
  static VALUE ca_iter_finish_sync (VALUE arg);
3189
2359
 
2360
+ static VALUE
2361
+ ki_sync_detach_root (VALUE root)
2362
+ {
2363
+ ca_sync_detach((CArray *) root);
2364
+ return Qnil;
2365
+ }
2366
+
3190
2367
  void
3191
2368
  ca_iter_state_finish (ca_iter_state *st)
3192
2369
  {
@@ -3205,11 +2382,11 @@ ca_iter_state_finish (ca_iter_state *st)
3205
2382
  /* Lifecycle cleanup — orders matter slightly (free scratch before
3206
2383
  detaching parent so the kernel iterator's resources are released
3207
2384
  symmetrically with init):
3208
- - scratch_ptr: owned by iter (CAStride non-contig L1, or
3209
- descriptor materialise via ca_axis_dispatch_attach). xfree.
3210
- - root: descriptor parent attached at init (sub-step 5.1+) or
3211
- CAStride L2 compose-fold root attached at init (step 3).
3212
- ca_detach.
2385
+ - scratch_ptr: owned by the iterator (every gather into scratch:
2386
+ CAStride non-contig, descriptor, SRC_ATTACH, per-slab, stack).
2387
+ ki_data_free.
2388
+ - root: descriptor parent or CAStride compose-fold root attached
2389
+ at init. ca_detach, or ca_sync_detach on a WRITE walk.
3213
2390
  - else (alias paths): src was attached at init, detach. */
3214
2391
  if ( st->scratch_ptr ) {
3215
2392
  ki_data_free(st->scratch_ptr, st->scratch_cap);
@@ -3220,7 +2397,7 @@ ca_iter_state_finish (ca_iter_state *st)
3220
2397
  xfree(st->scratch_mask);
3221
2398
  st->scratch_mask = NULL;
3222
2399
  }
3223
- /* PROPOSAL_FIBER_DELIVERY F.1a/F.1b: per-fiber scratch lifecycle. */
2400
+ /* The per-fiber scratch. */
3224
2401
  if ( st->fiber_data_scratch ) {
3225
2402
  ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
3226
2403
  st->fiber_data_scratch = NULL;
@@ -3231,9 +2408,10 @@ ca_iter_state_finish (ca_iter_state *st)
3231
2408
  st->fiber_mask_scratch = NULL;
3232
2409
  st->fiber_mask_scratch_cap = 0;
3233
2410
  }
3234
- /* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2 + P.2 Case A: detach
3235
- K parents (+ K parent masks if cached) and free the cached ptr
3236
- arrays. Symmetric with init_l2 per-parent ca_attach loop. */
2411
+ /* Detach the K parents init attached and free the cached ptr array
2412
+ (symmetric with init_l2's per-parent ca_attach loop). The parent
2413
+ mask copies are freed separately below; they were never attached
2414
+ for the walk. */
3237
2415
  if ( st->stack_parent_ptrs ) {
3238
2416
  CAStack *stack = (CAStack *) st->src;
3239
2417
  for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
@@ -3242,7 +2420,7 @@ ca_iter_state_finish (ca_iter_state *st)
3242
2420
  xfree(st->stack_parent_ptrs);
3243
2421
  st->stack_parent_ptrs = NULL;
3244
2422
  }
3245
- /* pilot/castack-axis0-loop-interchange: free tile cache. */
2423
+ /* The STACK tile cache. */
3246
2424
  if ( st->stack_tile_cache ) {
3247
2425
  ki_data_free(st->stack_tile_cache, 0);
3248
2426
  st->stack_tile_cache = NULL;
@@ -3260,37 +2438,52 @@ ca_iter_state_finish (ca_iter_state *st)
3260
2438
  st->stack_parent_mask_ptrs = NULL;
3261
2439
  }
3262
2440
  st->stack_n_parents = 0;
3263
- if ( st->root ) {
3264
- /* A write walk wrote into the root's attach buffer, which is the root
3265
- itself only for an entity: a view root (a transpose, a non-contig
3266
- block, a view lending no memory) gave a buffer of its own. */
3267
- CArray *root = st->root;
3268
- st->root = NULL;
3269
- if ( st->flags & CA_KERNEL_WRITE ) {
3270
- ca_sync_detach(root);
2441
+ {
2442
+ /* The state is emptied before anything here can raise: syncing a
2443
+ write walk's root goes through the root's slots. A finish that
2444
+ raised is then finished already, and a second call returns at once
2445
+ instead of detaching src again. */
2446
+ CArray *root = st->root;
2447
+ CArray *src = st->src;
2448
+ CArray *own = (st->flags & CA_ITER_FLAG_OWNS_SRC) ? src : NULL;
2449
+ int write = (st->flags & CA_KERNEL_WRITE) != 0;
2450
+ int detach_src = root == NULL
2451
+ && ( st->alias_mode == CA_ITER_ALIAS_CONTIG
2452
+ || st->alias_mode == CA_ITER_ALIAS_STRIDED );
2453
+ int tag = 0;
2454
+ st->root = NULL;
2455
+ st->src = NULL;
2456
+ st->alias_ptr = NULL;
2457
+ st->flags &= ~CA_ITER_FLAG_OWNS_SRC;
2458
+ if ( root ) {
2459
+ /* A write walk wrote into the root's attach buffer, which is the root
2460
+ itself only for an entity: a view root (a transpose, a non-contig
2461
+ block, a view lending no memory) gave a buffer of its own. */
2462
+ if ( write ) {
2463
+ /* protect: frees the iterator's own copy before the raise passes on */
2464
+ rb_protect(ki_sync_detach_root, (VALUE) root, &tag);
2465
+ }
2466
+ else {
2467
+ ca_detach(root);
2468
+ }
3271
2469
  }
3272
- else {
3273
- ca_detach(root);
2470
+ else if ( detach_src ) {
2471
+ /* alias path (CONTIG/STRIDED for CAStride family): we attached
2472
+ src directly at init, so detach it here. A SRC_ATTACH walk owns
2473
+ its own scratch (alias_mode = NONE, freed above) and attached
2474
+ nothing to detach. */
2475
+ ca_detach(src);
3274
2476
  }
3275
- } else if ( st->alias_mode == CA_ITER_ALIAS_CONTIG
3276
- || st->alias_mode == CA_ITER_ALIAS_STRIDED ) {
3277
- /* alias path (CONTIG/STRIDED for CAStride family): we attached
3278
- src directly at init, so detach it here.
3279
- Note: SRC_ATTACH used to be in this list (ALIAS_ATTACH); after
3280
- the 2026-05-31 refactor it owns its own scratch (alias_mode =
3281
- NONE, scratch_ptr xfree'd above), so no src.detach needed. */
3282
- ca_detach(st->src);
3283
- }
3284
- if ( st->flags & CA_ITER_FLAG_OWNS_SRC ) {
3285
- CArray *own = st->src;
3286
- st->flags &= ~CA_ITER_FLAG_OWNS_SRC;
3287
- if ( ca_is_object_type(own) ) {
3288
- ca_gc_release_buffer(own->ptr);
2477
+ if ( own ) {
2478
+ if ( ca_is_object_type(own) ) {
2479
+ ca_gc_release_buffer(own->ptr);
2480
+ }
2481
+ ca_free(own);
2482
+ }
2483
+ if ( tag ) {
2484
+ rb_jump_tag(tag);
3289
2485
  }
3290
- ca_free(own);
3291
2486
  }
3292
- st->src = NULL;
3293
- st->alias_ptr = NULL;
3294
2487
  }
3295
2488
 
3296
2489
  /* Finish a walk the way the block macros leave it: a slab of a write walk
@@ -3352,15 +2545,31 @@ typedef struct {
3352
2545
  char **out_ptr;
3353
2546
  boolean8_t **out_mask;
3354
2547
  ca_size_t *out_n;
3355
- ca_size_t *out_stride_bytes;
3356
2548
  int rc;
3357
2549
  } ca_iter_call_t;
3358
2550
 
3359
- /* Whether next_slab / sync_slab on this walk need the protection. */
2551
+ /* Whether next_slab / sync_slab on this walk need the protection.
2552
+
2553
+ Shortcut PROTECT_ONLY_MAY_RAISE.
2554
+ Layer the exits (promise 4): which walks run next_slab /
2555
+ sync_slab under rb_protect.
2556
+ Speeds up walks that cannot raise there: they skip rb_protect.
2557
+ Why about 1.5x on a per-axis sum over 200k short fibers.
2558
+ Enters the route is not PER_FIBER_FUSED and the walk does not
2559
+ write back out of scratch.
2560
+ Promises 4 (a walk left out cannot raise between init and finish).
2561
+ Switch ca_fast_paths_enabled(): off, every walk is protected.
2562
+ Checked by test_iter_raise_cleanup.rb; the raise rows of
2563
+ test_iter_delivery_detectors.rb. */
3360
2564
  static void
3361
2565
  ca_iter_mark_walk_may_raise (ca_iter_state *st)
3362
2566
  {
3363
2567
  if ( st == NULL ) return;
2568
+ if ( ! ca_fast_paths_enabled() ) {
2569
+ /* the general path protects every walk */
2570
+ st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
2571
+ return;
2572
+ }
3364
2573
  if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
3365
2574
  /* gathers each fiber through the source's xfer_stride slot */
3366
2575
  st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
@@ -3369,7 +2578,8 @@ ca_iter_mark_walk_may_raise (ca_iter_state *st)
3369
2578
  && (st->scratch_ptr != NULL
3370
2579
  || st->src_kind == CA_ITER_SRC_ATTACH) ) {
3371
2580
  /* writes back out of iterator-owned scratch, through the source's
3372
- xfer_all / sync_data slot. A write into aliased memory needs no
2581
+ transfer (ki_scratch_push_changed: ca_xfer_addrs / ca_xfer_all PUT)
2582
+ or the descriptor scatter. A write into aliased memory needs no
3373
2583
  push-back at all: sync_slab returns at once. */
3374
2584
  st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
3375
2585
  }
@@ -3378,6 +2588,8 @@ ca_iter_mark_walk_may_raise (ca_iter_state *st)
3378
2588
  /* When body raises, finish st -- and the state it is paired with (see
3379
2589
  ca_iter_state_init_l2_paired), which is open in the same frame and
3380
2590
  would otherwise be jumped over. */
2591
+ static VALUE ca_iter_ensure_finish_one (VALUE arg);
2592
+
3381
2593
  static void
3382
2594
  ca_iter_protect (ca_iter_state *st, VALUE (*body)(VALUE), VALUE arg)
3383
2595
  {
@@ -3385,10 +2597,16 @@ ca_iter_protect (ca_iter_state *st, VALUE (*body)(VALUE), VALUE arg)
3385
2597
  rb_protect(body, arg, &tag);
3386
2598
  if ( tag ) {
3387
2599
  ca_iter_state *pair = (ca_iter_state *) st->pair;
3388
- ca_iter_state_finish(st);
2600
+ volatile VALUE error = rb_errinfo();
2601
+ int ignored = 0;
2602
+ /* protect: finishes the pair even when finishing st raises; the
2603
+ body's exception is the one passed on */
2604
+ rb_protect(ca_iter_ensure_finish_one, (VALUE) st, &ignored);
3389
2605
  if ( pair != NULL ) {
3390
- ca_iter_state_finish(pair);
2606
+ /* protect: as above */
2607
+ rb_protect(ca_iter_ensure_finish_one, (VALUE) pair, &ignored);
3391
2608
  }
2609
+ rb_set_errinfo(error);
3392
2610
  rb_jump_tag(tag);
3393
2611
  }
3394
2612
  }
@@ -3420,15 +2638,6 @@ ca_iter_call_next_slab (VALUE arg)
3420
2638
  return Qnil;
3421
2639
  }
3422
2640
 
3423
- static VALUE
3424
- ca_iter_call_next_slab_strided (VALUE arg)
3425
- {
3426
- ca_iter_call_t *c = (ca_iter_call_t *) arg;
3427
- c->rc = ca_iter_state_next_slab_strided_unprotected(c->st, c->out_ptr,
3428
- c->out_mask, c->out_n,
3429
- c->out_stride_bytes);
3430
- return Qnil;
3431
- }
3432
2641
 
3433
2642
  static VALUE
3434
2643
  ca_iter_call_next_slab_axes (VALUE arg)
@@ -3520,24 +2729,6 @@ ca_iter_state_next_slab (ca_iter_state *st,
3520
2729
  return ca_iter_mark_dirty(st, c.rc);
3521
2730
  }
3522
2731
 
3523
- int
3524
- ca_iter_state_next_slab_strided (ca_iter_state *st,
3525
- char **out_ptr,
3526
- boolean8_t **out_mask,
3527
- ca_size_t *out_n,
3528
- ca_size_t *out_stride_bytes)
3529
- {
3530
- ca_iter_call_t c;
3531
- if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
3532
- return ca_iter_mark_dirty(st,
3533
- ca_iter_state_next_slab_strided_unprotected(st, out_ptr, out_mask,
3534
- out_n, out_stride_bytes));
3535
- }
3536
- c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.out_n = out_n;
3537
- c.out_stride_bytes = out_stride_bytes; c.rc = 0;
3538
- ca_iter_protect(st, ca_iter_call_next_slab_strided, (VALUE) &c);
3539
- return ca_iter_mark_dirty(st, c.rc);
3540
- }
3541
2732
 
3542
2733
  int
3543
2734
  ca_iter_state_next_slab_axes (ca_iter_state *st,
@@ -3673,15 +2864,39 @@ typedef struct {
3673
2864
  ca_iter_state *states;
3674
2865
  } ca_iter_ensure_t;
3675
2866
 
2867
+ static VALUE
2868
+ ca_iter_ensure_finish_one (VALUE arg)
2869
+ {
2870
+ ca_iter_state_finish((ca_iter_state *) arg);
2871
+ return Qnil;
2872
+ }
2873
+
3676
2874
  /* Finishes the walks in reverse order of their slots, as nested windows
3677
- close. ca_iter_state_finish only detaches and frees, so it cannot
3678
- raise in turn. */
2875
+ close. A finish can raise: on a WRITE walk whose root is a view, it
2876
+ syncs the root (ca_sync_detach) through the view's slots. Each finish
2877
+ is therefore run under rb_protect, so a raise from one does not leave
2878
+ the states before it unfinished; the first raise is re-raised once
2879
+ every state is closed. */
3679
2880
  static VALUE
3680
2881
  ca_iter_ensure_close (VALUE arg)
3681
2882
  {
3682
2883
  ca_iter_ensure_t *e = (ca_iter_ensure_t *) arg;
2884
+ int first = 0;
2885
+ volatile VALUE first_err = Qnil;
3683
2886
  for ( int32_t i = e->n - 1; i >= 0; i-- ) {
3684
- ca_iter_state_finish(&e->states[i]);
2887
+ int status = 0;
2888
+ rb_protect(ca_iter_ensure_finish_one, (VALUE) &e->states[i], &status);
2889
+ if ( status ) {
2890
+ if ( ! first ) {
2891
+ first = status;
2892
+ first_err = rb_errinfo();
2893
+ }
2894
+ rb_set_errinfo(Qnil);
2895
+ }
2896
+ }
2897
+ if ( first ) {
2898
+ rb_set_errinfo(first_err);
2899
+ rb_jump_tag(first);
3685
2900
  }
3686
2901
  return Qnil;
3687
2902
  }
@@ -3712,10 +2927,27 @@ ca_iter_ensure (int32_t n, ca_iter_state *states,
3712
2927
  * or `CARRAY_DEV=1 rake build_ext`). These helpers expose internal
3713
2928
  * engine state to Ruby for spec_ai regression pins. Do not consume
3714
2929
  * from user code.
3715
- *
3716
- * See devel/PROPOSAL_SMOKE_DEV_BUILD_GATE.md
3717
2930
  * ============================================================ */
3718
2931
 
2932
+ /* The innermost-axis walk the L2 smokes run: one strided run per
2933
+ innermost row, handed out as (pointer, cell count, byte stride). */
2934
+ static int
2935
+ t1_smoke_open_rows (ca_iter_state *st, CArray *src, uint32_t flags)
2936
+ {
2937
+ int8_t ax = (int8_t) (src->ndim - 1);
2938
+ return ca_iter_state_init_l2(st, src, CA_SLAB_AXES, &ax, 1, flags);
2939
+ }
2940
+
2941
+ static int
2942
+ t1_smoke_next_row (ca_iter_state *st, char **p, ca_size_t *n,
2943
+ ca_size_t *stride)
2944
+ {
2945
+ if ( ! ca_iter_state_next_slab_axes(st, p, NULL) ) return 0;
2946
+ *n = st->slab_dims[0];
2947
+ *stride = st->slab_strides[0];
2948
+ return 1;
2949
+ }
2950
+
3719
2951
  /* ---- Ruby smoke surface --------------------------------------------- */
3720
2952
 
3721
2953
  /* L1 smoke (steps 1+2): CArray.t1_smoke(ca) -> Hash with
@@ -3760,7 +2992,7 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
3760
2992
  return result;
3761
2993
  }
3762
2994
 
3763
- /* L2 smoke (step 3): CArray.t1_smoke_strided(ca) -> Hash with
2995
+ /* L2 smoke: CArray.t1_smoke_strided(ca) -> Hash with
3764
2996
  rc => Integer
3765
2997
  slabs => Integer (= total_slabs)
3766
2998
  total_elems => Integer
@@ -3769,8 +3001,6 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
3769
3001
  reported (ptr, n, stride_bytes) tuples, in
3770
3002
  iteration order; should equal view.to_ca.dump_binary
3771
3003
  strides => Array<Integer> — stride_bytes per yielded slab
3772
- (constant across yields under WHOLE policy in
3773
- step 3; surfaced for inspection)
3774
3004
  On error rc != OK, the walk fields are 0 / empty. */
3775
3005
  static VALUE
3776
3006
  rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
@@ -3789,9 +3019,9 @@ rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
3789
3019
  data = rb_str_new(0, 0);
3790
3020
  strides_arr = rb_ary_new();
3791
3021
 
3792
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
3022
+ rc = t1_smoke_open_rows(&st, src, 0);
3793
3023
  if ( rc == CA_ITER_OK ) {
3794
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
3024
+ while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
3795
3025
  if ( p != NULL && n > 0 ) {
3796
3026
  ca_size_t i;
3797
3027
  for ( i = 0; i < n; i++ ) {
@@ -3838,15 +3068,15 @@ rb_t1_smoke_sum_strided_f64 (VALUE klass, VALUE vsrc)
3838
3068
  "t1_smoke_sum_strided_f64 expects a float64 source");
3839
3069
  }
3840
3070
 
3841
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
3071
+ rc = t1_smoke_open_rows(&st, src, 0);
3842
3072
  if ( rc != CA_ITER_OK ) {
3843
3073
  rb_raise(rb_eRuntimeError, "ca_iter_state_init L2 failed (rc=%d)", rc);
3844
3074
  }
3845
3075
 
3846
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
3847
- /* Step 8.1: macro picks contig fast path when stride == sizeof(double),
3848
- falls back to strided loop otherwise. Removes the SIMD inhibition
3849
- observed in step 5.5 §10.4.5 on descriptor materialise paths. */
3076
+ while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
3077
+ /* CA_L2_FOR_EACH takes a contiguous loop when stride ==
3078
+ sizeof(double), which the compiler can vectorise, and a strided
3079
+ loop otherwise. */
3850
3080
  CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
3851
3081
  acc += *dp;
3852
3082
  });
@@ -3887,9 +3117,8 @@ rb_t1_smoke_write_fill_f64 (VALUE klass, VALUE vsrc, VALUE vval)
3887
3117
  return INT2NUM(CA_ITER_OK);
3888
3118
  }
3889
3119
 
3890
- /* WRITE smoke: in-place fill via L2 strided dispatch. Used by the
3891
- step 5.5 aggregate bench to round out the matrix (L1 WRITE was
3892
- already in t1_smoke_write_fill_f64). Kernel walks the strided
3120
+ /* WRITE smoke: in-place fill via an L2 walk (the L1 form is
3121
+ t1_smoke_write_fill_f64). Kernel walks the strided
3893
3122
  slab and writes val at every position; sync_slab scatters back. */
3894
3123
  static VALUE
3895
3124
  rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
@@ -3906,14 +3135,12 @@ rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
3906
3135
  rb_raise(rb_eTypeError, "expects a float64 source");
3907
3136
  }
3908
3137
 
3909
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0,
3910
- CA_KERNEL_WRITE);
3138
+ rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
3911
3139
  if ( rc != CA_ITER_OK ) return INT2NUM(rc);
3912
3140
 
3913
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
3914
- /* Step 8.1: contig fast path via macro (stride == sizeof(double)
3915
- on materialise scratch, which is the descriptor L2 path that
3916
- hit +18-22% in step 5.5 — macro removes that overhead). */
3141
+ while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
3142
+ /* Contiguous loop when stride == sizeof(double) (a gathered
3143
+ scratch), strided otherwise. */
3917
3144
  CA_L2_FOR_EACH(double, p, n, stride, dp, {
3918
3145
  *dp = v;
3919
3146
  });
@@ -3981,13 +3208,11 @@ cmp_double (const void *a, const void *b)
3981
3208
 
3982
3209
  /* WRITE smoke: per-row sort via L2 strided dispatch. For a 2D
3983
3210
  float64 src of shape [m, n], sort each of the m rows in ascending
3984
- order. Uses next_slab_strided so each row is a strided slab; the
3211
+ order. Walks the innermost axis so each row is a strided slab; the
3985
3212
  kernel materialises into a tight contig scratch, qsorts, then
3986
3213
  writes back via the same stride. This exercises L2 WRITE
3987
3214
  mechanics (multi-slab walk, per-slab fill of strided cells)
3988
- without over-engineering a strided qsort itself — that would be a
3989
- Pattern H specialised op, out of step 4 scope (reviewer advice #3).
3990
- */
3215
+ without a strided qsort of its own. */
3991
3216
  static VALUE
3992
3217
  rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
3993
3218
  {
@@ -4003,17 +3228,16 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
4003
3228
  rb_raise(rb_eTypeError, "expects a float64 source");
4004
3229
  }
4005
3230
 
4006
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0,
4007
- CA_KERNEL_WRITE);
3231
+ rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
4008
3232
  if ( rc != CA_ITER_OK ) return INT2NUM(rc);
4009
3233
 
4010
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
3234
+ while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
4011
3235
  if ( n > (ca_size_t) (sizeof(scratch) / sizeof(double)) ) {
4012
3236
  ca_iter_state_finish(&st);
4013
3237
  rb_raise(rb_eRuntimeError, "row too large for smoke scratch");
4014
3238
  }
4015
- /* strided -> contig scratch. Step 8.1: macro handles the
4016
- stride == bytes contig fast path automatically. */
3239
+ /* strided -> contig scratch (CA_L2_FOR_EACH takes the contiguous
3240
+ loop when stride == bytes). */
4017
3241
  ca_size_t _sc_i = 0;
4018
3242
  CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
4019
3243
  scratch[_sc_i++] = *dp;
@@ -4021,7 +3245,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
4021
3245
  /* in-place qsort on the contig scratch */
4022
3246
  qsort(scratch, n, sizeof(double), cmp_double);
4023
3247
  /* contig scratch -> strided (writes back to parent via alias_ptr,
4024
- case A direct write). Step 8.1: macro contig fast path. */
3248
+ a direct write). */
4025
3249
  ca_size_t _wb_i = 0;
4026
3250
  CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
4027
3251
  *dp = scratch[_wb_i++];
@@ -4032,7 +3256,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
4032
3256
  return INT2NUM(CA_ITER_OK);
4033
3257
  }
4034
3258
 
4035
- /* Step 6 smoke: L1 walk that exposes BOTH the value slab and the
3259
+ /* Mask smoke: L1 walk that exposes BOTH the value slab and the
4036
3260
  mask slab. Used to pin "mask is delivered to kernel" semantics.
4037
3261
  Returns rc + mask_seen (Boolean: was out_mask non-NULL at first
4038
3262
  yield) + mask_bytes (String: concatenated mask bytes). */
@@ -4108,9 +3332,9 @@ rb_t1_smoke_attach (VALUE klass, VALUE vsrc)
4108
3332
  return SIZE2NUM(total);
4109
3333
  }
4110
3334
 
4111
- /* L2 bench-grade smoke (sub-step 5.3+). Same shape as t1_smoke_attach
4112
- but uses init_l2 / next_slab_strided so descriptor sources can be
4113
- exercised through the L2 path. */
3335
+ /* L2 bench-grade smoke. Same shape as t1_smoke_attach
3336
+ but walks the innermost axis through init_l2 so descriptor sources
3337
+ can be exercised through the L2 path. */
4114
3338
  static VALUE
4115
3339
  rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
4116
3340
  {
@@ -4123,10 +3347,10 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
4123
3347
  volatile char sink = 0;
4124
3348
 
4125
3349
  TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
4126
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
3350
+ rc = t1_smoke_open_rows(&st, src, 0);
4127
3351
  if ( rc != CA_ITER_OK ) return INT2NUM(rc);
4128
3352
 
4129
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
3353
+ while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
4130
3354
  if ( p != NULL && n > 0 ) sink ^= p[0];
4131
3355
  total += n;
4132
3356
  }
@@ -4135,7 +3359,7 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
4135
3359
  return SIZE2NUM(total);
4136
3360
  }
4137
3361
 
4138
- /* step 7: minimal smoke that exposes the `flags` argument to Ruby
3362
+ /* Flags smoke: minimal smoke that exposes the `flags` argument to Ruby
4139
3363
  tests. Just runs init_l1 with the requested flags and returns
4140
3364
  the rc — used to verify NO_MASK enforcement (and any future
4141
3365
  flag-gated rejection paths) without needing a full kernel walk. */
@@ -4145,6 +3369,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
4145
3369
  CArray *src;
4146
3370
  ca_iter_state st;
4147
3371
  int rc;
3372
+ /* fits: a development smoke argument, not a cell */
4148
3373
  uint32_t flags = (uint32_t) NUM2UINT(vflags);
4149
3374
 
4150
3375
  TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
@@ -4153,7 +3378,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
4153
3378
  return INT2NUM(rc);
4154
3379
  }
4155
3380
 
4156
- /* ---- Phase A capstone: CA_SLAB_AXES smoke -------------------------
3381
+ /* ---- CA_SLAB_AXES smoke ----------------------------------------------
4157
3382
  sum a float64 source over user-specified slab axes via init_l2 +
4158
3383
  next_slab_axes. K-D walk over slab dims using slab_strides for data
4159
3384
  and slab_mask_strides for mask (so non-contig CAStride sources work
@@ -4183,10 +3408,11 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
4183
3408
  rb_raise(rb_eTypeError, "t1_smoke_sum_axes_f64 expects a float64 source");
4184
3409
  }
4185
3410
  naxes = (int8_t) (argc - 1);
4186
- if ( naxes > CA_RANK_MAX ) {
3411
+ if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
4187
3412
  rb_raise(rb_eArgError, "too many axes");
4188
3413
  }
4189
3414
  for ( i = 0; i < naxes; i++ ) {
3415
+ /* fits: an axis number, checked by the engine's init */
4190
3416
  slab_axes[i] = (int8_t) NUM2INT(argv[i + 1]);
4191
3417
  }
4192
3418
 
@@ -4222,7 +3448,7 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
4222
3448
  return DBL2NUM(acc);
4223
3449
  }
4224
3450
 
4225
- /* ---- Phase B.3 helper smoke: rb_ca_parse_reduce_axes ---------------
3451
+ /* ---- helper smoke: rb_ca_parse_reduce_axes ---------------------------
4226
3452
  Calls the helper and returns the parsed axes as a Ruby Array of
4227
3453
  Integer. Lets tests inspect parsing behaviour directly.
4228
3454
  Signature: CArray.t1_test_parse_reduce_axes(ca, *args) */
@@ -4248,7 +3474,7 @@ rb_t1_test_parse_reduce_axes (int argc, VALUE *argv, VALUE klass)
4248
3474
  return arr;
4249
3475
  }
4250
3476
 
4251
- /* ---- API harmonisation A.1 smoke: rb_ca_parse_reduce_axes_kw -------
3477
+ /* ---- helper smoke: rb_ca_parse_reduce_axes_kw ------------------------
4252
3478
  Calls the kwarg helper and returns the parsed axes as a Ruby Array.
4253
3479
  Lets tests inspect kwarg-form parsing (Qnil / Integer / Array) +
4254
3480
  validation parity with the variadic entry.
@@ -4276,7 +3502,7 @@ rb_test_parse_reduce_axes_kw (int argc, VALUE *argv, VALUE klass)
4276
3502
  return arr;
4277
3503
  }
4278
3504
 
4279
- /* ---- Phase A.3 helper smoke: rb_ca_new_reduced ---------------------
3505
+ /* ---- helper smoke: rb_ca_new_reduced ---------------------------------
4280
3506
  Exposes rb_ca_new_reduced (carray_core.c) for unit testing. Returns
4281
3507
  the newly allocated output CArray (= tests verify shape + data_type).
4282
3508
  Signature: CArray.t1_test_new_reduced(ca, data_type, axis1, axis2, ...) */
@@ -4291,21 +3517,21 @@ rb_t1_test_new_reduced (int argc, VALUE *argv, VALUE klass)
4291
3517
  rb_raise(rb_eArgError,
4292
3518
  "t1_test_new_reduced expects (ca, data_type, axis...) with >= 1 axis");
4293
3519
  }
4294
- /* PROPOSAL_DTYPE_SYMBOL_FLIP: accept Symbol / Integer / Class / String
4295
- uniformly via rb_ca_guess_type so post-flip CA_* (Symbol) callers
4296
- and legacy Integer-code callers both work. */
3520
+ /* Accept Symbol / Integer / Class / String alike through
3521
+ rb_ca_guess_type. */
4297
3522
  data_type = (int32_t) rb_ca_guess_type(argv[1]);
4298
3523
  naxes = (int8_t) (argc - 2);
4299
- if ( naxes > CA_RANK_MAX ) {
3524
+ if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
4300
3525
  rb_raise(rb_eArgError, "too many axes");
4301
3526
  }
4302
3527
  for ( int i = 0; i < naxes; i++ ) {
3528
+ /* fits: an axis number, checked by the engine's init */
4303
3529
  axes[i] = (int8_t) NUM2INT(argv[2 + i]);
4304
3530
  }
4305
3531
  return rb_ca_new_reduced(argv[0], axes, naxes, data_type, 0);
4306
3532
  }
4307
3533
 
4308
- /* ---- step 9.3: CAReduce-specific smoke -----------------------------
3534
+ /* ---- CAReduce smoke --------------------------------------------------
4309
3535
  CAReduce has no public Ruby ctor (it's an internal class used by
4310
3536
  CARefer mask handling). Tests need to drive kernel_iterator over
4311
3537
  a CAReduce, so we expose a thin construction helper plus dedicated
@@ -4370,8 +3596,9 @@ rb_t1_smoke_reduce_read (VALUE klass, VALUE vparent, VALUE vcount, VALUE voffset
4370
3596
  }
4371
3597
 
4372
3598
  /* WRITE broadcast smoke for CAReduce: fills the reduce view with
4373
- `val` (boolean / 0 or 1) and lets sync_slab → ca_sync run the
4374
- broadcast scatter back to parent. After the call parent's
3599
+ `val` (boolean / 0 or 1) and lets sync_slab send the changed cells
3600
+ back through the view (ki_scratch_push_changed), which broadcasts them
3601
+ to the parent. After the call parent's
4375
3602
  [offset..offset+elems*count) bytes should all equal val. Returns
4376
3603
  rc; on rc != OK parent is not modified. */
4377
3604
  static VALUE
@@ -4384,6 +3611,7 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
4384
3611
  ca_size_t n, i;
4385
3612
  int rc;
4386
3613
  CArray *reduce;
3614
+ /* fits: a development smoke argument, not a cell */
4387
3615
  uint8_t v = (uint8_t) NUM2UINT(vval);
4388
3616
 
4389
3617
  TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
@@ -4399,15 +3627,15 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
4399
3627
  }
4400
3628
  while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
4401
3629
  for ( i = 0; i < n; i++ ) p[i] = v;
4402
- ca_iter_state_sync_slab(&st); /* triggers ca_sync → broadcast */
3630
+ ca_iter_state_sync_slab(&st); /* sends the changed cells back */
4403
3631
  }
4404
3632
  ca_iter_state_finish(&st);
4405
3633
  ca_free(reduce);
4406
3634
  return INT2NUM(CA_ITER_OK);
4407
3635
  }
4408
3636
 
4409
- /* ---- M.6: CARemap kernel_iterator smoke helpers --------------------
4410
- CARemap has no public Ruby ctor (internal-only per §2.2), so we
3637
+ /* ---- CARemap smoke helpers --------------------------------------------
3638
+ CARemap has no public Ruby constructor, so we
4411
3639
  construct it from C using the (ref, idx) args. Pattern mirrors
4412
3640
  rb_t1_smoke_reduce_*: build the view, run the iterator state
4413
3641
  machine, return the standard hash (rc / slabs / total_elems /
@@ -4474,10 +3702,10 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
4474
3702
  view = ca_remap_new(ref, idx);
4475
3703
  data = rb_str_new(0, 0);
4476
3704
 
4477
- rc = ca_iter_state_init_l2(&st, view, CA_SLAB_WHOLE, NULL, 0, 0);
3705
+ rc = t1_smoke_open_rows(&st, view, 0);
4478
3706
  if ( rc == CA_ITER_OK ) {
4479
3707
  ca_size_t stride;
4480
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
3708
+ while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
4481
3709
  ca_size_t i;
4482
3710
  for ( i = 0; i < n; i++ ) {
4483
3711
  rb_str_cat(data, p + i * stride, st.bytes);
@@ -4500,7 +3728,8 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
4500
3728
  }
4501
3729
 
4502
3730
  /* WRITE smoke: fill view's slab buffer with `val` (float64), let
4503
- sync_slab → ca_sync run the scatter back to ref via xfer_all PUT. */
3731
+ sync_slab send the changed cells back to ref through the view
3732
+ (ki_scratch_push_changed: ca_xfer_addrs or ca_xfer_all PUT). */
4504
3733
  static VALUE
4505
3734
  rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
4506
3735
  VALUE vval)
@@ -4533,17 +3762,17 @@ rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
4533
3762
  return INT2NUM(CA_ITER_OK);
4534
3763
  }
4535
3764
 
4536
- /* ---- Phase C C.3: CA_FOR_EACH_SLAB macro smoke surfaces -------------
3765
+ /* ---- CA_FOR_EACH_SLAB macro smoke surfaces ---------------------------
4537
3766
  Smoke kernels that exercise the block macros end-to-end. Used by
4538
3767
  spec_ai/test_ca_for_each_slab_macros.rb to pin behavioural
4539
3768
  correctness of the macro expansion (= same byte-parity result as
4540
3769
  the raw API equivalent). */
4541
3770
 
4542
- /* sum reduction using CA_FOR_EACH_SLAB. innermost slab axis only,
4543
- float64 source, accumulator = scalar (single full-reduction slab
4544
- when naxes == src->ndim, otherwise per-outer-slab accumulator
4545
- written to a row-major output buffer). Mirrors carray_kernel_sum
4546
- structurally but uses the macro for lifecycle. */
3771
+ /* sum reduction using CA_FOR_EACH_SLAB over the slab axes given in
3772
+ argv[1..] (K-D slabs). Reads the source as float64 without checking
3773
+ its data_type. Accumulator = scalar (single full-reduction slab when
3774
+ naxes == src->ndim, otherwise per-outer-slab accumulator written to a
3775
+ row-major output buffer). */
4547
3776
  static VALUE
4548
3777
  rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
4549
3778
  {
@@ -4562,6 +3791,7 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
4562
3791
  rb_raise(rb_eArgError, "bad axes count");
4563
3792
  }
4564
3793
  for ( int8_t k = 0; k < naxes; k++ ) {
3794
+ /* fits: an axis number, checked by the engine's init */
4565
3795
  slab_axes[k] = (int8_t) NUM2INT(argv[1 + k]);
4566
3796
  if ( slab_axes[k] < 0 ) slab_axes[k] += ca->ndim;
4567
3797
  }
@@ -4631,8 +3861,8 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
4631
3861
 
4632
3862
  /* map kernel using CA_FOR_EACH_SLAB_INOUT. Doubles each element of
4633
3863
  a float64 source into a fresh same-shape output. Slab axis is
4634
- always the innermost (= argv[1] is unused for simplicity, axis
4635
- fixed to ca->ndim - 1). Demonstrates the 2-iter pattern. */
3864
+ always the innermost (ca->ndim - 1); there is no axis argument.
3865
+ Demonstrates the 2-iter pattern. */
4636
3866
  static VALUE
4637
3867
  rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
4638
3868
  {
@@ -4676,11 +3906,11 @@ rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
4676
3906
  return vout;
4677
3907
  }
4678
3908
 
4679
- /* ---- PROPOSAL_FIBER_DELIVERY F.2: catalog macro smokes -------------
3909
+ /* ---- CA_FOR_EACH_FIBER macro smokes ----------------------------------
4680
3910
  Exercise each of the 4 catalog forms end-to-end with float64 fibers,
4681
3911
  verifying contig delivery for both innermost (= stride==bytes) and
4682
3912
  non-innermost (= stride>bytes, gather path) axis positions. Used by
4683
- spec_ai/test_fiber_delivery.rb (= F.3). */
3913
+ spec_ai/test_fiber_delivery.rb. */
4684
3914
 
4685
3915
  /* form 1 (NO_MASK single): per-axis sum, returns total sum.
4686
3916
  Catalog: CA_FOR_EACH_FIBER. */
@@ -4706,8 +3936,7 @@ rb_caf_fiber_smoke_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4706
3936
  }
4707
3937
 
4708
3938
  /* form 2 (NO_MASK INOUT): per-axis fiber copy * scalar.
4709
- Catalog: CA_FOR_EACH_FIBER_INOUT. Sorts each fiber for non-trivial
4710
- gather/scatter behavior. */
3939
+ Catalog: CA_FOR_EACH_FIBER_INOUT. Doubles each cell of each fiber. */
4711
3940
  static VALUE
4712
3941
  rb_caf_fiber_smoke_double_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4713
3942
  {
@@ -4791,9 +4020,9 @@ rb_caf_fiber_smoke_zero_masked_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4791
4020
  return vout;
4792
4021
  }
4793
4022
 
4794
- /* F.5 bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
4795
- Functional equivalent of ext/carray_order.c::rb_ca_sort_copy_axis_*
4796
- (= hand-rolled gather/scatter with explicit slab_strides[0] loops).
4023
+ /* Bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
4024
+ Functional equivalent of the per-axis sort_copy (rb_ca_sort_copy in
4025
+ ext/carray_sort.c).
4797
4026
  The macro form lets us A/B the author-side overhead of stride math
4798
4027
  vs the catalog-macro-driven engine gather/scatter. */
4799
4028
  static int
@@ -4836,7 +4065,7 @@ rb_caf_fiber_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4836
4065
  return vout;
4837
4066
  }
4838
4067
 
4839
- /* F.5 bench helper variant: sort_copy via SLAB macro + manual
4068
+ /* Bench helper variant: sort_copy via SLAB macro + manual
4840
4069
  gather/scatter (= what hand-rolled sort_copy effectively does, but
4841
4070
  driven through the public CA_FOR_EACH_SLAB_INOUT macro instead of
4842
4071
  the raw next_slab_axes API). Lets us isolate macro overhead from
@@ -4902,7 +4131,7 @@ rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4902
4131
  return vout;
4903
4132
  }
4904
4133
 
4905
- /* F.5 follow-up bench: per-fiber fused xfer_stride direct into a contig
4134
+ /* Bench helper: per-fiber fused xfer_stride direct into a contig
4906
4135
  scratch, skipping the kernel_iterator SRC_ATTACH whole-view materialise.
4907
4136
  Tests user's hypothesis: for transform views (CAFake/CAByteSwap etc.),
4908
4137
  does bypassing the whole-view scratch_ptr and calling ca_xfer_stride
@@ -4981,6 +4210,14 @@ rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4981
4210
  return rb_float_new(total);
4982
4211
  }
4983
4212
 
4213
+ /* CArray.__iter_buffers__: see ki_buffers_out. */
4214
+ static VALUE
4215
+ rb_ki_buffers_out (VALUE klass)
4216
+ {
4217
+ (void) klass;
4218
+ return LONG2NUM(__atomic_load_n(&ki_buffers_out, __ATOMIC_RELAXED));
4219
+ }
4220
+
4984
4221
  #endif /* CARRAY_DEV_BUILD — end smoke surface fence */
4985
4222
 
4986
4223
  void
@@ -4990,10 +4227,12 @@ Init_ca_kernel_iterator (void)
4990
4227
 
4991
4228
  #ifdef CARRAY_DEV_BUILD
4992
4229
  /* ==== smoke surface registrations (dev-only, stripped in release) ====
4993
- * See PROPOSAL_SMOKE_DEV_BUILD_GATE.md. All `t1_smoke_*`, `caf_smoke_*`,
4230
+ * All `t1_smoke_*`, `caf_smoke_*`,
4994
4231
  * `caf_fiber_smoke_*`, `caf_*_bench_*`, helper smokes, and `T1_*` test
4995
4232
  * constants are gated here together — none of them are consumed by
4996
4233
  * production code (lib/ / other ext/), only by spec_ai regression pins. */
4234
+ rb_define_singleton_method(rb_cCArray, "__iter_buffers__",
4235
+ rb_ki_buffers_out, 0);
4997
4236
  rb_define_singleton_method(rb_cCArray, "t1_smoke",
4998
4237
  rb_t1_smoke, 1);
4999
4238
  rb_define_singleton_method(rb_cCArray, "t1_smoke_strided",
@@ -5016,20 +4255,20 @@ Init_ca_kernel_iterator (void)
5016
4255
  rb_t1_smoke_with_mask, 1);
5017
4256
  rb_define_singleton_method(rb_cCArray, "t1_smoke_init_rc",
5018
4257
  rb_t1_smoke_init_rc, 2);
5019
- /* Phase A capstone: CA_SLAB_AXES smoke (variadic axes) */
4258
+ /* CA_SLAB_AXES smoke (variadic axes) */
5020
4259
  rb_define_singleton_method(rb_cCArray, "t1_smoke_sum_axes_f64",
5021
4260
  rb_t1_smoke_sum_axes_f64, -1);
5022
- /* Phase A.3: rb_ca_new_reduced helper smoke */
4261
+ /* rb_ca_new_reduced helper smoke */
5023
4262
  rb_define_singleton_method(rb_cCArray, "t1_test_new_reduced",
5024
4263
  rb_t1_test_new_reduced, -1);
5025
- /* Phase B.3: rb_ca_parse_reduce_axes helper smoke */
4264
+ /* rb_ca_parse_reduce_axes helper smoke */
5026
4265
  rb_define_singleton_method(rb_cCArray, "t1_test_parse_reduce_axes",
5027
4266
  rb_t1_test_parse_reduce_axes, -1);
5028
4267
 
5029
- /* API harmonisation A.1: rb_ca_parse_reduce_axes_kw helper smoke */
4268
+ /* rb_ca_parse_reduce_axes_kw helper smoke */
5030
4269
  rb_define_singleton_method(rb_cCArray, "test_parse_reduce_axes_kw",
5031
4270
  rb_test_parse_reduce_axes_kw, -1);
5032
- /* CAReduce-specific (step 9.3): no public Ruby API */
4271
+ /* CAReduce: no public Ruby API */
5033
4272
  rb_define_singleton_method(rb_cCArray, "t1_make_reduce",
5034
4273
  rb_t1_make_reduce, 3);
5035
4274
  rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_read",
@@ -5037,19 +4276,19 @@ Init_ca_kernel_iterator (void)
5037
4276
  rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_write_broadcast",
5038
4277
  rb_t1_smoke_reduce_write_broadcast, 4);
5039
4278
 
5040
- /* M.6 CARemap kernel_iterator smokes (test-only). */
4279
+ /* CARemap smokes (test-only). */
5041
4280
  rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read",
5042
4281
  rb_t1_smoke_remap_read, 2);
5043
4282
  rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read_strided",
5044
4283
  rb_t1_smoke_remap_read_strided, 2);
5045
4284
  rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_write_fill_f64",
5046
4285
  rb_t1_smoke_remap_write_fill_f64, 3);
5047
- /* Phase C C.3 (2026-05-27): block macro smokes */
4286
+ /* block macro smokes */
5048
4287
  rb_define_singleton_method(rb_cCArray, "caf_smoke_sum_f64",
5049
4288
  rb_caf_smoke_sum_f64, -1);
5050
4289
  rb_define_singleton_method(rb_cCArray, "caf_smoke_double_f64",
5051
4290
  rb_caf_smoke_double_f64, 1);
5052
- /* PROPOSAL_FIBER_DELIVERY F.2: catalog macro smokes */
4291
+ /* fiber macro smokes */
5053
4292
  rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_sum_f64",
5054
4293
  rb_caf_fiber_smoke_sum_f64, 2);
5055
4294
  rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_double_f64",
@@ -5058,7 +4297,7 @@ Init_ca_kernel_iterator (void)
5058
4297
  rb_caf_fiber_smoke_unmasked_sum_f64, 2);
5059
4298
  rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_zero_masked_f64",
5060
4299
  rb_caf_fiber_smoke_zero_masked_f64, 2);
5061
- /* F.5 bench helper */
4300
+ /* bench helper */
5062
4301
  rb_define_singleton_method(rb_cCArray, "caf_fiber_bench_sort_copy_f64",
5063
4302
  rb_caf_fiber_bench_sort_copy_f64, 2);
5064
4303
  rb_define_singleton_method(rb_cCArray, "caf_slab_bench_sort_copy_f64",
@@ -5078,5 +4317,6 @@ Init_ca_kernel_iterator (void)
5078
4317
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_CONTIG", INT2NUM(CA_ITER_ALIAS_CONTIG));
5079
4318
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_STRIDED", INT2NUM(CA_ITER_ALIAS_STRIDED));
5080
4319
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_ATTACH", INT2NUM(CA_ITER_ALIAS_ATTACH));
4320
+ rb_define_const(rb_cCArray, "T1_ITER_ALIAS_PER_SLAB", INT2NUM(CA_ITER_ALIAS_PER_SLAB));
5081
4321
  #endif /* CARRAY_DEV_BUILD */
5082
4322
  }