carray 3.0.2 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +568 -0
  3. data/README.md +6 -6
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_descriptor.h +27 -24
  6. data/ext/ca_axis_dispatch.c +220 -0
  7. data/ext/ca_axis_group.c +201 -123
  8. data/ext/ca_binop_dispatch.c +46 -52
  9. data/ext/ca_binop_dispatch.h +1 -1
  10. data/ext/ca_categorical_iterator.c +249 -183
  11. data/ext/ca_compare.c +7 -12
  12. data/ext/ca_for_buffer.h +21 -68
  13. data/ext/ca_group_iter.c +38 -23
  14. data/ext/ca_kernel_iterator.c +1765 -2076
  15. data/ext/ca_kernel_iterator.h +584 -566
  16. data/ext/ca_monop_dispatch.c +86 -63
  17. data/ext/ca_monop_dispatch.h +5 -0
  18. data/ext/ca_obj_array.c +223 -56
  19. data/ext/ca_obj_bincmp.c +172 -138
  20. data/ext/ca_obj_binop.c +116 -230
  21. data/ext/ca_obj_bitarray.c +40 -26
  22. data/ext/ca_obj_bitfield.c +100 -68
  23. data/ext/ca_obj_block.c +19 -9
  24. data/ext/ca_obj_byte_swap.c +45 -46
  25. data/ext/ca_obj_const_string.c +269 -44
  26. data/ext/ca_obj_face.c +612 -89
  27. data/ext/ca_obj_face.h +111 -47
  28. data/ext/ca_obj_fake.c +36 -79
  29. data/ext/ca_obj_farray.c +7 -3
  30. data/ext/ca_obj_field.c +33 -8
  31. data/ext/ca_obj_fixlen_string.c +23 -24
  32. data/ext/ca_obj_grid.c +541 -210
  33. data/ext/ca_obj_meld.c +331 -111
  34. data/ext/ca_obj_moncmp.c +80 -65
  35. data/ext/ca_obj_monop.c +177 -149
  36. data/ext/ca_obj_object.c +664 -248
  37. data/ext/ca_obj_record.c +63 -23
  38. data/ext/ca_obj_reduce.c +18 -16
  39. data/ext/ca_obj_refer.c +140 -58
  40. data/ext/ca_obj_remap.c +85 -24
  41. data/ext/ca_obj_repeat.c +24 -9
  42. data/ext/ca_obj_roll.c +121 -16
  43. data/ext/ca_obj_select.c +61 -58
  44. data/ext/ca_obj_select_axis.c +73 -107
  45. data/ext/ca_obj_shift.c +32 -102
  46. data/ext/ca_obj_stack.c +372 -122
  47. data/ext/ca_obj_stride.c +348 -140
  48. data/ext/ca_obj_string.c +88 -22
  49. data/ext/ca_obj_tile.c +38 -21
  50. data/ext/ca_obj_time.c +45 -31
  51. data/ext/ca_obj_timedelta.c +43 -29
  52. data/ext/ca_obj_transpose.c +17 -6
  53. data/ext/ca_obj_triop.c +125 -141
  54. data/ext/ca_obj_window.c +330 -44
  55. data/ext/ca_op_ipower.c +24 -11
  56. data/ext/ca_op_powi.h +57 -30
  57. data/ext/ca_pow5_table.h +656 -0
  58. data/ext/ca_sort_kernels.h +70 -0
  59. data/ext/ca_sweep_engine.c +104 -81
  60. data/ext/ca_sweep_engine.h +7 -20
  61. data/ext/ca_triop_dispatch.c +4 -1
  62. data/ext/ca_triop_dispatch.h +5 -2
  63. data/ext/caframe_csv_reader.c +542 -0
  64. data/ext/carray.h +241 -92
  65. data/ext/carray_access.c +608 -276
  66. data/ext/carray_address_basis.c +494 -97
  67. data/ext/carray_attribute.c +129 -4
  68. data/ext/carray_bincount.c +39 -7
  69. data/ext/carray_broadcast.c +56 -19
  70. data/ext/carray_build_flags.h +1 -1
  71. data/ext/carray_call_cfunc.c +108 -28
  72. data/ext/carray_cast.c +1576 -223
  73. data/ext/carray_cast_func.rb +77 -24
  74. data/ext/carray_class.c +4 -3
  75. data/ext/carray_conversion.c +208 -75
  76. data/ext/carray_copy.c +82 -24
  77. data/ext/carray_core.c +646 -162
  78. data/ext/carray_count.c +171 -88
  79. data/ext/carray_element.c +94 -60
  80. data/ext/carray_factorize.c +670 -324
  81. data/ext/carray_generate.c +260 -83
  82. data/ext/carray_histogram.c +209 -106
  83. data/ext/carray_hold.c +14 -16
  84. data/ext/carray_index_classifier.c +422 -243
  85. data/ext/carray_internal.h +692 -3
  86. data/ext/carray_kernels_bincmp.c +7 -0
  87. data/ext/carray_kernels_binop.c +487 -120
  88. data/ext/carray_kernels_init.c +653 -50
  89. data/ext/carray_kernels_moncmp.c +15 -8
  90. data/ext/carray_kernels_monop.c +99 -43
  91. data/ext/carray_kernels_reduce_aggregate.c +3925 -6112
  92. data/ext/carray_kernels_reduce_boolean.c +112 -36
  93. data/ext/carray_kernels_reduce_cumulative.c +2127 -6903
  94. data/ext/carray_kernels_reduce_extreme.c +3032 -2892
  95. data/ext/carray_kernels_reduce_variance.c +439 -500
  96. data/ext/carray_kernels_scan.c +1097 -488
  97. data/ext/carray_kernels_search.c +3787 -2176
  98. data/ext/carray_kernels_sort.c +1200 -920
  99. data/ext/carray_kernels_triop.c +479 -7
  100. data/ext/carray_key_windows.c +216 -0
  101. data/ext/carray_lazy.c +967 -153
  102. data/ext/carray_loop.c +81 -65
  103. data/ext/carray_mask.c +910 -324
  104. data/ext/carray_math_kernel.h +1 -0
  105. data/ext/carray_median_percentile.c +299 -100
  106. data/ext/carray_memory_view.c +872 -758
  107. data/ext/carray_operator.c +1171 -364
  108. data/ext/carray_order.c +505 -133
  109. data/ext/carray_partition.c +112 -65
  110. data/ext/carray_random.c +327 -239
  111. data/ext/carray_round.c +273 -0
  112. data/ext/carray_scatter.c +346 -78
  113. data/ext/carray_segment.c +257 -0
  114. data/ext/carray_slab.c +249 -86
  115. data/ext/carray_sort.c +224 -227
  116. data/ext/carray_sort_kernel.c +8 -1
  117. data/ext/carray_struct.c +118 -39
  118. data/ext/carray_test.c +193 -69
  119. data/ext/carray_time_text.c +1182 -0
  120. data/ext/carray_undef.c +4 -0
  121. data/ext/carray_utils.c +579 -113
  122. data/ext/extconf.rb +27 -2
  123. data/ext/mk_call_cfunc.rb +19 -4
  124. data/ext/mk_pow5_table.rb +35 -0
  125. data/ext/mkkernel.rb +1202 -1014
  126. data/ext/ruby_carray.c +17 -5
  127. data/ext/version.h +4 -4
  128. data/ext/version.rb +4 -2
  129. data/lib/carray/attribute.rb +160 -17
  130. data/lib/carray/autoload_carray.rb +23 -27
  131. data/lib/carray/autoload_method_extension.rb +20 -5
  132. data/lib/carray/axis_group.rb +360 -63
  133. data/lib/carray/basics.rb +162 -28
  134. data/lib/carray/bincount_nd.rb +18 -2
  135. data/lib/carray/block_iterator.rb +231 -58
  136. data/lib/carray/boolean_reduce.rb +34 -42
  137. data/lib/carray/categorical.rb +242 -52
  138. data/lib/carray/categorical_iterator.rb +437 -827
  139. data/lib/carray/complex.rb +41 -34
  140. data/lib/carray/conditional.rb +150 -14
  141. data/lib/carray/const_string.rb +59 -4
  142. data/lib/carray/construct.rb +15 -11
  143. data/lib/carray/data_type_extension.rb +1 -0
  144. data/lib/carray/distance_limit.rb +77 -0
  145. data/lib/carray/fixlen_string.rb +7 -1
  146. data/lib/carray/frame/column_verbs.rb +70 -0
  147. data/lib/carray/frame/concat.rb +157 -6
  148. data/lib/carray/frame/convert.rb +82 -7
  149. data/lib/carray/frame/csv_parser.rb +688 -87
  150. data/lib/carray/frame/describe.rb +80 -0
  151. data/lib/carray/frame/frame.rb +369 -45
  152. data/lib/carray/frame/group.rb +631 -90
  153. data/lib/carray/frame/io.rb +506 -79
  154. data/lib/carray/frame/join.rb +411 -62
  155. data/lib/carray/frame/records.rb +116 -19
  156. data/lib/carray/frame/reshape.rb +455 -0
  157. data/lib/carray/frame/sort.rb +30 -18
  158. data/lib/carray/frame/verbs.rb +868 -61
  159. data/lib/carray/frame/windows.rb +117 -0
  160. data/lib/carray/frame.rb +4 -0
  161. data/lib/carray/fuse_source.rb +118 -2
  162. data/lib/carray/fusion.rb +257 -42
  163. data/lib/carray/histogram.rb +6 -1
  164. data/lib/carray/inspect.rb +54 -13
  165. data/lib/carray/iterator.rb +99 -9
  166. data/lib/carray/key_windows.rb +620 -0
  167. data/lib/carray/lazy.rb +216 -193
  168. data/lib/carray/mask_gap_fill.rb +112 -92
  169. data/lib/carray/math.rb +32 -19
  170. data/lib/carray/meld_reduce.rb +52 -21
  171. data/lib/carray/methods/align_addr.rb +8 -4
  172. data/lib/carray/methods/bin.rb +26 -18
  173. data/lib/carray/methods/bincount.rb +18 -2
  174. data/lib/carray/methods/bit_string.rb +15 -4
  175. data/lib/carray/methods/choose.rb +16 -3
  176. data/lib/carray/methods/composition.rb +18 -10
  177. data/lib/carray/methods/factorize.rb +2 -2
  178. data/lib/carray/methods/gather_nd.rb +25 -5
  179. data/lib/carray/methods/insert_block.rb +9 -14
  180. data/lib/carray/methods/is_between.rb +59 -0
  181. data/lib/carray/methods/is_in.rb +140 -10
  182. data/lib/carray/methods/join.rb +2 -7
  183. data/lib/carray/methods/lag.rb +49 -0
  184. data/lib/carray/methods/locate_addr.rb +182 -45
  185. data/lib/carray/methods/mode.rb +18 -1
  186. data/lib/carray/methods/nunique.rb +11 -6
  187. data/lib/carray/methods/repeat.rb +1 -20
  188. data/lib/carray/methods/resize.rb +21 -23
  189. data/lib/carray/methods/snap.rb +34 -12
  190. data/lib/carray/methods/unique.rb +11 -2
  191. data/lib/carray/methods/value_counts.rb +4 -11
  192. data/lib/carray/record_order.rb +208 -0
  193. data/lib/carray/rng.rb +3 -0
  194. data/lib/carray/runtime.rb +65 -17
  195. data/lib/carray/segment_iterator.rb +1221 -0
  196. data/lib/carray/serialize.rb +214 -30
  197. data/lib/carray/slab_iterator.rb +101 -27
  198. data/lib/carray/stack.rb +10 -4
  199. data/lib/carray/string.rb +105 -5
  200. data/lib/carray/string_operation_extension.rb +12 -4
  201. data/lib/carray/struct.rb +137 -58
  202. data/lib/carray/struct_builder.rb +124 -17
  203. data/lib/carray/time.rb +1222 -186
  204. data/lib/carray/window_iterator.rb +242 -81
  205. data/lib/carray.rb +9 -2
  206. data/yard-stubs/ca_obj_array.rb +4 -4
  207. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  208. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  209. data/yard-stubs/ca_obj_fake.rb +1 -1
  210. data/yard-stubs/ca_obj_field.rb +5 -1
  211. data/yard-stubs/ca_obj_grid.rb +21 -10
  212. data/yard-stubs/ca_obj_refer.rb +5 -4
  213. data/yard-stubs/ca_obj_roll.rb +2 -3
  214. data/yard-stubs/ca_obj_shift.rb +5 -1
  215. data/yard-stubs/ca_obj_stride.rb +10 -7
  216. data/yard-stubs/ca_obj_tile.rb +1 -1
  217. data/yard-stubs/ca_obj_transpose.rb +1 -1
  218. data/yard-stubs/ca_obj_window.rb +9 -5
  219. data/yard-stubs/carray_access.rb +11 -5
  220. data/yard-stubs/carray_attribute.rb +0 -5
  221. data/yard-stubs/carray_broadcast.rb +1 -1
  222. data/yard-stubs/carray_cast.rb +88 -43
  223. data/yard-stubs/carray_conversion.rb +15 -2
  224. data/yard-stubs/carray_copy.rb +12 -5
  225. data/yard-stubs/carray_core.rb +3 -3
  226. data/yard-stubs/carray_count.rb +9 -0
  227. data/yard-stubs/carray_element.rb +10 -4
  228. data/yard-stubs/carray_generate.rb +21 -1
  229. data/yard-stubs/carray_lazy.rb +5 -1
  230. data/yard-stubs/carray_loop.rb +4 -1
  231. data/yard-stubs/carray_mask.rb +60 -12
  232. data/yard-stubs/carray_math.rb +52 -14
  233. data/yard-stubs/carray_median_percentile.rb +15 -8
  234. data/yard-stubs/carray_memory_view.rb +40 -23
  235. data/yard-stubs/carray_order.rb +61 -29
  236. data/yard-stubs/carray_random.rb +13 -4
  237. data/yard-stubs/carray_scatter.rb +21 -9
  238. data/yard-stubs/carray_segment.rb +72 -0
  239. data/yard-stubs/carray_slab.rb +10 -2
  240. data/yard-stubs/carray_sort.rb +49 -8
  241. data/yard-stubs/carray_test.rb +15 -3
  242. data/yard-stubs/carray_utils.rb +0 -26
  243. metadata +19 -5
  244. data/ext/ca_for_each_element.h +0 -239
  245. data/ext/carray_kernels_map.c +0 -3467
  246. data/lib/carray/arrow_tensor.rb +0 -401
  247. data/lib/carray/table.rb +0 -108
@@ -1,61 +1,205 @@
1
1
  /* ---------------------------------------------------------------------------
2
2
 
3
- T1 kernel_iterator MVP — Phase 1 step 1 + step 2 + step 3 implementation.
4
-
5
- Scope after step 3 (PROPOSAL_T1_KERNEL_ITERATOR.md §10.3):
6
- - ca_iter_state struct + init / next_slab / next_slab_strided /
7
- finish state machine
8
- - CA_SLAB_WHOLE policy only (step 5+ adds AXES / FREE)
9
- - level == 1 (L1, contig kernel):
10
- - entity / CAStride contig: alias path (single slab, alias_ptr
11
- = parent->ptr, stride implicit = bytes)
12
- - CAStride family non-contig: scratch path (ca_copy_data
13
- compose-fold gather into a malloc'd buffer)
14
- - other sources: CA_ITER_ERR_NOT_CHEAP
15
- - level == 2 (L2, strided kernel):
16
- - entity / CAStride contig: alias path (single slab,
17
- stride_bytes = bytes)
18
- - CAStride family non-contig: alias_strided path — no scratch,
19
- per-outer-prefix yield with native inner stride_bytes
20
- - other sources: CA_ITER_ERR_NOT_CHEAP
21
- - READ-only (flags == 0). WRITE = step 4, NO_MASK = step 7.
3
+ ca_kernel_iterator.c -- the kernel iterator engine
4
+
5
+ A C kernel asks for the cells of an array, one slab at a time, and gets
6
+ them whatever the array is: an entity, a view of a view, a selection, a
7
+ lazy expression. This file decides, for each source, how to deliver
8
+ them (borrow its memory, gather a copy, walk a stack's parents) and, on
9
+ a write, how the cells the kernel changed get back to the array.
10
+
11
+ Where to read first:
12
+ - writing a kernel: guides/devel/11_kernel_iterator.md and
13
+ docs/authoring/HOW_TO_WRITE_KERNEL.md; the surface itself is
14
+ ca_kernel_iterator.h
15
+ - how the engine works (its routes, the promises every route keeps,
16
+ and which part keeps each): docs/internal/KernelIteratorEngine.md
17
+
18
+ Map of this file:
19
+ - scratch buffers (ki_data_alloc / ki_data_free) and helpers
20
+ - classification: ca_iter_classify_source, ca_iter_route_source
21
+ - the promise parts every route calls: ki_take_mask_copy,
22
+ ki_gather_view, ki_scratch_push_changed, ki_attach_stack_parents,
23
+ ki_fiber_contig_data / _mask
24
+ - opening a walk: init_l1, and for L2 ki_choose_l2_route (decides)
25
+ then ki_open_l2 (builds the state, with ki_lay_out_slabs)
26
+ - handing out slabs: next_slab, next_slab_axes
27
+ - writing back: sync_slab
28
+ - closing: finish, close, and the raise-safe entry points
29
+ (ca_iter_protect, ca_iter_ensure)
30
+ - development-build test surface (CARRAY_DEV_BUILD), at the end
31
+
32
+ A route added for speed is a shortcut, tagged next to its code with
33
+ "Shortcut NAME" and a fixed set of fields; each one is switched off by
34
+ ca_fast_paths_enabled().
22
35
 
23
36
  ---------------------------------------------------------------------------- */
24
37
 
25
38
  #include "carray.h"
26
39
  #include "carray_internal.h" /* per-obj_type view constructors */
27
40
  #include "ca_kernel_iterator.h"
28
- #include "ca_monop_dispatch.h" /* P.6.2.d: ca_monop_view_is_single_cast for F.6.2 gate */
29
- #include "ca_obj_face.h" /* PROPOSAL_CAFACE_PHASE_2 F.2.6 — ca_strip_face for SRC_* entry */
41
+ #include "ca_monop_dispatch.h" /* ca_monop_view_is_single_cast */
42
+ #include "ca_obj_face.h" /* ca_strip_face, to walk what a Face wraps */
30
43
 
31
44
  #include <assert.h>
32
45
  #include <string.h>
46
+ #include <ruby/ractor.h>
47
+
48
+ /* Private flags bit, kept in ca_iter_state.flags like
49
+ CA_ITER_FLAG_WALK_MAY_RAISE below: src is a copy the walk owns (see
50
+ ca_iter_reopen_on_copy), freed when the walk finishes. */
51
+ #define CA_ITER_FLAG_OWNS_SRC 0x40000000u
52
+
53
+ /* Data scratch. A scratch of object cells holds VALUEs no Ruby object
54
+ owns while the kernel calls Ruby for each cell, so it is filled with
55
+ Qnil and held for the GC until finish releases it.
56
+
57
+ One large scratch is kept when a walk frees it and handed to the next
58
+ walk that fits in it: a fresh allocation of that size costs the page
59
+ faults of touching it, which for a reduction is several times the
60
+ reduction itself. The kept buffer is owned by nobody, and a walk that
61
+ takes it owns it alone until it frees it again. Each Ractor keeps its
62
+ own (ractor-local storage), since walks in different Ractors run at
63
+ the same time.
64
+
65
+ Shortcut KEEP_SCRATCH (name tag; docs/internal/KernelIteratorEngine.md
66
+ section 3.4 has the table of all shortcuts).
67
+ Layer data: where a scratch comes from. The walk is unchanged.
68
+ Speeds up walks with a scratch of 1 MiB or more, run one after
69
+ another.
70
+ Why no difference measured on macOS (a per-axis sum of a
71
+ 2000x2000 fake view); the page faults it
72
+ avoids are expected to cost more on Linux (unmeasured).
73
+ Enters ki_data_alloc: bytes >= KI_KEEP_MIN and a kept buffer
74
+ that fits; ki_data_free: KI_KEEP_MIN <= bytes <=
75
+ KI_KEEP_MAX.
76
+ Promises 4 (a kept buffer has no owner, so no exit can leave it
77
+ held); 7 (object cells are cleared and held either way).
78
+ Switch ca_fast_paths_enabled(): off, every scratch is fresh.
79
+ Checked by test_iter_ractor_scratch.rb; the buffer count in
80
+ test_iter_delivery_detectors.rb. */
81
+
82
+ #define KI_KEEP_MIN ((ca_size_t) 1 << 20)
83
+ #define KI_KEEP_MAX ((ca_size_t) 64 << 20)
84
+
85
+ typedef struct {
86
+ void *p;
87
+ ca_size_t bytes;
88
+ } ki_kept_t;
89
+
90
+ static rb_ractor_local_key_t ki_kept_key;
91
+
92
+ static void
93
+ ki_kept_free (void *ptr)
94
+ {
95
+ ki_kept_t *k = (ki_kept_t *) ptr;
96
+ if ( k ) {
97
+ if ( k->p ) ruby_xfree(k->p);
98
+ ruby_xfree(k);
99
+ }
100
+ }
101
+
102
+ static const struct rb_ractor_local_storage_type ki_kept_type = {
103
+ NULL, ki_kept_free
104
+ };
105
+
106
+ static ki_kept_t *
107
+ ki_kept_slot (void)
108
+ {
109
+ ki_kept_t *k = (ki_kept_t *) rb_ractor_local_storage_ptr(ki_kept_key);
110
+ if ( k == NULL ) {
111
+ k = ALLOC(ki_kept_t);
112
+ k->p = NULL;
113
+ k->bytes = 0;
114
+ rb_ractor_local_storage_ptr_set(ki_kept_key, k);
115
+ }
116
+ return k;
117
+ }
118
+
119
+ /* Development build: the number of buffers ki_data_alloc has handed out
120
+ that ki_data_free has not taken back (CArray.__iter_buffers__). A walk
121
+ that ends leaves the count where it found it; the spec suite compares it
122
+ around each walk. */
123
+ #ifdef CARRAY_DEV_BUILD
124
+ static long ki_buffers_out = 0;
125
+ /* atomic: walks run in several Ractors at once */
126
+ #define KI_BUFFERS_OUT(d) ( (void) __atomic_add_fetch(&ki_buffers_out, (d), __ATOMIC_RELAXED) )
127
+ #else
128
+ #define KI_BUFFERS_OUT(d) ((void) 0)
129
+ #endif
130
+
131
+ static void *
132
+ ki_data_alloc (int8_t data_type, ca_size_t bytes)
133
+ {
134
+ void *p = NULL;
135
+ KI_BUFFERS_OUT(1);
136
+ if ( bytes >= KI_KEEP_MIN && ca_fast_paths_enabled() ) {
137
+ ki_kept_t *k = ki_kept_slot();
138
+ if ( k->p && bytes <= k->bytes ) {
139
+ p = k->p;
140
+ k->p = NULL;
141
+ k->bytes = 0;
142
+ }
143
+ }
144
+ if ( p == NULL ) {
145
+ p = xmalloc(bytes > 0 ? bytes : 1);
146
+ }
147
+ if ( data_type == CA_OBJECT ) {
148
+ ca_size_t n = bytes / (ca_size_t) sizeof(VALUE);
149
+ ca_object_cells_clear(p, n);
150
+ ca_gc_hold_buffer(p, n);
151
+ }
152
+ return p;
153
+ }
154
+
155
+ /* `bytes` is the size p was allocated with (0 when unknown: never kept). */
156
+ static void
157
+ ki_data_free (void *p, ca_size_t bytes)
158
+ {
159
+ KI_BUFFERS_OUT(-1);
160
+ ca_gc_release_buffer(p);
161
+ if ( bytes >= KI_KEEP_MIN && bytes <= KI_KEEP_MAX && ca_fast_paths_enabled() ) {
162
+ ki_kept_t *k = ki_kept_slot();
163
+ if ( bytes > k->bytes ) {
164
+ void *old = k->p;
165
+ k->p = p;
166
+ k->bytes = bytes;
167
+ if ( old ) {
168
+ xfree(old);
169
+ }
170
+ return;
171
+ }
172
+ }
173
+ xfree(p);
174
+ }
33
175
 
34
176
  /* Defined in carray_core.c. (ca_is_readonly / ca_has_mask /
35
177
  ca_sync_data are already declared in carray.h.) */
36
178
  extern int ca_attach_is_alias (void *ap);
37
179
  extern int ca_root_lends_no_memory (void *ap);
38
180
 
39
- /* CAStride family attach-fn marker (step 1-4). */
181
+ /* CAStride family attach-fn marker. */
40
182
  extern ca_operation_function_t ca_stride_func;
41
183
 
42
- /* Descriptor framework view ops (step 5+). Each view's describe_axes
43
- emits the per-axis descriptor + parent dim snapshot consumed by
44
- the P3 ca_axis_dispatch_* substrate. CAShift uses ca_window_func
45
- (Phase G typedef pattern, ca_shift_func is a copy with only free /
46
- clone / create_mask overridden) so it routes via attach pointer
47
- match against ca_window_func.attach. */
184
+ /* Descriptor framework view ops. Each view's describe_axes emits the
185
+ per-axis descriptor + parent dim snapshot consumed by the
186
+ ca_axis_dispatch_* engine. CAShift shares CAWindow's operation table
187
+ (ca_shift_func is a copy with only free / clone / create_mask
188
+ overridden), so it is recognised by its attach pointer matching
189
+ ca_window_func.attach. */
48
190
  extern ca_operation_function_t ca_select_axis_func;
49
191
  extern ca_operation_function_t ca_grid_func;
50
192
  extern ca_operation_function_t ca_select_func;
51
193
  extern ca_operation_function_t ca_window_func;
52
- /* ca_mapping_func retired in R.3 (PROPOSAL_CAMAPPING_REMOVAL). */
53
194
 
54
- /* SRC_ATTACH 5 view (step 9). Each defines its own attach that
55
- materialises src->ptr via view-specific transform (cast / swap /
56
- bit unpack / reduction). kernel_iterator treats them uniformly:
57
- ca_attach(src) → kernel sees src->ptr as contig → on WRITE,
58
- ca_sync(src) lets the view's sync_data scatter back. */
195
+ /* Operation tables the classifier compares a source's attach slot
196
+ against. Every view below is delivered as CA_ITER_SRC_ATTACH: the
197
+ engine gathers its cells into a scratch buffer of its own through the
198
+ view's transfer (ki_gather_view = ca_xfer_all GET) and, on a WRITE
199
+ walk, sends back the cells the kernel changed (ki_scratch_push_changed
200
+ = ca_xfer_addrs, or ca_xfer_all PUT when every cell changed). The
201
+ view's attach / sync slots are not called. Cast / byte swap / bit
202
+ unpack / reduction views: */
59
203
  extern ca_operation_function_t ca_fake_func;
60
204
  extern ca_operation_function_t ca_byte_swap_func;
61
205
  extern ca_operation_function_t ca_bitfield_func;
@@ -63,46 +207,29 @@ extern ca_operation_function_t ca_bitarray_func;
63
207
  extern ca_operation_function_t ca_reduce_func;
64
208
  extern ca_operation_function_t ca_object_func;
65
209
 
66
- /* F-2 follow-up (2026-05-26): connect CATile / CARoll to kernel_iterator
67
- via the SRC_ATTACH pattern. Per-cell modulo wrap does not fit the
68
- descriptor framework's kind enum {STRIDE, INDEX, SHIFT}, so the
69
- innermost-STRIDE L2 alias path is not used; but the view-specific
70
- func_attach (embed-region gather) materialises src->ptr and func_sync
71
- scatters back — structurally identical to SRC_ATTACH. Fills the gap
72
- in the "deliver" principle (= these were previously rejected via
73
- SRC_NONE). */
210
+ /* CATile / CARoll: the per-cell modulo wrap has no STRIDE / INDEX /
211
+ SHIFT descriptor, so they are delivered as SRC_ATTACH (above). */
74
212
  extern ca_operation_function_t ca_tile_func;
75
213
  extern ca_operation_function_t ca_roll_func;
76
214
 
77
- /* PROPOSAL_CASTACK.md Phase 3 (2026-06-18): CAStack via SRC_ATTACH.
78
- func_attach materialises src->ptr via per-parent ca_xfer_all GET
79
- (= K * parent.bytes alloc, caller responsibility per MEMO §3.4);
80
- func_sync scatters back via xfer_all PUT. kernel sees a flat
81
- contig slab over the stacked output of shape (K, *parent_shape). */
215
+ /* CAStack / CAMeld: SRC_ATTACH (above). A stack can also be walked
216
+ through its parents directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K,
217
+ see init_l2). */
82
218
  extern ca_operation_function_t ca_stack_func;
83
219
  extern ca_operation_function_t ca_meld_func;
84
220
 
85
- /* M.6 (PROPOSAL_CAREMAP_INTERNAL.md §5.2): CARemap is internal-only
86
- (no Ruby class constant) but participates in kernel_iterator as an
87
- SRC_ATTACH source. Per-element gather has no STRIDE structure to
88
- preserve, so SRC_ATTACH (scratch materialise via ca_remap_func_attach)
89
- is the natural acceptance path. func_attach allocates a fresh
90
- buffer and runs xfer_all(GET); kernel sees a flat contig slab.
91
- On WRITE, ca_sync routes through xfer_all(PUT). */
221
+ /* CARemap (internal, no Ruby class constant): a per-element gather with
222
+ no STRIDE structure, delivered as SRC_ATTACH (above). */
92
223
  extern ca_operation_function_t ca_remap_func;
93
224
 
94
- /* PROPOSAL_LAZY_ELEMENTWISE_VIEW Phase 4.5 P.4.5.1 (2026-06-07): lazy
95
- element-wise view family. CAMonOp (Phase 1) / CABinOp (Phase 2) /
96
- CABinCmp + CAMonCmp (Phase 4) all expose a func_attach that pulls
97
- the lazy tree's materialise into a fresh contig buffer; kernel sees
98
- a flat slab. Same SRC_ATTACH structural pattern as CAFake et al,
99
- so a 4-line addition to the classify_source list opens all 22
100
- mkkernel-generated reduction ops (sum / count / mean / variance /
101
- argmin / ...) to lazy operands — `(a.lazy + b).sum` now works. */
225
+ /* Lazy element-wise views (CAMonOp / CABinOp / CABinCmp / CAMonCmp /
226
+ CATriOp): SRC_ATTACH (above); gathering evaluates the expression into
227
+ the scratch, so every generated reduction accepts a lazy operand. */
102
228
  extern ca_operation_function_t ca_monop_func;
103
229
  extern ca_operation_function_t ca_binop_func;
104
230
  extern ca_operation_function_t ca_bincmp_func;
105
231
  extern ca_operation_function_t ca_moncmp_func;
232
+ extern ca_operation_function_t ca_triop_func;
106
233
  extern CArray *ca_remap_new (CArray *ref, CArray *idx);
107
234
 
108
235
  /* ca_reduce_new is declared in carray.h; rb_cCAReduce defined in
@@ -154,16 +281,27 @@ ca_iter_register_source_kind (int obj_type, uint8_t kind)
154
281
  ca_iter_registered_source_kind[obj_type] = kind;
155
282
  }
156
283
 
157
- /* Classify the source by routing kind (proposal §1 strategy table).
158
- Returns CA_ITER_SRC_NONE for sources not yet supported. */
284
+ /* Classify the source by how its cells are delivered. Returns
285
+ CA_ITER_SRC_NONE only for a NULL source or one with no attach slot;
286
+ every other view is delivered, as SRC_ATTACH if nothing more specific
287
+ applies. */
159
288
  static uint8_t
160
289
  ca_iter_classify_source (CArray *src)
161
290
  {
162
291
  if ( src == NULL ) return CA_ITER_SRC_NONE;
292
+
293
+ /* Development build, CARRAY_GENERAL_PATH=2: every source takes the
294
+ transfer walk (see ki_open_by_transfer), the reference the other
295
+ routes are checked against. */
296
+ if ( ca_iter_by_transfer_only() ) {
297
+ return ca_func[src->obj_type].attach != NULL ? CA_ITER_SRC_ATTACH
298
+ : CA_ITER_SRC_NONE;
299
+ }
300
+
163
301
  if ( ca_is_entity(src) ) return CA_ITER_SRC_CASTRIDE;
164
302
  if ( ca_iter_is_castride_family(src) ) return CA_ITER_SRC_CASTRIDE;
165
303
 
166
- /* Externally installed obj_types declare their routing (2026-08-07).
304
+ /* Externally installed obj_types declare their routing.
167
305
  Placed after the two structural cases and before the built-in list:
168
306
  entity and CAStride-family sources are recognised from the struct
169
307
  itself and are read directly, so a registration must not be able to
@@ -175,88 +313,73 @@ ca_iter_classify_source (CArray *src)
175
313
  if ( kind != CA_ITER_SRC_NONE ) return kind;
176
314
  }
177
315
 
178
- /* Descriptor framework views (step 5.1: CSA + CAGrid; 5.2: + CASelect /
179
- CAMapping / CAWindow / CAShift). CAShift uses ca_window_func
180
- for attach (Phase G typedef pattern) so it matches the CAWindow
181
- check below. */
316
+ /* Descriptor views: CASelectAxis / CAGrid / CASelect / CAWindow.
317
+ CAShift shares ca_window_func's attach, so it matches the CAWindow
318
+ check. */
182
319
  void *attach = ca_func[src->obj_type].attach;
183
320
  if ( attach == ca_select_axis_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
184
321
  if ( attach == ca_grid_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
185
322
  if ( attach == ca_select_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
186
323
  if ( attach == ca_window_func.attach ) return CA_ITER_SRC_DESCRIPTOR; /* + CAShift */
187
324
 
188
- /* Step 9: SRC_ATTACH 5 view. Each view-specific attach materialises
189
- src->ptr via per-element transform; kernel sees a flat contig slab. */
325
+ /* The views below are SRC_ATTACH: gathered into scratch through their
326
+ own transfer (ki_gather_view) and, on WRITE, sent back the changed
327
+ cells (ki_scratch_push_changed). Their attach / sync slots are not
328
+ called by the engine. */
190
329
  if ( attach == ca_fake_func.attach ) return CA_ITER_SRC_ATTACH;
191
330
  if ( attach == ca_byte_swap_func.attach ) return CA_ITER_SRC_ATTACH;
192
331
  if ( attach == ca_bitfield_func.attach ) return CA_ITER_SRC_ATTACH;
193
332
  if ( attach == ca_bitarray_func.attach ) return CA_ITER_SRC_ATTACH;
194
333
  if ( attach == ca_reduce_func.attach ) return CA_ITER_SRC_ATTACH;
195
- /* Step 11: CAObject — Ruby callback per-element bridge. Same
196
- SRC_ATTACH structural pattern (func_attach materialises via Ruby
197
- copy_data, func_sync scatters back via Ruby sync_data, CA_FLAG_
198
- READ_ONLY auto-rejects WRITE via ca_is_readonly). Bench gate
199
- n/a (Ruby callback overhead structurally dominant). */
334
+ /* CAObject: each transfer calls the Ruby callback. A read-only
335
+ CAObject is refused for WRITE by the ca_is_readonly check. */
200
336
  if ( attach == ca_object_func.attach ) return CA_ITER_SRC_ATTACH;
201
337
 
202
- /* F-2 follow-up: CATile / CARoll via SRC_ATTACH (embed-region gather
203
- materialises src->ptr; ca_sync scatters back per view-specific
204
- semantics — CATile tile decomposition, CARoll cyclic permutation). */
205
338
  if ( attach == ca_tile_func.attach ) return CA_ITER_SRC_ATTACH;
206
339
  if ( attach == ca_roll_func.attach ) return CA_ITER_SRC_ATTACH;
207
340
 
208
- /* Phase 3 (PROPOSAL_CASTACK.md): CAStack via SRC_ATTACH. func_attach
209
- materialises src->ptr via per-parent ca_xfer_all GET (= K * parent
210
- bytes alloc, caller responsibility per MEMO §3.4); func_sync
211
- scatters back via xfer_all PUT. kernel sees a flat contig slab
212
- over the stacked output. Routine ndim mismatch is irrelevant
213
- here (= attach delivers a self-owned buffer of shape (K, *parent
214
- shape), kernel reads/writes that buffer). */
341
+ /* CAStack: SRC_ATTACH here; init_l2 may instead walk its parents
342
+ directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K). */
215
343
  if ( attach == ca_stack_func.attach ) return CA_ITER_SRC_ATTACH;
216
344
 
217
- /* CAMeld — ragged concatenate along an existing axis. func_attach
218
- materialises via K per-parent xfer_all GET into a contig buffer at
219
- seg_offset[k] * tail_bytes offsets; func_sync scatters back. Same
220
- SRC_ATTACH structural pattern as CAStack; reduce hot paths bypass
221
- this via the per-parent decompose in lib/carray/meld_reduce.rb. */
345
+ /* CAMeld: reduce hot paths avoid the gather through the per-parent
346
+ decompose in lib/carray/meld_reduce.rb. */
222
347
  if ( attach == ca_meld_func.attach ) return CA_ITER_SRC_ATTACH;
223
348
 
224
- /* M.6: CARemap — per-element gather, internal-only. Same SRC_ATTACH
225
- structural pattern (func_attach materialises via xfer_all GET,
226
- func_sync via xfer_all PUT). */
227
349
  if ( attach == ca_remap_func.attach ) return CA_ITER_SRC_ATTACH;
228
350
 
229
- /* Phase 4.5 P.4.5.1: lazy view family — CAMonOp / CABinOp / CABinCmp /
230
- CAMonCmp. func_attach pulls the lazy chain materialise (arena-pooled
231
- scratches under the hood from Phase 3); kernel sees a flat contig
232
- slab. Read-only (= CA_FLAG_READ_ONLY) so WRITE auto-rejects at
233
- the ca_is_readonly check. Opens 22 mkkernel-generated reduction
234
- ops to lazy operands. */
351
+ /* Lazy views (CAMonOp / CABinOp / CABinCmp / CAMonCmp / CATriOp):
352
+ gathering evaluates the expression. They are read-only, so WRITE is
353
+ refused by the ca_is_readonly check. */
235
354
  if ( attach == ca_monop_func.attach ) return CA_ITER_SRC_ATTACH;
236
355
  if ( attach == ca_binop_func.attach ) return CA_ITER_SRC_ATTACH;
237
356
  if ( attach == ca_bincmp_func.attach ) return CA_ITER_SRC_ATTACH;
238
357
  if ( attach == ca_moncmp_func.attach ) return CA_ITER_SRC_ATTACH;
358
+ if ( attach == ca_triop_func.attach ) return CA_ITER_SRC_ATTACH;
359
+
360
+ /* Anything else -- a view class installed from outside that did not
361
+ register, or an in-tree class with no case above (such as the mask
362
+ views CAMaskOfOperands and CAObjectMask). It is classified as a
363
+ registration would declare it, SRC_ATTACH, and delivered by the
364
+ transfer walk through its xfer slots rather than refused. */
365
+ if ( attach != NULL ) return CA_ITER_SRC_ATTACH;
239
366
 
240
367
  return CA_ITER_SRC_NONE;
241
368
  }
242
369
 
243
- /* F-2 (PROPOSAL_F2_KERNEL_ITERATOR_ALIAS rev6): route a source by
244
- running classify_source and, for descriptor-routed views, also calling
245
- describe_axes to inspect the innermost axis kind. Returns the refined
246
- src_kind (= SRC_DESCRIPTOR_L2_ALIASABLE iff innermost axis is STRIDE,
247
- else SRC_DESCRIPTOR). out_descs / out_parent_dims / out_ndim are
248
- populated for descriptor sources so the caller (init_l1 / init_l2)
249
- does not have to re-call describe_axes. For non-descriptor sources
250
- (CASTRIDE / ATTACH / NONE) the out_* arguments are not touched and
251
- classify_source's verdict is returned as-is.
252
-
253
- Cost analysis: classify_source is O(1) pointer compares. For
254
- SRC_DESCRIPTOR candidates we add one describe_axes call (O(ndim) with
255
- ndim ≤ CA_RANK_MAX = 16) plus one innermost-axis kind compare.
256
- Per-walk overhead = 1 describe_axes call (descriptor sources only),
257
- which init was going to do anyway -- so the routing is net zero cost
258
- compared to the pre-rev6 path where classify_source + init both
259
- computed describe_axes redundantly. See prep doc rev2 §4.1.1. */
370
+ /* Routes a source by running classify_source and, for descriptor-routed views, also calling
371
+ describe_axes to inspect the innermost axis kind. A descriptor view
372
+ comes back as SRC_DESCRIPTOR_L2_ALIASABLE (innermost axis STRIDE) or
373
+ SRC_DESCRIPTOR, and then out_descs / out_parent_dims / out_ndim are
374
+ filled so the caller does not re-call describe_axes. A grid with holes
375
+ and a view whose parent lends no memory come back as SRC_ATTACH, and
376
+ the out_* arguments are not touched; nor are they for any other kind,
377
+ whose classify_source verdict is returned as-is.
378
+
379
+ Cost: classify_source is O(1) pointer compares; a descriptor view
380
+ adds one describe_axes call (O(ndim)), which init would make anyway,
381
+ and init reads the descriptors from here instead of describing the
382
+ axes again. */
260
383
  static uint8_t ca_iter_classify_source (CArray *src);
261
384
  static void ca_iter_describe_axes (CArray *src, ca_axis_desc_t *,
262
385
  ca_size_t *, int8_t *);
@@ -270,6 +393,25 @@ ca_iter_route_source (CArray *src,
270
393
  uint8_t kind = ca_iter_classify_source(src);
271
394
  if ( kind != CA_ITER_SRC_DESCRIPTOR ) return kind;
272
395
 
396
+ /* A grid with holes (masked index cells) is not described by the engine
397
+ either: its holes are delivered by its own transfers. */
398
+ if ( ca_func[src->obj_type].attach == ca_grid_func.attach
399
+ && ca_grid_has_holes(src) ) {
400
+ return CA_ITER_SRC_ATTACH;
401
+ }
402
+
403
+ /* The descriptor walk reads the parent's memory, which such a parent
404
+ only has after a whole materialise -- however few cells the view
405
+ selects. Deliver through the view's own transfer instead: SRC_ATTACH
406
+ gathers the view into scratch with ca_xfer_all, and the view asks its
407
+ parent for the selected cells alone. */
408
+ {
409
+ CArray *parent = ((CAView *) src)->parent;
410
+ if ( ! parent->ptr && ca_parent_lends_no_memory(parent) ) {
411
+ return CA_ITER_SRC_ATTACH;
412
+ }
413
+ }
414
+
273
415
  /* Descriptor source: describe_axes + inspect innermost. */
274
416
  ca_iter_describe_axes(src, out_descs, out_parent_dims, out_ndim);
275
417
  if ( ca_axis_dispatch_is_innermost_stride(out_descs, *out_ndim) ) {
@@ -278,9 +420,8 @@ ca_iter_route_source (CArray *src,
278
420
  return CA_ITER_SRC_DESCRIPTOR;
279
421
  }
280
422
 
281
- /* Dispatch to the view's describe_axes. Routing keyed on the
282
- shared attach pointer (CAShift shares with CAWindow per Phase G
283
- typedef). */
423
+ /* Dispatch to the view's describe_axes, keyed on the attach pointer
424
+ (CAShift shares CAWindow's). */
284
425
  static void
285
426
  ca_iter_describe_axes (CArray *src,
286
427
  ca_axis_desc_t *out_descs,
@@ -315,7 +456,8 @@ ca_iter_describe_axes (CArray *src,
315
456
  *out_ndim = 0;
316
457
  }
317
458
 
318
- /* Public alias eligibility predicate (proposal §11.3 case (a)). */
459
+ /* Public alias eligibility predicate: whether a walk at `level` could
460
+ read the source's own memory in place. */
319
461
  int
320
462
  ca_iter_can_alias (void *ap, int level)
321
463
  {
@@ -331,9 +473,8 @@ ca_iter_can_alias (void *ap, int level)
331
473
  return ca_iter_is_castride_family(ca);
332
474
 
333
475
  default:
334
- /* L3 (and any future level) — not implemented in Phase 1; L1
335
- fallback so the predicate stays well-defined for callers that
336
- probe ahead. */
476
+ /* L3 (and any future level) is not implemented; answer as L1 so
477
+ the predicate stays well-defined for callers that probe ahead. */
337
478
  return ca_attach_is_alias(ca);
338
479
  }
339
480
  }
@@ -353,24 +494,24 @@ ca_iter_build_rowmajor_strides (ca_size_t *strides,
353
494
  }
354
495
  }
355
496
 
356
- /* ---- PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 dispatch predicate -------- */
497
+ /* ---- PER_FIBER_FUSED: which views take it ----------------------------- */
357
498
 
358
- /* Hybrid (rev2 Q2): coarse src_kind branch + view-specific func pointer
359
- probe + fiber-axis effective stride check (Q3).
360
- F.6.1 substrate: returns 0 (= disabled) for all sources. F.6.2+
361
- (CAFake / CAByteSwap / CAShift / CAWindow / etc.) progressively enable
362
- specific source kinds with bench-driven justification.
499
+ /* A src_kind branch, then a check of the view's operation table, then of
500
+ the fiber axis's stride. Returns nonzero for
501
+ CAFake / CAByteSwap / a single-cast CAMonOp under SRC_ATTACH when the
502
+ fiber is contiguous in the parent, and for CAWindow / CAShift under
503
+ SRC_DESCRIPTOR; zero otherwise.
363
504
 
364
505
  The fiber_axis_stride argument is the effective byte stride of the
365
506
  fiber axis on the view (= what next_slab_axes would yield as
366
- slab_strides[0]). Q3-equivalence: fiber_axis_stride == src->bytes
367
- means the fiber is parent-memory contig, which is the precondition
368
- for X.1/X.4 per-region fused paths to deliver a 1-pass result.
369
-
370
- Called from init_l2 SRC_ATTACH branch right before the whole-view
371
- ca_xfer_all GET. When returning 1, caller skips xfer_all GET and
372
- sets alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED with fiber dispatch
373
- state populated. */
507
+ slab_strides[0]). fiber_axis_stride == src->bytes means the fiber is
508
+ a contiguous run of the parent's memory, which is what lets the view's
509
+ region transfer deliver it in one pass.
510
+
511
+ Called from two places in init_l2: the CA_SLAB_AXES descriptor branch
512
+ and the SRC_ATTACH branch, each before it would materialise. When it
513
+ returns nonzero the caller skips the materialise and sets alias_mode =
514
+ CA_ITER_ALIAS_PER_FIBER_FUSED with the fiber dispatch state filled. */
374
515
  static int
375
516
  ca_iter_should_per_fiber_fused (CArray *src,
376
517
  int src_kind,
@@ -379,27 +520,22 @@ ca_iter_should_per_fiber_fused (CArray *src,
379
520
  uint32_t flags)
380
521
  {
381
522
  (void) fiber_axis;
382
- (void) flags;
383
-
384
- /* F.6.2: CAFake / CAByteSwap (transform-fused, X.4 per-region).
385
- Enable when fiber-axis effective stride == view cell bytes
386
- (= Q3: fiber is parent-memory contig run, so ca_xfer_stride
387
- routes into ca_xfer_stride_transform_fused inner-contig fast
388
- path for 1-pass per-fiber delivery).
389
-
390
- F.5 bench: innermost-axis fiber 30-35% faster than whole-view
391
- materialise (CAFake) / 16-18% (CAByteSwap). Non-innermost (=
392
- fiber_axis_stride != bytes) was a loser, so the stride gate
393
- keeps current path for that case.
394
-
395
- Phase 6 P.6.2.d (Q13 α): single-cast CAMonOp (= post-migration
396
- successor of CAFake numeric path) is recognised via
397
- ca_monop_view_is_single_cast and routed through the same F.6.2
398
- fast path. Chain CAMonOp (depth ≥ 2) is excluded — its
399
- materialise path (= arena-pooled scratches from Phase 3) is
400
- structurally different from the X.4 inner-contig fast path. */
523
+
524
+ /* CAFake / CAByteSwap: when the fiber is a contiguous run of the
525
+ parent's memory, ca_xfer_stride takes the view's converting fast
526
+ path (ca_xfer_stride_transform_fused) and delivers the fiber in one
527
+ pass. A fiber along another axis was measured slower this way, so
528
+ the stride gate keeps it on the gathered route. A single-cast
529
+ CAMonOp (ca_monop_view_is_single_cast) is the lazy form of the same
530
+ conversion and takes the same route; a chain of lazy operations
531
+ gathers through the lazy arena instead, which this does not fit. */
401
532
  void *attach = ca_func[src->obj_type].attach;
402
- if ( src_kind == CA_ITER_SRC_ATTACH ) {
533
+ /* A conversion view is read-only on this path. Its put-back sends the
534
+ whole fiber, and a cell the kernel did not write does not come back
535
+ unchanged through a conversion that loses information (an int64
536
+ above 2^53 seen as float64), so a write takes the transfer walk,
537
+ which sends back only the changed cells (promise 3). */
538
+ if ( src_kind == CA_ITER_SRC_ATTACH && !(flags & CA_KERNEL_WRITE) ) {
403
539
  if ( attach == ca_fake_func.attach
404
540
  || attach == ca_byte_swap_func.attach
405
541
  || ca_monop_view_is_single_cast(src) ) {
@@ -407,64 +543,26 @@ ca_iter_should_per_fiber_fused (CArray *src,
407
543
  }
408
544
  }
409
545
 
410
- /* F.6.3: CAShift / CAWindow with OOB-fused materialise path (X.1).
411
- Per-fiber ca_xfer_stride wins decisively when the view would
412
- otherwise fall into a per-slab gather / whole-view materialise
413
- path (= SRC_DESCRIPTOR, e.g. CAShift always, CAWindow with SHIFT
414
- axes). But when the view qualifies for the L2 alias fast path
415
- (SRC_DESCRIPTOR_L2_ALIASABLE, = interior-only CAWindow with F-1
416
- STRIDE promotion), the existing alias path is already optimal
417
- (parent.ptr + composed offsets, zero materialise). Per-fiber
418
- dispatch in that case adds ca_xfer_stride per-call overhead with
419
- no payoff -- F.6.3 bench measured 0.35-0.82x (regression). Gate
420
- on SRC_DESCRIPTOR only. */
546
+ /* CAShift / CAWindow: the view's region transfer fills out-of-bounds
547
+ cells itself, so a fiber can be fetched in one call instead of a
548
+ per-slab or whole-view gather (SRC_DESCRIPTOR). A view that can
549
+ lend the parent's memory (SRC_DESCRIPTOR_L2_ALIASABLE, an interior
550
+ CAWindow) is already read in place; fetching it fiber by fiber was
551
+ measured slower (0.35-0.82x), so only SRC_DESCRIPTOR takes it. */
421
552
  if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
422
- /* Both CAWindow and CAShift match ca_window_func.attach (Phase G
423
- typedef pattern: ca_shift_func = ca_window_func). */
553
+ /* CAWindow and CAShift share ca_window_func.attach. */
424
554
  if ( attach == ca_window_func.attach ) {
425
555
  return 1;
426
556
  }
427
557
  }
428
558
 
429
- /* F.6.4 audit (devel/bench_f6_4_audit.rb): CSA (CASelectAxis) /
430
- CAGrid / CASelect / CAMapping all kept on the whole-view
431
- materialise path. Per-fiber ca_xfer_stride is a LOSER for
432
- these views (1.25-4.7x slower across all axis positions
433
- measured) because the descriptor framework's
434
- ca_axis_dispatch_gather is already a single batched per-axis
435
- kind gather; splitting into N per-fiber calls loses the
436
- batched throughput. Structurally different from
437
- CAFake/CAByteSwap (transform-fused at byte level, X.4) and
438
- CAShift/CAWindow OOB (per-region fused with bound check, X.1):
439
- descriptor framework views do not have a per-region fused fast
440
- path designed for per-fiber entry.
441
-
442
- Note (rev4): bench split fiber-axis kind shows STRIDE fiber
443
- 1.25x slower vs INDEX fiber 4-5x slower. STRIDE-fiber case
444
- is close to parity and could conceivably reach win with finer
445
- gating (= bypass per-axis kind classification when the outer
446
- prefix is homogeneous STRIDE). Not pursued at current 1.25x
447
- deficit; recorded as a future possible refinement should an
448
- application motivate it. See proposal §3.1 SRC_DESCRIPTOR
449
- (CSA, CAGrid, ...) inline comment.
450
-
451
- F.6.6 audit (devel/bench_f6_6_bit_audit.rb): CABitarray /
452
- CABitfield kept on current path. Bench measured via
453
- bits.fake(:float64) (= production CAFake-wrap hot path) shows
454
- per-fiber 1.23-5.86x slower on the F.6.2-gate-off axes.
455
-
456
- The CAFake wrap overhead applies equally to both per-fiber and
457
- whole-view paths (same total cell-cast cost), so the 5x ratio
458
- is essentially the CABitarray whole-view-vs-per-fiber ratio
459
- itself. Phase A bench note "xfer_stride 0.98x parity" is a
460
- single-call comparison: N per-fiber calls cumulatively cost
461
- N x parity > 1 x xfer_all (+ N dispatch overhead + per-call
462
- bit alignment math). This reasoning generalises to direct
463
- CABitarray / CABitfield source (= when a non-f64 fiber smoke
464
- is added in a future phase) -- no separate audit needed for
465
- that case. Predicate stays unchanged.
466
-
467
- F.6.7 will audit CAReduce. */
559
+ /* Not taken by the other views, each measured slower fiber by fiber:
560
+ CASelectAxis / CAGrid / CASelect (1.25-4.7x), whose descriptor
561
+ engine already gathers the whole view in one batched pass and has no
562
+ per-region fast path to enter; CABitarray / CABitfield (1.2-5.9x,
563
+ through a fake over them), where N per-fiber calls each pay the bit
564
+ alignment that one whole-view transfer pays once. CAReduce is not
565
+ routed here either. */
468
566
  return 0;
469
567
  }
470
568
 
@@ -480,44 +578,38 @@ ca_iter_validate_inputs (ca_iter_state *st,
480
578
  uint32_t flags)
481
579
  {
482
580
  if ( st == NULL || src == NULL ) return CA_ITER_ERR_FLAGS;
483
- /* Phase A capstone: CA_SLAB_AXES accepted (init_l2 path only — init_l1
484
- does not implement CA_SLAB_AXES yet, see init_l1's policy gate). */
581
+ /* CA_SLAB_AXES is accepted here and refused by init_l1 itself; init_l2
582
+ refuses CA_SLAB_WHOLE (see each init's policy gate). */
485
583
  if ( policy != CA_SLAB_WHOLE && policy != CA_SLAB_AXES )
486
584
  return CA_ITER_ERR_POLICY;
487
585
 
488
- /* Step 6: accept CA_KERNEL_WRITE and CA_KERNEL_NO_MASK.
489
- CA_KERNEL_CHUNK_HINT is reserved for T2 — still rejected.
490
- NO_MASK enforcement (= reject masked source if NO_MASK is set)
491
- lands in step 7; step 6 accepts the flag but does not enforce.
492
- CA_KERNEL_FIBER_CONTIG (PROPOSAL_FIBER_DELIVERY F.1a) accepted —
493
- activates per-fiber contig delivery for naxes==1 in next_slab_axes. */
586
+ /* Accepted: CA_KERNEL_WRITE, CA_KERNEL_NO_MASK (enforced below) and
587
+ CA_KERNEL_FIBER_CONTIG (per-fiber contig delivery for naxes==1 in
588
+ next_slab_axes). CA_KERNEL_CHUNK_HINT is reserved and rejected. */
494
589
  const uint32_t accepted = CA_KERNEL_WRITE | CA_KERNEL_NO_MASK
495
590
  | CA_KERNEL_FIBER_CONTIG;
496
591
  if ( flags & ~accepted ) return CA_ITER_ERR_FLAGS;
497
592
 
498
- /* WRITE on a readonly view (CARepeat stride-0 / value_array /
499
- CAWrap readonly) — explicit reject, would otherwise SEGV on
500
- write. */
593
+ /* WRITE on a readonly view (CARepeat stride-0 / a lazy expression /
594
+ a read-only CAWrap / any view of a frozen or read-only array) --
595
+ explicit reject, would otherwise SEGV on write. */
501
596
  if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
502
597
  return CA_ITER_ERR_READONLY;
503
598
  }
504
599
 
505
- /* Step 6: mask is default-borne (bakeoff #5). The step-4
506
- CA_ITER_ERR_MASK gate is lifted (= masked sources are accepted).
507
- Step 7: enforce CA_KERNEL_NO_MASK as an explicit kernel-character
508
- declaration — if a kernel says "I cannot handle mask" and a
509
- masked source is handed in, reject with a dedicated error code
510
- so the caller can decide whether to peel via .value /
511
- .strip_mask(fill) or pick a mask-aware kernel. */
600
+ /* Masked sources are accepted; the mask travels with the slab, so
601
+ nothing returns CA_ITER_ERR_MASK. CA_KERNEL_NO_MASK is the kernel
602
+ declaring it cannot handle a mask: a masked source is then refused
603
+ with a dedicated code, so the caller can decide whether to peel via
604
+ .value / .strip_mask(fill) or pick a mask-aware kernel. */
512
605
  if ( (flags & CA_KERNEL_NO_MASK) && ca_has_mask(src) ) {
513
606
  return CA_ITER_ERR_MASK_NOT_ALLOWED;
514
607
  }
515
608
 
516
- /* Gate: classify source. sub-step 5.1 routing accepts entity +
517
- CAStride family (step 1-4) and CSA + CAGrid (step 5.1). Other
518
- descriptor views (CASelect / CAMapping / CAWindow / CAShift)
519
- and overlay views (CAFake / ...) still reject — handled in
520
- 5.2 and Phase 2 respectively. */
609
+ /* Gate: classify source. Entity, CAStride family, descriptor views
610
+ and SRC_ATTACH views are accepted; a source the classifier does not
611
+ recognise (an externally installed view that has not registered a
612
+ kind) is refused. */
521
613
  if ( ca_iter_classify_source(src) == CA_ITER_SRC_NONE ) {
522
614
  return CA_ITER_ERR_NOT_CHEAP;
523
615
  }
@@ -553,6 +645,29 @@ ca_iter_strip_storage_wrapper (CArray *src)
553
645
  return src;
554
646
  }
555
647
 
648
+ /* Raise unless a and b have the same shape. The INOUT and PAIR fiber
649
+ macros call it before opening either walk, so a refusal leaves nothing
650
+ to close. */
651
+ int
652
+ ca_iter_check_same_shape (struct _CArray *a, struct _CArray *b)
653
+ {
654
+ int8_t k;
655
+ int same = ( a->ndim == b->ndim );
656
+ for ( k = 0; same && k < a->ndim; k++ ) {
657
+ if ( a->dim[k] != b->dim[k] ) same = 0;
658
+ }
659
+ if ( ! same ) {
660
+ VALUE sa = rb_ary_new(), sb = rb_ary_new();
661
+ for ( k = 0; k < a->ndim; k++ ) rb_ary_push(sa, SIZE2NUM(a->dim[k]));
662
+ for ( k = 0; k < b->ndim; k++ ) rb_ary_push(sb, SIZE2NUM(b->dim[k]));
663
+ rb_raise(rb_eArgError,
664
+ "kernel iterator: the two arrays of a paired walk "
665
+ "differ in shape (%"PRIsVALUE" vs %"PRIsVALUE")",
666
+ rb_inspect(sa), rb_inspect(sb));
667
+ }
668
+ return 0;
669
+ }
670
+
556
671
  int
557
672
  ca_iter_check_init (int rc)
558
673
  {
@@ -562,7 +677,7 @@ ca_iter_check_init (int rc)
562
677
 
563
678
  switch ( rc ) {
564
679
  case CA_ITER_ERR_NOT_CHEAP:
565
- why = "the source cannot be read without materialising it";
680
+ why = "the iterator does not recognise this kind of source";
566
681
  break;
567
682
  case CA_ITER_ERR_POLICY:
568
683
  why = "the slab policy or the axes given do not apply to this source";
@@ -587,6 +702,170 @@ ca_iter_check_init (int rc)
587
702
  return rc; /* not reached */
588
703
  }
589
704
 
705
+ /* Attaches a CAStack's parents and keeps their buffers. stack_n_parents
706
+ counts the parents attached so far, so an attach that raises leaves
707
+ finish with exactly those to detach (promise 4). The mask of a stack
708
+ reads its parts' masks, some perhaps through stand-ins: those give way
709
+ first, as the stack's own slots would let them. */
710
+ static void
711
+ ki_attach_stack_parents (ca_iter_state *st, CAStack *stack)
712
+ {
713
+ if ( stack->mask_owner != NULL ) {
714
+ CAStack *o = (CAStack *) stack->mask_owner;
715
+ ca_multi_parent_mask_refresh((CArray *) stack, o->parents,
716
+ o->mask_standins, o->n_parents);
717
+ }
718
+ st->stack_n_parents = 0;
719
+ st->stack_parent_ptrs =
720
+ (char **) xmalloc(stack->n_parents * sizeof(char *));
721
+ for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
722
+ ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
723
+ st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
724
+ st->stack_n_parents = kk + 1;
725
+ }
726
+ }
727
+
728
+ /* Promise 2: the mask cursor a walk hands out is the walk's own copy of
729
+ the source's mask, taken when the walk opens -- one flag per cell, in
730
+ the view's row-major order -- never the live mask, so a write through
731
+ it reaches nothing. A source without a mask leaves alias_mask NULL.
732
+ finish frees the copy. */
733
+ static void
734
+ ki_take_mask_copy (ca_iter_state *st, CArray *src)
735
+ {
736
+ if ( ! ca_has_mask(src) ) return;
737
+ st->scratch_mask = (boolean8_t *) xmalloc(src->elements > 0 ? src->elements : 1);
738
+ if ( src->elements > 0 ) {
739
+ ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
740
+ }
741
+ st->alias_mask = st->scratch_mask;
742
+ }
743
+
744
+ /* Whole-view scratch. The view is gathered into the walk's scratch,
745
+ by the view's own transfer (by_copy = 0) or by ca_copy_data (by_copy =
746
+ 1, the CAStride compose-fold). A WRITE walk sends back only the cells
747
+ the kernel changed (ki_scratch_push_changed), so the scratch is twice
748
+ the view and its second half keeps what was gathered: a cell the
749
+ kernel left alone must not be sent back, for it may be another copy
750
+ of a cell the kernel did write (a tile, a stack of one array with
751
+ itself) or a cell a conversion view would not bring back unchanged. */
752
+ static void
753
+ ki_gather_view (ca_iter_state *st, CArray *src, uint32_t flags, int by_copy)
754
+ {
755
+ ca_size_t view_bytes = (ca_size_t) src->elements * src->bytes;
756
+ st->scratch_cap = (flags & CA_KERNEL_WRITE) ? 2 * view_bytes : view_bytes;
757
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
758
+ if ( src->elements > 0 ) {
759
+ if ( by_copy ) {
760
+ ca_copy_data(src, st->scratch_ptr);
761
+ }
762
+ else {
763
+ ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
764
+ }
765
+ if ( flags & CA_KERNEL_WRITE ) {
766
+ memcpy(st->scratch_ptr + view_bytes, st->scratch_ptr, view_bytes);
767
+ }
768
+ }
769
+ }
770
+
771
+ /* Send back the cells of the current slab (the whole view for a WHOLE
772
+ walk) that differ from what was gathered, and record them as sent.
773
+ The slab of a CA_SLAB_AXES walk starts at last_data_off in the
774
+ scratch, which is laid out row-major over the view, so a cell's
775
+ offset divided by the cell size is its address in the view. */
776
+ static void
777
+ ki_scratch_push_changed (ca_iter_state *st)
778
+ {
779
+ CArray *src = st->src;
780
+ ca_size_t bytes = src->bytes;
781
+ ca_size_t view_bytes = (ca_size_t) src->elements * bytes;
782
+ char *now = st->scratch_ptr;
783
+ char *was = st->scratch_ptr + view_bytes;
784
+ int whole = ( st->level == 1 ); /* L1 is the whole array */
785
+ ca_size_t n = whole ? src->elements : st->slab_elements;
786
+ ca_size_t idx[CA_RANK_MAX];
787
+ ca_size_t *addrs;
788
+ char *vals;
789
+ ca_size_t n_changed = 0;
790
+ ca_size_t i;
791
+ int8_t k;
792
+ volatile VALUE h_addrs = 0, h_vals = 0;
793
+
794
+ if ( n <= 0 ) return;
795
+
796
+ addrs = ALLOCV_N(ca_size_t, h_addrs, n);
797
+ vals = (char *) ALLOCV(h_vals, n * bytes);
798
+ for ( k = 0; k < st->slab_ndim; k++ ) idx[k] = 0;
799
+
800
+ for ( i = 0; i < n; i++ ) {
801
+ ca_size_t off;
802
+ if ( whole ) {
803
+ off = i * bytes;
804
+ }
805
+ else {
806
+ off = st->last_data_off;
807
+ for ( k = 0; k < st->slab_ndim; k++ ) off += idx[k] * st->slab_strides[k];
808
+ for ( k = st->slab_ndim - 1; k >= 0; k-- ) {
809
+ if ( ++idx[k] < st->slab_dims[k] ) break;
810
+ idx[k] = 0;
811
+ }
812
+ }
813
+ if ( memcmp(now + off, was + off, bytes) != 0 ) {
814
+ memcpy(was + off, now + off, bytes);
815
+ memcpy(vals + n_changed * bytes, now + off, bytes);
816
+ addrs[n_changed++] = off / bytes;
817
+ }
818
+ }
819
+
820
+ /* Shortcut PUSH_ALL_CHANGED.
821
+ Layer write-back.
822
+ Speeds up a whole-view write walk that changed every cell: one
823
+ ca_xfer_all PUT instead of a list of all the addresses.
824
+ Why about 5% on a whole write of a fake view of 1M cells.
825
+ Enters whole && n_changed == n.
826
+ Promises 3 (the same cells go back; the list would hold them all).
827
+ Switch ca_fast_paths_enabled(): off, the address list.
828
+ Checked by the write rows of the delivery matrix (L1 / L2 whole). */
829
+ if ( whole && n_changed == n && ca_fast_paths_enabled() ) {
830
+ ca_xfer_all(src, now, CA_XFER_PUT);
831
+ }
832
+ else if ( n_changed > 0 ) {
833
+ ca_xfer_addrs(src, n_changed, addrs, vals, CA_XFER_PUT);
834
+ }
835
+ ALLOCV_END(h_vals);
836
+ ALLOCV_END(h_addrs);
837
+ }
838
+
839
+ /* The transfer walk: the whole view gathered into scratch through the
840
+ view's own transfer, its mask likewise, and yielded as one contiguous
841
+ slab; on WRITE, sync_slab sends the changed cells back through the same
842
+ transfer (ki_scratch_push_changed).
843
+
844
+ Every promise the iterator makes is kept here by the view itself --
845
+ what a cell is, where its mask flag lives, which root cell a write
846
+ lands on -- so this walk is the reference the other routes are checked
847
+ against (CARRAY_GENERAL_PATH=2 sends every source through it). It is
848
+ also the route for a view the classifier knows nothing more about.
849
+
850
+ The caller has filled src / src_kind / level / policy / ndim / flags /
851
+ bytes / naxes. */
852
+ static void
853
+ ki_open_by_transfer (ca_iter_state *st, CArray *src, uint32_t flags)
854
+ {
855
+ ki_gather_view(st, src, flags, 0);
856
+ st->alias_mode = CA_ITER_ALIAS_NONE;
857
+ st->alias_ptr = st->scratch_ptr;
858
+ st->composed_strides[0] = src->bytes;
859
+ st->composed_base = 0;
860
+ st->slab_n = src->elements;
861
+ st->total_slabs = 1;
862
+ st->slabs_emitted = 0;
863
+ st->chunk_size = st->slab_n;
864
+ st->outer_idx = NULL;
865
+
866
+ ki_take_mask_copy(st, src);
867
+ }
868
+
590
869
  static int
591
870
  ca_iter_state_init_l1_unprotected (ca_iter_state *st,
592
871
  struct _CArray *src,
@@ -601,10 +880,11 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
601
880
  times rather than whatever its stack frame happened to contain. */
602
881
  if ( st == NULL ) return CA_ITER_ERR_FLAGS;
603
882
  memset(st, 0, sizeof(*st));
883
+ if ( src == NULL ) return CA_ITER_ERR_FLAGS;
604
884
 
605
- /* PROPOSAL_CAFACE_PHASE_2 F.2.6 + PROPOSAL_LAZY_MARKER_LIFT Phase 0:
606
- storage-identical wrapper strip at entry (= same rationale as init_l2
607
- below). Strip before validate_inputs.
885
+ /* A Face or a lazy marker wraps storage identical to its parent, so it
886
+ is stripped at entry (the same reason as in init_l2 below), before
887
+ validate_inputs.
608
888
 
609
889
  The WRITE rejection is taken against the wrapper, not what it wraps: a
610
890
  CALazyMarker is read-only while its parent is not, and stripping first
@@ -620,12 +900,32 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
620
900
  int rc = ca_iter_validate_inputs(st, src, policy, flags);
621
901
  if ( rc != CA_ITER_OK ) return rc;
622
902
 
623
- /* Phase A capstone: CA_SLAB_AXES is L2-only (kernels needing K-D slab
624
- yield use init_l2). init_l1 rejects so callers don't get a silent
625
- misdispatch. */
903
+ /* CA_SLAB_AXES is L2 only (a kernel that wants a K-D slab uses
904
+ init_l2); init_l1 refuses it rather than walking the whole array. */
626
905
  if ( policy == CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
627
906
 
628
- uint8_t src_kind = ca_iter_classify_source(src);
907
+ /* The same routing as init_l2: a descriptor view whose parent lends no
908
+ memory, or a grid with holes, takes the transfer walk. An L1 walk
909
+ yields the whole view as one slab, so a descriptor view the L2 walk
910
+ could borrow is gathered here like any other descriptor view. */
911
+ ca_axis_desc_t route_descs[CA_RANK_MAX];
912
+ ca_size_t route_pdims[CA_RANK_MAX];
913
+ int8_t route_ndim = 0;
914
+ uint8_t src_kind = ca_iter_route_source(src, route_descs, route_pdims,
915
+ &route_ndim);
916
+ if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
917
+ src_kind = CA_ITER_SRC_DESCRIPTOR;
918
+ }
919
+ /* A write over a descriptor view takes the transfer walk, as an L2
920
+ write over an INDEX or SHIFT slab axis does. Gathering with the
921
+ descriptor engine would leave nothing to send back but the whole
922
+ view, and a view that shows a root cell twice (a grid with a
923
+ repeated index) would then let the copy the kernel did not write
924
+ overwrite the one it did (promise 3: only the changed cells go
925
+ back). */
926
+ if ( src_kind == CA_ITER_SRC_DESCRIPTOR && (flags & CA_KERNEL_WRITE) ) {
927
+ src_kind = CA_ITER_SRC_ATTACH;
928
+ }
629
929
 
630
930
  st->src = src;
631
931
  st->src_kind = src_kind;
@@ -634,7 +934,6 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
634
934
  st->ndim = src->ndim;
635
935
  st->flags = flags;
636
936
  st->bytes = src->bytes;
637
- st->axes = axes;
638
937
  st->naxes = naxes;
639
938
 
640
939
  st->slab_n = src->elements;
@@ -643,45 +942,18 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
643
942
  st->chunk_size = st->slab_n;
644
943
 
645
944
  if ( src_kind == CA_ITER_SRC_ATTACH ) {
646
- /* === SRC_ATTACH path (step 9 + 2026-05-31 refactor): CAFake /
647
- CAByteSwap / CABitfield / CABitarray / CAReduce =================
648
- View's xfer_all delivers materialised data through the unified
649
- dispatch surface. Previously used `ca_attach(src) + src->ptr`
650
- (= view's attach slot) and `ca_sync(src)` for WRITE; now uses
651
- `ca_xfer_all(src, scratch, GET/PUT)` with iterator-owned scratch.
652
- Decouples kernel_iterator from per-view attach/sync lifecycle
653
- and inherits xfer reform improvements (transform-fused, etc.)
654
- automatically. */
655
- st->scratch_cap = (ca_size_t) src->elements * src->bytes;
656
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
657
- if ( src->elements > 0 ) {
658
- ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
659
- }
660
- st->alias_mode = CA_ITER_ALIAS_NONE; /* scratch-owned, no src.detach */
661
- st->alias_ptr = st->scratch_ptr;
662
- st->composed_strides[0] = src->bytes;
663
- st->composed_base = 0;
664
- st->outer_idx = NULL;
665
- /* Step 6: mask gather (xfer_all on mask -- ca_copy_data is a thin
666
- wrapper, this just removes the extra hop). */
667
- if ( ca_has_mask(src) ) {
668
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
669
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
670
- if ( src->elements > 0 ) {
671
- ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
672
- }
673
- st->alias_mask = st->scratch_mask;
674
- }
945
+ ki_open_by_transfer(st, src, flags);
675
946
  return CA_ITER_OK;
676
947
  }
677
948
 
678
949
  if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
679
- /* === descriptor path (step 5.1: CSA + CAGrid) ======================
680
- Reuse P3 substrate: describe_axes → prepare → layout. For now
681
- always materialise via ca_axis_dispatch_attach into scratch and
682
- yield one contig slab (alias_mode = NONE). The view-transparency
683
- principle (proposal §0): kernel sees a flat slab regardless of
684
- per-axis kind; INDEX / SHIFT mix is handled inside the engine. */
950
+ /* === descriptor path (CASelectAxis / CAGrid / CASelect / CAWindow /
951
+ CAShift) ==========================================================
952
+ describe_axes → prepare → layout, then gather the view into
953
+ scratch with ca_axis_dispatch_gather and yield one contig slab
954
+ (alias_mode = NONE). The kernel sees a flat slab whatever the
955
+ per-axis kinds are; an INDEX / SHIFT mix is handled inside the
956
+ engine. */
685
957
  ca_axis_desc_t raw_descs[CA_RANK_MAX];
686
958
  int8_t raw_ndim = 0;
687
959
  ca_iter_describe_axes(src, raw_descs, st->parent_axis_dims, &raw_ndim);
@@ -699,49 +971,34 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
699
971
  }
700
972
  st->total_elements = src->elements;
701
973
 
702
- /* Materialise via the engine. ca_axis_dispatch_attach handles
703
- parent attach + scratch alloc + gather (INDEX axes included);
704
- we own the resulting buffer. Descriptor views inherit from
705
- CAView so parent is at the CAVIEW prefix slot. */
974
+ /* Materialise: attach the parent here, allocate the scratch, and
975
+ gather into it with ca_axis_dispatch_gather (INDEX axes included);
976
+ the iterator owns the buffer. Descriptor views inherit from CAView
977
+ so parent is at the CAVIEW prefix slot. */
706
978
  CArray *parent = CAVIEW(src)->parent;
707
979
  ca_attach(parent); /* engine reads parent->ptr */
708
980
  st->root = parent; /* finish() detaches */
709
981
  st->scratch_cap = src->elements * src->bytes;
710
- /* Engine ndim = descriptor ndim from describe_axes (not view.ndim).
711
- Critical for CAMapping where view.ndim > 1 but the descriptor is
712
- 1-D INDEX gather.
713
- bound_fill = CAWindow/CAShift fill value (Tier 2.B SHIFT-axis
714
- OOB cell), NULL for non-window views (CSA / CAGrid / CASelect /
715
- CAMapping don't have OOB semantics — engine sees no SHIFT axes). */
982
+ /* bound_fill = the CAWindow / CAShift fill value for an out-of-bounds
983
+ SHIFT cell; NULL for the other descriptor views, which have no
984
+ SHIFT axes. */
716
985
  const void *bound_fill = NULL;
717
986
  if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
718
987
  bound_fill = ((CAWindow *) src)->fill;
719
988
  }
720
- st->scratch_ptr = ca_axis_dispatch_attach(parent,
721
- st->parent_axis_dims,
722
- raw_descs, raw_ndim,
723
- src->bytes,
724
- st->total_elements,
725
- bound_fill);
989
+ st->scratch_cap = st->total_elements * src->bytes;
990
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
991
+ ca_axis_dispatch_gather(parent, st->parent_axis_dims, raw_descs, raw_ndim,
992
+ src->bytes, st->total_elements, bound_fill,
993
+ st->scratch_ptr);
726
994
  st->alias_mode = CA_ITER_ALIAS_NONE;
727
995
  st->alias_ptr = st->scratch_ptr;
728
996
 
729
- /* Step 6: gather mask via ca_copy_data on src->mask (works for
730
- all descriptor views — mask propagates through their
731
- func_copy_data via the descriptor framework's own mask
732
- handling). */
733
- if ( ca_has_mask(src) ) {
734
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
735
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
736
- if ( src->elements > 0 ) {
737
- ca_copy_data(src->mask, (char *) st->scratch_mask);
738
- }
739
- st->alias_mask = st->scratch_mask;
740
- }
997
+ ki_take_mask_copy(st, src);
741
998
  return CA_ITER_OK;
742
999
  }
743
1000
 
744
- /* === CAStride path (step 1-4) ============================== */
1001
+ /* === CAStride path ========================================= */
745
1002
  if ( ca_iter_can_alias(src, 1) ) {
746
1003
  /* alias path: entity or CAStride contig */
747
1004
  st->alias_mode = CA_ITER_ALIAS_CONTIG;
@@ -752,1430 +1009,692 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
752
1009
  to ca_stride_func_copy_data which composes leaf strides up to
753
1010
  the root entity and gathers in one pass — no per-intermediate
754
1011
  view materialise. */
755
- ca_size_t cap = src->elements * src->bytes;
756
- st->scratch_cap = cap;
757
- st->scratch_ptr = xmalloc(cap > 0 ? cap : 1);
758
- if ( cap > 0 ) {
759
- ca_copy_data(src, st->scratch_ptr);
760
- }
1012
+ ki_gather_view(st, src, flags, 1);
761
1013
  st->alias_mode = CA_ITER_ALIAS_NONE;
762
1014
  st->alias_ptr = st->scratch_ptr;
763
1015
  }
764
1016
 
765
- /* Step 6: gather mask into scratch_mask if source carries one.
766
- We always gather into a contig boolean8_t buffer for uniformity
767
- across alias / scratch / descriptor paths — the kernel sees a
768
- simple `boolean8_t *` aligned with the value slab. Future
769
- optimisation: alias mask directly for CAStride contig. Mask is
770
- informational; kernel uses CA_FOR_EACH_UNMASKED macros to skip
771
- masked cells. */
772
- if ( ca_has_mask(src) ) {
773
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
774
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
775
- if ( src->elements > 0 ) {
776
- ca_copy_data(src->mask, (char *) st->scratch_mask);
777
- }
778
- st->alias_mask = st->scratch_mask;
779
- }
1017
+ ki_take_mask_copy(st, src);
780
1018
  return CA_ITER_OK;
781
1019
  }
782
1020
 
783
- static int
784
- ca_iter_state_init_l2_unprotected (ca_iter_state *st,
785
- struct _CArray *src,
786
- ca_slab_policy_t policy,
787
- int8_t *axes,
788
- int8_t naxes,
789
- uint32_t flags)
790
- {
791
- /* Leave the state defined before anything can return. Every error
792
- return below hands control back to a caller that may not look at the
793
- code, so what it is left holding has to be a state that iterates zero
794
- times rather than whatever its stack frame happened to contain. */
795
- if ( st == NULL ) return CA_ITER_ERR_FLAGS;
796
- memset(st, 0, sizeof(*st));
797
-
798
- /* PROPOSAL_CAFACE_PHASE_2 F.2.6 (= MEMO §3.5 kernel_iterator entry strip)
799
- + PROPOSAL_LAZY_MARKER_LIFT Phase 0.
1021
+ /* ---- L2 walks: choosing a route, then building the state for it --------
1022
+
1023
+ init_l2 checks the request, classifies the source and chooses one of the
1024
+ routes below (ki_choose_l2_route, which only decides). ki_open_l2 then
1025
+ builds the state for that route: the part only that route needs, the
1026
+ axis layout every route shares (ki_lay_out_slabs) and the mask. So the
1027
+ state is assembled in one place and init_l2 has one way out. */
1028
+
1029
+ typedef enum {
1030
+ KI_L2_FIBER_FUSED, /* PER_FIBER_FUSED: a CAWindow / CAShift fiber, or a
1031
+ read of a conversion view's innermost fiber */
1032
+ KI_L2_PER_SLAB, /* descriptor view, read, a slab axis INDEX or SHIFT:
1033
+ one slab gathered at a time (PER_SLAB / _HOIST) */
1034
+ KI_L2_DESC_WHOLE, /* descriptor view, read: the whole view gathered by
1035
+ the descriptor engine (outer SHIFT, SHIFT_WHOLE_VIEW) */
1036
+ KI_L2_DESC_STRIDED, /* descriptor view whose slab axes are all STRIDE:
1037
+ the parent's memory is borrowed */
1038
+ KI_L2_STACK, /* CAStack read along its stack axis */
1039
+ KI_L2_STACK_OUTER_K, /* CAStack read with the stack axis outside the slab */
1040
+ KI_L2_TRANSFER, /* the whole view gathered by its own transfer */
1041
+ KI_L2_ROOT_STRIDED, /* CAStride: the root walked through composed strides */
1042
+ KI_L2_VIEW_SCRATCH, /* CAStride over a root that lends no memory */
1043
+ KI_L2_CONTIG /* entity, or a contiguous CAStride */
1044
+ } ki_l2_route;
800
1045
 
801
- A Face only layers a semantic identifier and a CALazyMarker only layers
802
- "leaf of a lazy chain"; storage is identical to the parent either way, so
803
- strip at the kernel-compute entry and descend. Routing and alias
804
- decisions belong to the parent. Strip before validate_inputs —
805
- classify_source rejects both as knows-no.
806
-
807
- The Face identifier is re-wrapped onto the result by the caller's lift
808
- hook (= primary operators / reductions / etc.). The marker is not:
809
- a reduction over a lazy marker yields a plain entity.
1046
+ typedef struct {
1047
+ ki_l2_route route;
1048
+ uint8_t src_kind; /* what st->src_kind records */
1049
+ int8_t in_slab[CA_RANK_MAX]; /* 1 for each slab axis */
1050
+ ca_axis_desc_t descs[CA_RANK_MAX]; /* descriptor routes: the view's */
1051
+ ca_size_t pdims[CA_RANK_MAX]; /* axes and the parent's dims */
1052
+ int8_t ndesc;
1053
+ CArray *root; /* KI_L2_ROOT_STRIDED */
1054
+ ca_size_t composed[CA_RANK_MAX];
1055
+ ca_size_t base;
1056
+ } ki_l2_plan;
1057
+
1058
+ /* Per view axis: its extent, its byte stride when it is a slab axis, its
1059
+ byte stride when it is an outer axis, and its mask element stride. */
1060
+ typedef struct {
1061
+ ca_size_t dims[CA_RANK_MAX];
1062
+ ca_size_t slab_strides[CA_RANK_MAX];
1063
+ ca_size_t outer_strides[CA_RANK_MAX];
1064
+ ca_size_t mask_strides[CA_RANK_MAX];
1065
+ } ki_axis_layout;
810
1066
 
811
- WRITE is rejected against the wrapper, before the strip — a marker is
812
- read-only while its parent is not. validate_inputs re-checks the
813
- stripped source, which is harmless. */
814
- if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
815
- return CA_ITER_ERR_READONLY;
816
- }
817
- if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
818
- src = ca_iter_strip_storage_wrapper(src);
1067
+ /* Row-major byte and element strides over the view's own shape. */
1068
+ static void
1069
+ ki_view_row_major (CArray *src, ca_size_t *bytes, ca_size_t *elems)
1070
+ {
1071
+ ca_size_t b = src->bytes, e = 1;
1072
+ for ( int8_t k = src->ndim - 1; k >= 0; k-- ) {
1073
+ if ( bytes ) bytes[k] = b;
1074
+ if ( elems ) elems[k] = e;
1075
+ b *= src->dim[k];
1076
+ e *= src->dim[k];
819
1077
  }
1078
+ }
820
1079
 
821
- int rc = ca_iter_validate_inputs(st, src, policy, flags);
822
- if ( rc != CA_ITER_OK ) return rc;
823
-
824
- /* F-2 (rev6): route_source returns the refined src_kind and (for
825
- descriptor sources) populates raw_descs / raw_pdims / raw_ndim so we
826
- don't re-call describe_axes when SRC_DESCRIPTOR_L2_ALIASABLE upgrades
827
- into the alias branch. Non-descriptor sources leave the out_*
828
- buffers untouched. */
829
- ca_axis_desc_t raw_descs[CA_RANK_MAX];
830
- ca_size_t raw_pdims[CA_RANK_MAX];
831
- int8_t raw_ndim = 0;
832
- uint8_t src_kind = ca_iter_route_source(src, raw_descs, raw_pdims, &raw_ndim);
833
-
834
- /* Phase B capstone (T2): CA_SLAB_AXES + descriptor view (= SRC_DESCRIPTOR
835
- and SRC_DESCRIPTOR_L2_ALIASABLE), accept when slab axes are all-STRIDE
836
- kind and no outer SHIFT. Other combinations:
837
- - slab has INDEX/SHIFT kind → reject (Phase C T3 scope)
838
- - outer has SHIFT axis → reject (B.1.5 materialise downgrade scope)
839
- - SRC_ATTACH → reject (overlay views; no kind structure)
840
- SRC_CASTRIDE still goes through the Phase A branch below. */
841
- if ( policy == CA_SLAB_AXES ) {
842
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR
843
- || src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
844
- /* Validate axes early (range / duplicate). */
845
- if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
846
- return CA_ITER_ERR_POLICY;
847
- }
848
- int8_t in_slab[CA_RANK_MAX];
849
- int8_t k;
850
- for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
851
- for ( k = 0; k < naxes; k++ ) {
852
- int8_t ax = axes[k];
853
- if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
854
- if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
855
- in_slab[ax] = 1;
856
- }
857
-
858
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.3 hook (= parallel to
859
- F.6.2 SRC_ATTACH hook): for descriptor views CAWindow /
860
- CAShift (predicate returns 1 unconditionally per F.5 always-
861
- win bench), skip materialise (alias / C.1 PER_SLAB / B.1.5)
862
- and use per-fiber ca_xfer_stride which routes to the view's
863
- X.1 OOB-fused per-region fast path. */
864
- if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
865
- && raw_ndim == src->ndim ) {
866
- int8_t fiber_ax = axes[0];
867
- ca_size_t row_byte_strides[CA_RANK_MAX];
868
- {
869
- ca_size_t b = src->bytes;
870
- for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
871
- row_byte_strides[kk] = b;
872
- b *= src->dim[kk];
873
- }
874
- }
875
- if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
876
- row_byte_strides[fiber_ax],
877
- flags) ) {
878
- st->src = src;
879
- st->src_kind = src_kind;
880
- st->level = 2;
881
- st->policy = policy;
882
- st->ndim = src->ndim;
883
- st->flags = flags;
884
- st->bytes = src->bytes;
885
- st->axes = axes;
886
- st->naxes = naxes;
887
-
888
- st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
889
- st->alias_ptr = NULL;
890
- st->fiber_axis = fiber_ax;
891
- for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
892
- st->fiber_native_strides[kk] = row_byte_strides[kk];
893
- }
894
-
895
- /* Row-major element strides for mask (= identity element
896
- index strides over src->dim). */
897
- ca_size_t row_elem_strides[CA_RANK_MAX];
898
- {
899
- ca_size_t e = 1;
900
- for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
901
- row_elem_strides[kk] = e;
902
- e *= src->dim[kk];
903
- }
904
- }
905
-
906
- int8_t sp = 0, op = 0;
907
- st->slab_elements = 1;
908
- for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
909
- if ( in_slab[kk] ) {
910
- st->slab_axes_buf[sp] = kk;
911
- st->slab_dims[sp] = src->dim[kk];
912
- st->slab_strides[sp] = row_byte_strides[kk];
913
- st->slab_mask_strides[sp] = row_elem_strides[kk];
914
- st->slab_elements *= src->dim[kk];
915
- sp++;
916
- } else {
917
- st->outer_axes[op] = kk;
918
- st->outer_dims[op] = src->dim[kk];
919
- st->outer_strides[op] = row_byte_strides[kk];
920
- st->outer_mask_strides[op] = row_elem_strides[kk];
921
- op++;
922
- }
923
- }
924
- st->slab_ndim = sp;
925
- st->outer_ndim = op;
926
- st->desc_ndim = 0;
927
-
928
- ca_size_t total = 1;
929
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
930
- st->total_slabs = total;
931
- st->slab_n = st->slab_elements;
932
- st->slabs_emitted = 0;
933
- st->chunk_size = st->slab_n;
934
-
935
- if ( st->outer_ndim > 0 ) {
936
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
937
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
938
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
939
- } else {
940
- st->outer_idx = NULL;
941
- }
942
-
943
- /* Mask: per-fiber gather via next_slab_axes; no whole-view
944
- materialise (rev2 §3.3, (data,mask) pair travels together). */
945
- st->alias_mask = NULL;
946
- return CA_ITER_OK;
947
- }
948
- }
949
-
950
- /* Phase C T3 (C.1): slab axis has INDEX or SHIFT kind → per-slab
951
- materialise fallback path (D1.1 (B) + D1.2 (A)). CAMapping-style
952
- views (raw_ndim != src->ndim) are out of C.1 scope — descriptor
953
- exposes a flat 1-D INDEX gather that doesn't expose the view-
954
- axis partition the user gave via slab_axes. Reject explicitly. */
955
- if ( raw_ndim != src->ndim ) {
956
- return CA_ITER_ERR_POLICY;
957
- }
958
- int slab_has_non_stride = 0;
959
- for ( k = 0; k < src->ndim; k++ ) {
960
- if ( in_slab[k] && raw_descs[k].kind != CA_AXIS_KIND_STRIDE ) {
961
- slab_has_non_stride = 1;
962
- break;
963
- }
964
- }
965
- if ( slab_has_non_stride ) {
966
- /* C.1 scope: READ-only. WRITE = future sub-step (C.1c). */
967
- if ( flags & CA_KERNEL_WRITE ) {
968
- return CA_ITER_ERR_FLAGS;
969
- }
970
-
971
- st->src = src;
972
- st->src_kind = CA_ITER_SRC_DESCRIPTOR;
973
- st->level = 2;
974
- st->policy = policy;
975
- st->ndim = src->ndim;
976
- st->flags = flags;
977
- st->bytes = src->bytes;
978
- st->axes = axes;
979
- st->naxes = naxes;
980
-
981
- CArray *parent = CAVIEW(src)->parent;
982
- ca_attach(parent);
983
- st->root = parent;
984
-
985
- /* Persist raw_descs / pdims / pstrides for per-slab subset
986
- construction (PER_SLAB fallback) or for the hoisted manual
987
- gather (PER_SLAB_HOIST specialised path). desc_ndim =
988
- src->ndim so has_descs branch in next_slab_axes can read
989
- st->descs (though T3 path takes its own branch before that
990
- check). */
991
- memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
992
- memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
993
- {
994
- ca_size_t s = src->bytes;
995
- for ( k = src->ndim - 1; k >= 0; k-- ) {
996
- st->pstrides[k] = s;
997
- s *= raw_pdims[k];
998
- }
999
- }
1000
- st->desc_ndim = src->ndim;
1001
-
1002
- /* View-row-major element strides (= mask scratch layout, since
1003
- mask is gathered whole-view once at init). */
1004
- ca_size_t row_elem_strides[CA_RANK_MAX];
1005
- {
1006
- ca_size_t e = 1;
1007
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1008
- row_elem_strides[k] = e;
1009
- e *= src->dim[k];
1010
- }
1011
- }
1012
-
1013
- /* Row-major byte strides over SLAB axes only (= scratch_ptr
1014
- layout, since scratch_ptr is sized slab_elements × bytes
1015
- and refilled per outer iter as a contig row-major slab). */
1016
- ca_size_t slab_data_strides[CA_RANK_MAX];
1017
- {
1018
- ca_size_t b = src->bytes;
1019
- ca_size_t e_slab = 1;
1020
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1021
- if ( in_slab[k] ) {
1022
- slab_data_strides[k] = b;
1023
- b *= raw_descs[k].count;
1024
- e_slab *= raw_descs[k].count;
1025
- } else {
1026
- slab_data_strides[k] = 0; /* unused for outer axes */
1027
- }
1028
- }
1029
- st->slab_elements = e_slab;
1030
- }
1031
-
1032
- int8_t sp = 0, op = 0;
1033
- for ( k = 0; k < src->ndim; k++ ) {
1034
- if ( in_slab[k] ) {
1035
- st->slab_axes_buf[sp] = k;
1036
- st->slab_dims[sp] = raw_descs[k].count;
1037
- st->slab_strides[sp] = slab_data_strides[k];
1038
- /* Slab mask stride is view-row-major elem stride along
1039
- this view axis (mask scratch is whole-view layout). */
1040
- st->slab_mask_strides[sp] = row_elem_strides[k];
1041
- sp++;
1042
- } else {
1043
- st->outer_axes[op] = k;
1044
- st->outer_dims[op] = raw_descs[k].count;
1045
- /* T3 path: outer data offset is always 0 (scratch refilled
1046
- per slab, ptr = scratch_ptr). */
1047
- st->outer_strides[op] = 0;
1048
- /* Outer mask offset uses view-row-major elem stride into
1049
- whole-view mask scratch. */
1050
- st->outer_mask_strides[op] = row_elem_strides[k];
1051
- op++;
1052
- }
1053
- }
1054
- st->slab_ndim = sp;
1055
- st->outer_ndim = op;
1056
- st->composed_base = 0; /* unused for T3 fallback */
1057
-
1058
- ca_size_t total = 1;
1059
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1060
- st->total_slabs = total;
1061
- st->slab_n = st->slab_elements;
1062
- st->slabs_emitted = 0;
1063
- st->chunk_size = st->slab_n;
1064
-
1065
- if ( st->outer_ndim > 0 ) {
1066
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1067
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1068
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1069
- } else {
1070
- st->outer_idx = NULL;
1071
- }
1072
-
1073
- /* Alloc per-slab data scratch (D1.1 (B): one max-slab buffer
1074
- refilled per outer iter, instead of per-iter alloc/free or
1075
- full-view materialise). */
1076
- ca_size_t slab_bytes = st->slab_elements * src->bytes;
1077
- st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
1078
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap);
1079
- st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
1080
-
1081
- /* Specialisation (B-1b, C.1b): if innermost slab axis is STRIDE
1082
- and no SHIFT axes anywhere in the view, use the hoisted
1083
- manual gather path (= outer + non-innermost-slab hoist,
1084
- inner pure STRIDE linear memcpy). Else (innermost INDEX /
1085
- SHIFT-anywhere): use the engine fallback (A). */
1086
- int specialised_eligible = (sp > 0);
1087
- if ( specialised_eligible ) {
1088
- int8_t inner_view_ax = st->slab_axes_buf[sp - 1];
1089
- if ( raw_descs[inner_view_ax].kind != CA_AXIS_KIND_STRIDE ) {
1090
- specialised_eligible = 0;
1091
- }
1092
- }
1093
- if ( specialised_eligible ) {
1094
- /* Any SHIFT axis (slab or outer) → fallback to engine for
1095
- clean bound_fill semantics in C.1b scope. */
1096
- for ( k = 0; k < src->ndim; k++ ) {
1097
- if ( raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
1098
- specialised_eligible = 0;
1099
- break;
1100
- }
1101
- }
1102
- }
1103
- st->alias_mode = specialised_eligible
1104
- ? CA_ITER_ALIAS_PER_SLAB_HOIST
1105
- : CA_ITER_ALIAS_PER_SLAB;
1106
-
1107
- /* Mask: gather whole view mask once into scratch_mask (= same
1108
- strategy as B.1.5). Per-slab mask offset is computed via
1109
- outer_idx × outer_mask_strides in next_slab_axes. */
1110
- if ( ca_has_mask(src) ) {
1111
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1112
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1113
- if ( src->elements > 0 ) {
1114
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1115
- }
1116
- st->alias_mask = st->scratch_mask;
1117
- }
1118
-
1119
- return CA_ITER_OK;
1120
- }
1121
- /* slab is all-STRIDE: fall through to B.1.5 / Phase B paths. */
1122
- /* Phase B.1.5: outer SHIFT axis → materialise downgrade. Alias
1123
- can't deliver OOB cells (= need fill_value), and the view-family
1124
- surface prioritises delivering the cells over avoiding a copy, so
1125
- we materialise the entire view into a row-major scratch
1126
- buffer via ca_axis_dispatch_attach (= same engine as the
1127
- existing SRC_DESCRIPTOR L2 NONE path), then walk it with
1128
- Phase A-style row-major K-D strides. Slab / outer partition
1129
- applies to view-axes; the scratch IS the flat row-major view. */
1130
- int outer_has_shift = 0;
1131
- for ( k = 0; k < src->ndim; k++ ) {
1132
- if ( !in_slab[k] && raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
1133
- outer_has_shift = 1;
1134
- break;
1135
- }
1136
- }
1080
+ /* Splits the view's axes into slab and outer axes (both in ascending axis
1081
+ order, so the user's order does not matter), counts the slabs and opens
1082
+ the outer cursor. */
1083
+ static void
1084
+ ki_lay_out_slabs (ca_iter_state *st, CArray *src, const int8_t *in_slab,
1085
+ const ki_axis_layout *lay)
1086
+ {
1087
+ int8_t sp = 0, op = 0;
1088
+ st->slab_elements = 1;
1089
+ for ( int8_t k = 0; k < src->ndim; k++ ) {
1090
+ if ( in_slab[k] ) {
1091
+ st->slab_axes_buf[sp] = k;
1092
+ st->slab_dims[sp] = lay->dims[k];
1093
+ st->slab_strides[sp] = lay->slab_strides[k];
1094
+ st->slab_mask_strides[sp] = lay->mask_strides[k];
1095
+ st->slab_elements *= lay->dims[k];
1096
+ sp++;
1097
+ } else {
1098
+ st->outer_axes[op] = k;
1099
+ st->outer_dims[op] = lay->dims[k];
1100
+ st->outer_strides[op] = lay->outer_strides[k];
1101
+ st->outer_mask_strides[op] = lay->mask_strides[k];
1102
+ op++;
1103
+ }
1104
+ }
1105
+ st->slab_ndim = sp;
1106
+ st->outer_ndim = op;
1137
1107
 
1138
- if ( outer_has_shift ) {
1139
- st->src = src;
1140
- st->src_kind = CA_ITER_SRC_DESCRIPTOR;
1141
- st->level = 2;
1142
- st->policy = policy;
1143
- st->ndim = src->ndim;
1144
- st->flags = flags;
1145
- st->bytes = src->bytes;
1146
- st->axes = axes;
1147
- st->naxes = naxes;
1148
-
1149
- CArray *parent = CAVIEW(src)->parent;
1150
- ca_attach(parent);
1151
- st->root = parent;
1152
-
1153
- ca_axis_desc_t local_descs[CA_RANK_MAX];
1154
- memcpy(local_descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
1155
- const void *bound_fill = NULL;
1156
- if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
1157
- bound_fill = ((CAWindow *) src)->fill;
1158
- }
1159
- st->scratch_cap = src->elements * src->bytes;
1160
- st->scratch_ptr = ca_axis_dispatch_attach(parent, raw_pdims,
1161
- local_descs, src->ndim,
1162
- src->bytes, src->elements,
1163
- bound_fill);
1164
- st->alias_mode = CA_ITER_ALIAS_NONE;
1165
- st->alias_ptr = st->scratch_ptr;
1166
-
1167
- /* Row-major byte / element strides on the scratch (= view layout). */
1168
- ca_size_t row_byte_strides[CA_RANK_MAX];
1169
- ca_size_t row_elem_strides[CA_RANK_MAX];
1170
- {
1171
- ca_size_t b = src->bytes, e = 1;
1172
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1173
- row_byte_strides[k] = b;
1174
- row_elem_strides[k] = e;
1175
- b *= src->dim[k];
1176
- e *= src->dim[k];
1177
- }
1178
- }
1108
+ ca_size_t total = 1;
1109
+ for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1110
+ st->total_slabs = total;
1111
+ st->slab_n = st->slab_elements;
1112
+ st->slabs_emitted = 0;
1113
+ st->chunk_size = st->slab_n;
1179
1114
 
1180
- int8_t sp = 0, op = 0;
1181
- st->slab_elements = 1;
1182
- for ( k = 0; k < src->ndim; k++ ) {
1183
- if ( in_slab[k] ) {
1184
- st->slab_axes_buf[sp] = k;
1185
- st->slab_dims[sp] = src->dim[k];
1186
- st->slab_strides[sp] = row_byte_strides[k];
1187
- st->slab_mask_strides[sp] = row_elem_strides[k];
1188
- st->slab_elements *= src->dim[k];
1189
- sp++;
1190
- } else {
1191
- st->outer_axes[op] = k;
1192
- st->outer_dims[op] = src->dim[k];
1193
- st->outer_strides[op] = row_byte_strides[k];
1194
- st->outer_mask_strides[op] = row_elem_strides[k];
1195
- op++;
1196
- }
1197
- }
1198
- st->slab_ndim = sp;
1199
- st->outer_ndim = op;
1200
- st->composed_base = 0;
1201
- /* desc_ndim = 0: next_slab_axes treats outer like Phase A
1202
- (no INDEX lookup), correct because scratch is row-major
1203
- contig (= STRIDE everywhere). */
1204
- st->desc_ndim = 0;
1205
-
1206
- ca_size_t total = 1;
1207
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1208
- st->total_slabs = total;
1209
- st->slab_n = st->slab_elements;
1210
- st->slabs_emitted = 0;
1211
- st->chunk_size = st->slab_n;
1212
-
1213
- if ( st->outer_ndim > 0 ) {
1214
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1215
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1216
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1217
- } else {
1218
- st->outer_idx = NULL;
1219
- }
1115
+ if ( st->outer_ndim > 0 ) {
1116
+ CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1117
+ st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1118
+ for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1119
+ } else {
1120
+ st->outer_idx = NULL;
1121
+ }
1122
+ }
1220
1123
 
1221
- if ( ca_has_mask(src) ) {
1222
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1223
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1224
- if ( src->elements > 0 ) {
1225
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1226
- }
1227
- st->alias_mask = st->scratch_mask;
1228
- }
1229
- return CA_ITER_OK;
1230
- }
1124
+ /* Chooses the route. Decides only: nothing is attached or allocated, so
1125
+ a refusal leaves nothing to close. */
1126
+ static int
1127
+ ki_choose_l2_route (CArray *src, int8_t *axes, int8_t naxes, uint32_t flags,
1128
+ ki_l2_plan *plan)
1129
+ {
1130
+ const int8_t *in_slab = plan->in_slab;
1131
+ ca_size_t row_bytes[CA_RANK_MAX];
1132
+ int8_t k;
1133
+
1134
+ ki_view_row_major(src, row_bytes, NULL);
1135
+
1136
+ if ( plan->src_kind == CA_ITER_SRC_DESCRIPTOR
1137
+ || plan->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
1138
+ /* Shortcut PER_FIBER_FUSED (CAWindow / CAShift).
1139
+ Layer all three, per fiber: each fiber is fetched with the
1140
+ view's ca_xfer_stride (which fills out-of-bounds
1141
+ cells itself) and put back the same way.
1142
+ Speeds up a fiber walk of a CAWindow / CAShift whose innermost
1143
+ axis is SHIFT.
1144
+ Why none measured: about 8% slower than the descriptor
1145
+ route on a sort of a 2000x2000 shift view.
1146
+ Enters CA_KERNEL_FIBER_CONTIG, one slab axis, the view's
1147
+ own rank, and ca_iter_should_per_fiber_fused. An
1148
+ outer-only SHIFT arrives as L2_ALIASABLE and never
1149
+ reaches this hook.
1150
+ Promises 1 (the fiber scratch is contiguous), 2 (the mask is
1151
+ fetched into the fiber mask scratch), 3 (ca_xfer_stride
1152
+ PUT of a fiber that neither converts nor repeats a
1153
+ cell), 4 (CA_ITER_FLAG_WALK_MAY_RAISE).
1154
+ Switch ca_fast_paths_enabled(): off, the descriptor route.
1155
+ Checked by the fiber rows of the "shift" view in the delivery
1156
+ matrix. */
1157
+ if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
1158
+ && plan->ndesc == src->ndim && ca_fast_paths_enabled()
1159
+ && ca_iter_should_per_fiber_fused(src, plan->src_kind, axes[0],
1160
+ row_bytes[axes[0]], flags) ) {
1161
+ plan->route = KI_L2_FIBER_FUSED;
1162
+ return CA_ITER_OK;
1163
+ }
1231
1164
 
1232
- st->src = src;
1233
- st->src_kind = CA_ITER_SRC_DESCRIPTOR; /* Phase B alias path */
1234
- st->level = 2;
1235
- st->policy = policy;
1236
- st->ndim = src->ndim;
1237
- st->flags = flags;
1238
- st->bytes = src->bytes;
1239
- st->axes = axes;
1240
- st->naxes = naxes;
1241
-
1242
- /* Parent row-major byte strides for offset computation. */
1243
- ca_size_t pstrides[CA_RANK_MAX];
1244
- {
1245
- ca_size_t s = src->bytes;
1246
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1247
- pstrides[k] = s;
1248
- s *= raw_pdims[k];
1249
- }
1165
+ /* The rank check guards a descriptor whose rank differs from the
1166
+ view's (it would not expose the axis partition slab_axes names).
1167
+ Every describe_axes in the engine reports the view's rank, so the
1168
+ check does not fire today; it stays as the contract's guard. */
1169
+ if ( plan->ndesc != src->ndim ) return CA_ITER_ERR_POLICY;
1170
+
1171
+ int slab_has_non_stride = 0;
1172
+ int slab_has_index = 0;
1173
+ int outer_has_shift = 0;
1174
+ for ( k = 0; k < src->ndim; k++ ) {
1175
+ if ( in_slab[k] && plan->descs[k].kind != CA_AXIS_KIND_STRIDE ) {
1176
+ slab_has_non_stride = 1;
1177
+ if ( plan->descs[k].kind == CA_AXIS_KIND_INDEX ) slab_has_index = 1;
1250
1178
  }
1251
- /* View row-major element strides (= mask scratch layout). */
1252
- ca_size_t row_elem_strides[CA_RANK_MAX];
1253
- {
1254
- ca_size_t s = 1;
1255
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1256
- row_elem_strides[k] = s;
1257
- s *= src->dim[k];
1258
- }
1179
+ if ( !in_slab[k] && plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) {
1180
+ outer_has_shift = 1;
1259
1181
  }
1182
+ }
1260
1183
 
1261
- /* Partition + populate slab / outer metadata. Slab axes contribute
1262
- to a constant slab_base (since they're all STRIDE: start +
1263
- step*idx, where the base is start*pstride summed). Outer axes
1264
- drive per-slab offset via direct (start + step*idx)*pstride or
1265
- indices[idx]*pstride (= no classify_prefix engine, since outer
1266
- axes here are not a prefix of raw_descs[] — they're a
1267
- complement of slab_axes). */
1268
- int8_t sp = 0, op = 0;
1269
- ca_size_t slab_base = 0;
1270
- st->slab_elements = 1;
1271
- for ( k = 0; k < src->ndim; k++ ) {
1272
- if ( in_slab[k] ) {
1273
- /* STRIDE-kind slab axis: start*pstride goes to slab_base,
1274
- step*pstride is the per-cell byte stride. */
1275
- slab_base += raw_descs[k].start * pstrides[k];
1276
- st->slab_axes_buf[sp] = k;
1277
- st->slab_dims[sp] = raw_descs[k].count;
1278
- st->slab_strides[sp] = raw_descs[k].step * pstrides[k];
1279
- st->slab_mask_strides[sp] = row_elem_strides[k];
1280
- st->slab_elements *= raw_descs[k].count;
1281
- sp++;
1282
- } else {
1283
- /* Outer axis: STRIDE or INDEX (SHIFT was rejected above).
1284
- Store axis index + view's outer dim/stride for descriptor
1285
- walk in next_slab_axes. We reuse outer_strides/outer_mask
1286
- _strides as the OUTER walk metadata; outer-axis kind is
1287
- carried implicitly via raw_descs[outer_axes[m]] in the
1288
- state struct's descs[] field (populated below). */
1289
- st->outer_axes[op] = k;
1290
- st->outer_dims[op] = raw_descs[k].count;
1291
- /* For STRIDE: per-axis stride = step * pstride; INDEX axes
1292
- use indices[] from raw_descs and need a 0 stride here
1293
- (next_slab_axes branches on descs[].kind to compute the
1294
- real offset). We persist the full raw_descs in st->descs
1295
- so the next_slab_axes implementation can dispatch. */
1296
- if ( raw_descs[k].kind == CA_AXIS_KIND_STRIDE ) {
1297
- st->outer_strides[op] = raw_descs[k].step * pstrides[k];
1298
- /* Bake in start*pstride into slab_base for STRIDE outer too
1299
- so per-cell offset is purely step*idx. */
1300
- slab_base += raw_descs[k].start * pstrides[k];
1301
- } else {
1302
- /* INDEX kind: outer_strides[m] = pstrides[k] (element-unit
1303
- from indices[]). We DON'T pre-bake into slab_base for
1304
- INDEX axes; per-iter offset = indices[idx[m]] * pstrides[k]
1305
- is computed in next_slab_axes. */
1306
- st->outer_strides[op] = pstrides[k];
1307
- }
1308
- st->outer_mask_strides[op] = row_elem_strides[k];
1309
- op++;
1310
- }
1311
- }
1312
- st->slab_ndim = sp;
1313
- st->outer_ndim = op;
1314
- st->composed_base = slab_base;
1315
-
1316
- /* Persist raw_descs / pstrides in state so next_slab_axes can
1317
- distinguish STRIDE vs INDEX outer per axis. */
1318
- memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
1319
- memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
1320
- memcpy(st->pstrides, pstrides, src->ndim * sizeof(ca_size_t));
1321
- st->desc_ndim = src->ndim;
1322
-
1323
- ca_size_t total = 1;
1324
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1325
- st->total_slabs = total;
1326
- st->slab_n = st->slab_elements;
1327
- st->slabs_emitted = 0;
1328
- st->chunk_size = st->slab_n;
1329
-
1330
- if ( st->outer_ndim > 0 ) {
1331
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1332
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1333
- } else {
1334
- st->outer_idx = NULL;
1184
+ /* A slab with a SHIFT axis that leaves out the innermost view axis
1185
+ would be gathered one strided cell at a time, once per slab. Read
1186
+ walks take the whole-view route instead: the view is gathered
1187
+ once, row by row, and the slabs are walked in the copy.
1188
+
1189
+ Shortcut SHIFT_WHOLE_VIEW.
1190
+ Layer data; the fiber layer (ki_fiber_contig_*) sits on top.
1191
+ Speeds up read walks of a CAWindow / CAShift whose slab has a
1192
+ SHIFT axis and not the innermost axis.
1193
+ Why about 1.5x on a median along axis 0 of a 2000x2000
1194
+ shift view.
1195
+ Enters a non-stride slab axis, no INDEX slab axis, the
1196
+ innermost axis outside the slab, a read walk.
1197
+ Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
1198
+ only when the view has one).
1199
+ Switch ca_fast_paths_enabled(): off, the per-slab gather.
1200
+ Checked by the slab-axis rows of the "shift" and "window" views
1201
+ in the delivery matrix. */
1202
+ int whole_view = slab_has_non_stride && ! slab_has_index
1203
+ && ! in_slab[src->ndim - 1]
1204
+ && ! ( flags & CA_KERNEL_WRITE )
1205
+ && ca_fast_paths_enabled();
1206
+
1207
+ /* The descriptor engine's gathers (per slab and whole view) read
1208
+ only. A write takes the transfer walk instead: the view is
1209
+ gathered whole and the changed cells go back through its own
1210
+ transfer (promise 3). */
1211
+ if ( (slab_has_non_stride && ! whole_view) || outer_has_shift ) {
1212
+ if ( flags & CA_KERNEL_WRITE ) {
1213
+ plan->src_kind = CA_ITER_SRC_ATTACH;
1214
+ goto by_transfer;
1335
1215
  }
1216
+ }
1217
+ plan->src_kind = CA_ITER_SRC_DESCRIPTOR;
1218
+ if ( slab_has_non_stride && ! whole_view ) {
1219
+ plan->route = KI_L2_PER_SLAB;
1220
+ }
1221
+ else if ( outer_has_shift || whole_view ) {
1222
+ plan->route = KI_L2_DESC_WHOLE;
1223
+ }
1224
+ else {
1225
+ plan->route = KI_L2_DESC_STRIDED;
1226
+ }
1227
+ return CA_ITER_OK;
1228
+ }
1336
1229
 
1337
- /* Attach parent for alias path. */
1338
- CArray *parent = CAVIEW(src)->parent;
1339
- ca_attach(parent);
1340
- st->root = parent;
1341
- st->alias_mode = CA_ITER_ALIAS_STRIDED;
1342
- st->alias_ptr = (char *) parent->ptr;
1343
-
1344
- /* Mask gather (= view row-major boolean8_t). */
1345
- if ( ca_has_mask(src) ) {
1346
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1347
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1348
- if ( src->elements > 0 ) {
1349
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1350
- }
1351
- st->alias_mask = st->scratch_mask;
1352
- }
1230
+ if ( plan->src_kind == CA_ITER_SRC_ATTACH ) {
1231
+ by_transfer:
1232
+ /* Shortcut STACK.
1233
+ Layer data and fiber: the slab is cut contiguous from the K
1234
+ parents through a tile cache.
1235
+ Speeds up read walks of an unmasked CAStack whose one slab axis
1236
+ is the stack axis.
1237
+ Why about 2x on a mean along the stack axis of 4 parents
1238
+ of 500x2000.
1239
+ Enters a CAStack, no mask, a read walk, one slab axis that is
1240
+ k_axis.
1241
+ Promises 1 (the slab is contiguous), 6 (no mask exists to touch).
1242
+ Switch ca_fast_paths_enabled(): off, the transfer walk.
1243
+ Checked by the slab-axis and fiber rows of the stack parent along
1244
+ axis 0 in the delivery matrix. */
1245
+ if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1246
+ && !ca_has_mask(src)
1247
+ && !(flags & CA_KERNEL_WRITE)
1248
+ && naxes == 1 && axes[0] == ((CAStack *) src)->k_axis
1249
+ && ca_fast_paths_enabled() ) {
1250
+ plan->route = KI_L2_STACK;
1353
1251
  return CA_ITER_OK;
1354
1252
  }
1355
- if ( src_kind == CA_ITER_SRC_ATTACH ) {
1356
- /* Phase B.5: CA_SLAB_AXES + SRC_ATTACH (= CAFake / CAByteSwap /
1357
- CABitfield / CABitarray / CAReduce). These overlay views
1358
- materialise via their own ca_attach (= no descriptor framework
1359
- path), so we let ca_attach(src) populate src->ptr in view
1360
- row-major layout and walk it with Phase A-style K-D strides.
1361
-
1362
- The most common entry path here is wrap_readonly(int_src,
1363
- CA_FLOAT64) → CAFake, enabling sum_ki etc. to accept any
1364
- numeric data_type source via auto-cast (= D2.2 confirmed in
1365
- rev2 sparring). */
1366
- if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
1367
- return CA_ITER_ERR_POLICY;
1368
- }
1369
- int8_t in_slab[CA_RANK_MAX];
1370
- int8_t k;
1371
- for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
1372
- for ( k = 0; k < naxes; k++ ) {
1373
- int8_t ax = axes[k];
1374
- if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
1375
- if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
1376
- in_slab[ax] = 1;
1377
- }
1378
-
1379
- st->src = src;
1380
- st->src_kind = CA_ITER_SRC_ATTACH;
1381
- st->level = 2;
1382
- st->policy = policy;
1383
- st->ndim = src->ndim;
1384
- st->flags = flags;
1385
- st->bytes = src->bytes;
1386
- st->axes = axes;
1387
- st->naxes = naxes;
1388
-
1389
- /* Row-major byte / element strides on the attached view buffer.
1390
- Computed first so the F.6.1 predicate can inspect fiber-axis
1391
- effective stride before deciding whether to materialise. */
1392
- ca_size_t row_byte_strides[CA_RANK_MAX];
1393
- ca_size_t row_elem_strides[CA_RANK_MAX];
1394
- {
1395
- ca_size_t b = src->bytes, e = 1;
1396
- for ( k = src->ndim - 1; k >= 0; k-- ) {
1397
- row_byte_strides[k] = b;
1398
- row_elem_strides[k] = e;
1399
- b *= src->dim[k];
1400
- e *= src->dim[k];
1401
- }
1402
- }
1403
-
1404
- /* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev4 (direct per-
1405
- parent ptr access path + tile cache). Engages when:
1406
- - source.attach is ca_stack_func.attach (= CAStack identity)
1407
- - no mask (= horizontal mask propagation per-slab gather is
1408
- out of scope; falls back to SRC_ATTACH whole-view path)
1409
- - naxes == 1 && axes[0] == k_axis (= K-axis-only slab, the
1410
- demand-driving case = reduce along the stacked axis like
1411
- view.mean(axis: k_axis))
1412
- init attaches K parents up front (O(1) per entity parent),
1413
- caches parent->ptr aliases + uniform parent-native byte strides,
1414
- allocates a slab-sized scratch (= K * bytes) plus a tile cache
1415
- sized to fit the L1d budget. next_slab_axes refills TILE fibers
1416
- per K contig parent reads (rev4 2026-06-19) and serves the next
1417
- TILE-1 calls from the L1d-resident cache, beating SRC_ATTACH
1418
- across all measured sizes. No size-threshold gate: rev3's gate
1419
- was a perf trade-off justification that the tile cache erased.
1420
-
1421
- K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): engage predicate
1422
- generalised from `axes[0] == 0` to `axes[0] == k_axis`. The
1423
- tile cache mechanism + outer_idx -> parent axis mapping are
1424
- already k_axis-agnostic: outer_idx[m] maps to parent axis m
1425
- regardless of k_axis position (= for k_axis = 0 outer is stack
1426
- axes 1..N-1 = parent axes 0..N-2; for k_axis > 0 outer is stack
1427
- axes [0..k_axis-1, k_axis+1..N-1] = parent axes [0..k_axis-1,
1428
- k_axis..N-2]; either way outer_idx[m] = parent axis m). */
1429
- if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1430
- && !ca_has_mask(src)
1431
- && !(flags & CA_KERNEL_WRITE)
1432
- && naxes == 1 && axes[0] == ((CAStack *) src)->k_axis ) {
1433
- CAStack *stack = (CAStack *) src;
1434
- int8_t parent_ndim = src->ndim - 1;
1435
- int8_t sp = 0, op = 0;
1436
-
1437
- st->slab_elements = 1;
1438
- for ( k = 0; k < src->ndim; k++ ) {
1439
- if ( in_slab[k] ) {
1440
- st->slab_axes_buf[sp] = k;
1441
- st->slab_dims[sp] = src->dim[k];
1442
- st->slab_elements *= src->dim[k];
1443
- sp++;
1444
- } else {
1445
- st->outer_axes[op] = k;
1446
- st->outer_dims[op] = src->dim[k];
1447
- st->outer_strides[op] = row_byte_strides[k]; /* unused on STACK path */
1448
- st->outer_mask_strides[op] = row_elem_strides[k]; /* unused on STACK path */
1449
- op++;
1450
- }
1451
- }
1452
- st->slab_ndim = sp;
1453
- st->outer_ndim = op;
1454
- st->composed_base = 0;
1455
- st->desc_ndim = 0;
1456
- /* slab_strides on STACK path: slab_axes == [0], scratch is
1457
- packed contig over K elements (= K * bytes), so single-axis
1458
- slab walk uses bytes stride. */
1459
- st->slab_strides[0] = src->bytes;
1460
- st->slab_mask_strides[0] = 1; /* harmless on no-mask path */
1461
-
1462
- ca_size_t total = 1;
1463
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1464
- st->total_slabs = total;
1465
- st->slab_n = st->slab_elements;
1466
- st->slabs_emitted = 0;
1467
- st->chunk_size = st->slab_n;
1468
-
1469
- if ( st->outer_ndim > 0 ) {
1470
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1471
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1472
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1473
- } else {
1474
- st->outer_idx = NULL;
1475
- }
1476
-
1477
- /* Attach K parents and cache their ptr aliases. CAStack
1478
- guarantees uniform shape (MEMO §3.2) so one set of parent
1479
- native byte strides covers all K (= parents[0]->dim is
1480
- canonical). */
1481
- st->stack_n_parents = stack->n_parents;
1482
- st->stack_parent_ptrs =
1483
- (char **) xmalloc(stack->n_parents * sizeof(char *));
1484
- for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1485
- ca_attach(stack->parents[kk]);
1486
- st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
1487
- }
1488
- {
1489
- ca_size_t s = src->bytes;
1490
- for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
1491
- st->stack_parent_strides[kk] = s;
1492
- s *= stack->parents[0]->dim[kk];
1493
- }
1494
- }
1495
-
1496
- /* Slab-sized scratch (= K * bytes for slab_axes == [0]). */
1497
- st->scratch_cap = (ca_size_t) st->slab_elements * src->bytes;
1498
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
1499
-
1500
- /* Tile cache (pilot/castack-axis0-loop-interchange): refill TILE
1501
- fibers per K parent reads, serve next TILE-1 fibers from L1d.
1502
- TILE budget = ~32 KB (half of M2 L1d). Empirically tested
1503
- values; clamped to [8, 64] to keep the refill loop tight. */
1504
- {
1505
- ca_size_t K = (ca_size_t) stack->n_parents;
1506
- ca_size_t bytes = (ca_size_t) src->bytes;
1507
- ca_size_t tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
1508
- if ( tile < 8 ) tile = 8;
1509
- if ( tile > 64 ) tile = 64;
1510
- st->stack_tile_cap = tile;
1511
- st->stack_tile_pos = 0;
1512
- st->stack_tile_have = 0; /* force refill on first call */
1513
- st->stack_tile_cache = (char *) xmalloc(K * tile * bytes);
1514
- }
1515
-
1516
- st->alias_mode = CA_ITER_ALIAS_STACK;
1517
- st->alias_ptr = NULL; /* no whole-view buffer */
1518
- st->alias_mask = NULL;
1519
- return CA_ITER_OK;
1520
- }
1521
-
1522
- /* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18):
1523
- CAStack source + slab_axes excludes axis 0 (= K-axis stays in
1524
- outer iter, e.g. view.mean(axis: 1), view.mean(axis: 2),
1525
- view.mean(axis: 1, 2)). Each slab corresponds to a region
1526
- inside ONE parent selected by outer_idx[K_outer_pos]. Engine
1527
- pre-attaches K parents (+ K parent masks if mask present),
1528
- caches their ptrs, and next_slab_axes aliases parents[k]->ptr +
1529
- parent_off directly -- no scratch, no materialise, parent
1530
- entity case = eager-equivalent memory bandwidth.
1531
-
1532
- Layering: engine detects CAStack identity by function-pointer
1533
- comparison and accesses parents[] / mask via ca_func[STACK]
1534
- operation table where possible. CAStack downcast occurs only
1535
- for the n_parents / parents[] read in init_l2 (= same scope as
1536
- rev3 STACK; AC3 forbids parents[] walk in next_slab_axes only).
1537
- Q1 disposition: option (i) minimal-diff inline in SLAB_AXES
1538
- branch -- evaluated as smallest diff with acceptable layering
1539
- (proposal §3.1). */
1540
- /* STACK_OUTER_K: K-axis stays in outer iter (slab carved out of
1541
- one parent at a time). K.3 (2026-06-20): engage predicate +
1542
- stack-axis -> parent-axis mapping generalised for arbitrary
1543
- k_axis. stack axis s maps to parent axis (s if s < k_axis else
1544
- s - 1); only the K stack axis (s == k_axis) has no parent
1545
- counterpart. */
1546
- if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1547
- && !(flags & CA_KERNEL_WRITE)
1548
- && naxes >= 1 && !in_slab[((CAStack *) src)->k_axis] ) {
1549
- CAStack *stack = (CAStack *) src;
1550
- int8_t k_axis = stack->k_axis;
1551
- int8_t parent_ndim = src->ndim - 1;
1552
- int8_t sp = 0, op = 0;
1553
-
1554
- st->slab_elements = 1;
1555
- for ( k = 0; k < src->ndim; k++ ) {
1556
- if ( in_slab[k] ) {
1557
- st->slab_axes_buf[sp] = k;
1558
- st->slab_dims[sp] = src->dim[k];
1559
- st->slab_elements *= src->dim[k];
1560
- /* slab_strides[sp] = parent native byte stride at parent
1561
- axis (k if k < k_axis else k - 1). k == k_axis cannot
1562
- appear here -- the engage predicate above excludes it. */
1563
- sp++;
1564
- } else {
1565
- st->outer_axes[op] = k;
1566
- st->outer_dims[op] = src->dim[k];
1567
- st->outer_strides[op] = row_byte_strides[k];
1568
- st->outer_mask_strides[op] = row_elem_strides[k];
1569
- op++;
1570
- }
1571
- }
1572
- st->slab_ndim = sp;
1573
- st->outer_ndim = op;
1574
- st->composed_base = 0;
1575
- st->desc_ndim = 0;
1576
-
1577
- /* Locate the K-axis (= stack axis k_axis) within outer_axes. */
1578
- st->stack_k_outer_pos = -1;
1579
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
1580
- if ( st->outer_axes[m] == k_axis ) {
1581
- st->stack_k_outer_pos = m;
1582
- break;
1583
- }
1584
- }
1585
-
1586
- /* Uniform parent native byte strides + parent element strides
1587
- for mask (= MEMO §3.2 uniform shape across parents,
1588
- parents[0]->dim is canonical). Mask strides cached so
1589
- next_slab_axes can compute mask_off without downcasting
1590
- (= AC3 layering goal). */
1591
- {
1592
- ca_size_t sb = src->bytes;
1593
- ca_size_t se = 1;
1594
- for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
1595
- st->stack_parent_strides[kk] = sb;
1596
- st->stack_parent_mask_strides[kk] = se;
1597
- sb *= stack->parents[0]->dim[kk];
1598
- se *= stack->parents[0]->dim[kk];
1599
- }
1600
- }
1601
-
1602
- /* Fill slab_strides + slab_mask_strides for slab axes (all of
1603
- which are parent inner axes since K is in outer). stack ax
1604
- -> parent ax: s if s < k_axis else s - 1. */
1605
- for ( int8_t s_i = 0; s_i < st->slab_ndim; s_i++ ) {
1606
- int8_t stack_ax = st->slab_axes_buf[s_i];
1607
- int8_t parent_ax = (stack_ax < k_axis) ? stack_ax : (stack_ax - 1);
1608
- st->slab_strides[s_i] = st->stack_parent_strides[parent_ax];
1609
- st->slab_mask_strides[s_i] = st->stack_parent_mask_strides[parent_ax];
1610
- }
1611
-
1612
- ca_size_t total = 1;
1613
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1614
- st->total_slabs = total;
1615
- st->slab_n = st->slab_elements;
1616
- st->slabs_emitted = 0;
1617
- st->chunk_size = st->slab_n;
1618
-
1619
- if ( st->outer_ndim > 0 ) {
1620
- CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
1621
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1622
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1623
- } else {
1624
- st->outer_idx = NULL;
1625
- }
1253
+ /* Shortcut STACK_OUTER_K.
1254
+ Layer data; the fiber layer sits on top.
1255
+ Speeds up read walks of a CAStack whose stack axis is an outer
1256
+ axis: each slab is cut from the one parent the outer
1257
+ index picks, without gathering the stack.
1258
+ Why about 11x on a mean along axis 1 of 4 parents of
1259
+ 500x2000.
1260
+ Enters a CAStack, a read walk, the stack axis outside the slab.
1261
+ Promises 2 (the parents' masks are copied), 6 (masks are read
1262
+ only when a parent has one), 8 (alias_ptr stays NULL,
1263
+ so no slab is lent as a window).
1264
+ Switch ca_fast_paths_enabled(): off, the transfer walk.
1265
+ Checked by the slab-axis, fiber and two-slab-axes rows of the
1266
+ stack parent along axes 1 and 2 in the delivery matrix. */
1267
+ if ( ca_func[src->obj_type].attach == ca_stack_func.attach
1268
+ && !(flags & CA_KERNEL_WRITE)
1269
+ && naxes >= 1 && !in_slab[((CAStack *) src)->k_axis]
1270
+ && ca_fast_paths_enabled() ) {
1271
+ plan->route = KI_L2_STACK_OUTER_K;
1272
+ return CA_ITER_OK;
1273
+ }
1274
+ /* Shortcut PER_FIBER_FUSED (conversion views).
1275
+ Layer all three, per fiber: each fiber is fetched with the
1276
+ view's ca_xfer_stride into the fiber scratch, without
1277
+ gathering the whole view.
1278
+ Speeds up a read of a fiber of a fake or byte-swap view, or of a
1279
+ lazy monop that only casts, along its innermost axis.
1280
+ Why no difference measured on a sort along the last axis
1281
+ of a 2000x2000 fake view.
1282
+ Enters CA_KERNEL_FIBER_CONTIG, one slab axis, a read walk, and
1283
+ ca_iter_should_per_fiber_fused.
1284
+ Promises 1, 2 and 4, as the CAWindow / CAShift hook above. A
1285
+ write takes the transfer walk (promise 3).
1286
+ Switch ca_fast_paths_enabled(): off, the transfer walk.
1287
+ Checked by the fiber rows of the fake parent in the delivery
1288
+ matrix. */
1289
+ if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
1290
+ && ca_fast_paths_enabled()
1291
+ && ca_iter_should_per_fiber_fused(src, CA_ITER_SRC_ATTACH, axes[0],
1292
+ row_bytes[axes[0]], flags) ) {
1293
+ plan->route = KI_L2_FIBER_FUSED;
1294
+ return CA_ITER_OK;
1295
+ }
1296
+ plan->route = KI_L2_TRANSFER;
1297
+ return CA_ITER_OK;
1298
+ }
1626
1299
 
1627
- /* Attach K parents + cache ptr aliases. Symmetric with rev3
1628
- Case B; for entity parents attach is O(1). */
1629
- st->stack_n_parents = stack->n_parents;
1630
- st->stack_parent_ptrs =
1631
- (char **) xmalloc(stack->n_parents * sizeof(char *));
1632
- for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1633
- ca_attach(stack->parents[kk]);
1634
- st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
1635
- }
1300
+ if ( plan->src_kind != CA_ITER_SRC_CASTRIDE ) return CA_ITER_ERR_POLICY;
1636
1301
 
1637
- /* If CAStack carries mask (= horizontal propagation already
1638
- applied at create_mask), attach K parent masks + cache ptr
1639
- aliases. next_slab_axes aliases parent->mask[k]->ptr +
1640
- mask_off for the same slab. */
1641
- if ( ca_has_mask(src) ) {
1642
- st->stack_parent_mask_ptrs =
1643
- (boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
1644
- for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1645
- ca_attach(stack->parents[kk]->mask);
1646
- st->stack_parent_mask_ptrs[kk] =
1647
- (boolean8_t *) stack->parents[kk]->mask->ptr;
1648
- }
1649
- } else {
1650
- st->stack_parent_mask_ptrs = NULL;
1651
- }
1652
-
1653
- st->scratch_ptr = NULL; /* no scratch: aliasing parent->ptr */
1654
- st->scratch_cap = 0;
1655
- st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
1656
- st->alias_ptr = NULL;
1657
- st->alias_mask = NULL;
1658
- return CA_ITER_OK;
1659
- }
1302
+ if ( ca_iter_can_alias(src, 1) ) {
1303
+ plan->route = KI_L2_CONTIG;
1304
+ return CA_ITER_OK;
1305
+ }
1306
+ /* A non-contiguous CAStride: compose the strides up to the root. A
1307
+ root that holds nothing to read through would cost one whole-root
1308
+ materialise no matter how few cells the view touches -- the cliff
1309
+ the xfer path avoids by asking for a region -- so that view is
1310
+ gathered by its own region protocol instead: ca_copy_data walks the
1311
+ leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
1312
+ CAObject asks for exactly that block. */
1313
+ ca_stride_compose_to_root((CAStride *) src, &plan->root, plan->composed,
1314
+ &plan->base);
1315
+ plan->route = ca_root_lends_no_memory(plan->root) ? KI_L2_VIEW_SCRATCH
1316
+ : KI_L2_ROOT_STRIDED;
1317
+ return CA_ITER_OK;
1318
+ }
1660
1319
 
1661
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 hook: try per-fiber fused
1662
- dispatch before whole-view materialise. When predicate fires,
1663
- skip scratch alloc + xfer_all GET (= per-fiber ca_xfer_stride
1664
- is called on demand in next_slab_axes). Predicate is stubbed
1665
- in F.6.1 (returns 0); F.6.2+ enables specific source kinds. */
1666
- int per_fiber_fused = 0;
1667
- if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1 ) {
1668
- int8_t fiber_ax = axes[0];
1669
- ca_size_t fiber_ax_stride = row_byte_strides[fiber_ax];
1670
- if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
1671
- fiber_ax_stride, flags) ) {
1672
- per_fiber_fused = 1;
1673
- st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
1674
- st->alias_ptr = NULL; /* no whole-view buffer */
1675
- st->fiber_axis = fiber_ax;
1676
- for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
1677
- st->fiber_native_strides[kk] = row_byte_strides[kk];
1678
- }
1679
- }
1680
- }
1320
+ /* Builds the state for the chosen route. */
1321
+ static void
1322
+ ki_open_l2 (ca_iter_state *st, CArray *src, ca_slab_policy_t policy,
1323
+ int8_t *axes, int8_t naxes, uint32_t flags, ki_l2_plan *plan)
1324
+ {
1325
+ ki_axis_layout lay;
1326
+ int8_t k;
1681
1327
 
1682
- if ( !per_fiber_fused ) {
1683
- /* 2026-05-31 refactor: iter-owns scratch via ca_xfer_all (was
1684
- ca_attach(src) + alias src->ptr). */
1685
- st->scratch_cap = (ca_size_t) src->elements * src->bytes;
1686
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
1687
- if ( src->elements > 0 ) {
1688
- ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
1689
- }
1690
- st->alias_mode = CA_ITER_ALIAS_NONE;
1691
- st->alias_ptr = st->scratch_ptr;
1692
- }
1328
+ st->src = src;
1329
+ st->src_kind = plan->src_kind;
1330
+ st->level = 2;
1331
+ st->policy = policy;
1332
+ st->ndim = src->ndim;
1333
+ st->flags = flags;
1334
+ st->bytes = src->bytes;
1335
+ st->naxes = naxes;
1693
1336
 
1694
- int8_t sp = 0, op = 0;
1695
- st->slab_elements = 1;
1696
- for ( k = 0; k < src->ndim; k++ ) {
1697
- if ( in_slab[k] ) {
1698
- st->slab_axes_buf[sp] = k;
1699
- st->slab_dims[sp] = src->dim[k];
1700
- st->slab_strides[sp] = row_byte_strides[k];
1701
- st->slab_mask_strides[sp] = row_elem_strides[k];
1702
- st->slab_elements *= src->dim[k];
1703
- sp++;
1704
- } else {
1705
- st->outer_axes[op] = k;
1706
- st->outer_dims[op] = src->dim[k];
1707
- st->outer_strides[op] = row_byte_strides[k];
1708
- st->outer_mask_strides[op] = row_elem_strides[k];
1709
- op++;
1710
- }
1711
- }
1712
- st->slab_ndim = sp;
1713
- st->outer_ndim = op;
1714
- st->composed_base = 0;
1715
- st->desc_ndim = 0; /* row-major STRIDE everywhere */
1716
-
1717
- ca_size_t total = 1;
1718
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
1719
- st->total_slabs = total;
1720
- st->slab_n = st->slab_elements;
1721
- st->slabs_emitted = 0;
1722
- st->chunk_size = st->slab_n;
1723
-
1724
- if ( st->outer_ndim > 0 ) {
1725
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
1726
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
1727
- } else {
1728
- st->outer_idx = NULL;
1729
- }
1337
+ /* The default layout: the view's own row-major strides, which is how
1338
+ a scratch the walk gathered is laid out and how the mask copy is
1339
+ laid out on every route that takes one. */
1340
+ for ( k = 0; k < src->ndim; k++ ) lay.dims[k] = src->dim[k];
1341
+ ki_view_row_major(src, lay.slab_strides, lay.mask_strides);
1342
+ memcpy(lay.outer_strides, lay.slab_strides, src->ndim * sizeof(ca_size_t));
1343
+
1344
+ switch ( plan->route ) {
1345
+
1346
+ case KI_L2_FIBER_FUSED:
1347
+ st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
1348
+ st->alias_ptr = NULL; /* no whole-view buffer */
1349
+ st->fiber_axis = axes[0];
1350
+ memcpy(st->fiber_native_strides, lay.slab_strides,
1351
+ src->ndim * sizeof(ca_size_t));
1352
+ break;
1730
1353
 
1731
- if ( ca_has_mask(src) ) {
1732
- if ( per_fiber_fused ) {
1733
- /* F.6.1 rev2 §3.3: mask travels with data per-fiber. Mask
1734
- buffer is delivered through fiber_mask_scratch by
1735
- next_slab_axes; no whole-view materialise. */
1736
- st->alias_mask = NULL;
1737
- } else {
1738
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1739
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1740
- if ( src->elements > 0 ) {
1741
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1742
- }
1743
- st->alias_mask = st->scratch_mask;
1744
- }
1354
+ case KI_L2_PER_SLAB: {
1355
+ /* One slab is gathered into a slab-sized scratch per outer index, so
1356
+ a slab axis steps through that scratch (packed row-major over the
1357
+ slab axes) and an outer axis moves nothing in it. The descriptors
1358
+ are kept for next_slab_axes to build each slab's subset. */
1359
+ CArray *parent = CAVIEW(src)->parent;
1360
+ ca_size_t b = src->bytes;
1361
+ ca_attach(parent); /* window: finish detaches st->root */
1362
+ st->root = parent;
1363
+ memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
1364
+ memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
1365
+ {
1366
+ ca_size_t s = src->bytes;
1367
+ for ( k = src->ndim - 1; k >= 0; k-- ) {
1368
+ st->pstrides[k] = s;
1369
+ s *= plan->pdims[k];
1745
1370
  }
1746
- return CA_ITER_OK;
1747
1371
  }
1748
- if ( src_kind != CA_ITER_SRC_CASTRIDE ) {
1749
- /* Any other unclassified kind. */
1750
- return CA_ITER_ERR_POLICY;
1372
+ st->desc_ndim = src->ndim;
1373
+ for ( k = src->ndim - 1; k >= 0; k-- ) {
1374
+ lay.dims[k] = plan->descs[k].count;
1375
+ lay.outer_strides[k] = 0;
1376
+ if ( plan->in_slab[k] ) {
1377
+ lay.slab_strides[k] = b;
1378
+ b *= plan->descs[k].count;
1379
+ }
1751
1380
  }
1381
+ break;
1752
1382
  }
1753
1383
 
1754
- /* F-2 minimal scope: SHIFT outer axes need OOB fill-slab support which
1755
- is not yet implemented; downgrade to materialise (= existing
1756
- SRC_DESCRIPTOR path) when present. Future work: allocate a 1-slab
1757
- fill scratch and yield it on OOB iterations (see rev6 §3.5 deferred). */
1758
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE
1759
- && ca_axis_dispatch_outer_has_shift(raw_descs, raw_ndim) ) {
1760
- src_kind = CA_ITER_SRC_DESCRIPTOR;
1761
- }
1762
-
1763
- /* Step 9 + 2026-05-31 refactor: L2 dispatch over SRC_ATTACH sources
1764
- (CAFake / CAByteSwap / CABitfield / CABitarray / CAReduce / CAObject
1765
- / CATile / CARoll). Same shape as L1 SRC_ATTACH: iterator-owns
1766
- scratch + xfer_all GET/PUT. Yield as one 1-D strided slab. */
1767
- if ( src_kind == CA_ITER_SRC_ATTACH ) {
1768
- st->src = src;
1769
- st->src_kind = CA_ITER_SRC_ATTACH;
1770
- st->level = 2;
1771
- st->policy = policy;
1772
- st->ndim = 1; /* logical 1-D L2 layout */
1773
- st->flags = flags;
1774
- st->bytes = src->bytes;
1775
- st->axes = axes;
1776
- st->naxes = naxes;
1777
-
1778
- st->scratch_cap = (ca_size_t) src->elements * src->bytes;
1779
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
1780
- if ( src->elements > 0 ) {
1781
- ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
1782
- }
1783
- st->alias_mode = CA_ITER_ALIAS_NONE;
1784
- st->alias_ptr = st->scratch_ptr;
1785
- st->composed_strides[0] = src->bytes;
1786
- st->composed_base = 0;
1787
- st->slab_n = src->elements;
1788
- st->total_slabs = 1;
1789
- st->slabs_emitted = 0;
1790
- st->chunk_size = st->slab_n;
1791
- st->outer_idx = NULL;
1792
-
1793
- if ( ca_has_mask(src) ) {
1794
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1795
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1796
- if ( src->elements > 0 ) {
1797
- ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
1798
- }
1799
- st->alias_mask = st->scratch_mask;
1384
+ case KI_L2_DESC_WHOLE: {
1385
+ /* The view gathered whole into a row-major scratch by the
1386
+ descriptor engine (which fills out-of-bounds cells), then walked
1387
+ with the default layout. */
1388
+ CArray *parent = CAVIEW(src)->parent;
1389
+ ca_axis_desc_t local_descs[CA_RANK_MAX];
1390
+ const void *bound_fill = NULL;
1391
+ ca_attach(parent); /* window: finish detaches st->root */
1392
+ st->root = parent;
1393
+ memcpy(local_descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
1394
+ if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
1395
+ bound_fill = ((CAWindow *) src)->fill;
1800
1396
  }
1801
- return CA_ITER_OK;
1397
+ st->scratch_cap = src->elements * src->bytes;
1398
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1399
+ ca_axis_dispatch_gather(parent, plan->pdims, local_descs, src->ndim,
1400
+ src->bytes, src->elements, bound_fill,
1401
+ st->scratch_ptr);
1402
+ st->total_elements = src->elements;
1403
+ st->alias_mode = CA_ITER_ALIAS_NONE;
1404
+ st->alias_ptr = st->scratch_ptr;
1405
+ break;
1802
1406
  }
1803
1407
 
1804
- /* F-2 (rev6, PROPOSAL_F2_KERNEL_ITERATOR_ALIAS): descriptor L2 alias
1805
- path. Eligibility (route_source verdict): innermost descriptor axis
1806
- is STRIDE kind, no outer SHIFT axis (downgraded above when present).
1807
- Setup: ca_attach(parent), alias_ptr = parent->ptr, no scratch alloc.
1808
- Each outer-prefix iteration yields a strided slab
1809
- slab_ptr = parent->ptr + inner_byte_start + outer_prefix_offset
1810
- slab_n = descs[ndim-1].count
1811
- slab_stride = descs[ndim-1].step * pstrides[ndim-1]
1812
- where outer_prefix_offset is computed via ca_axis_dispatch_prefix_offset
1813
- on the pre-classified prefix[]. next_slab_strided branches on
1814
- src_kind to dispatch to this offset formula. */
1815
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
1816
- int8_t nd = raw_ndim;
1817
- int8_t k;
1818
-
1819
- st->src = src;
1820
- st->src_kind = CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE;
1821
- st->level = 2;
1822
- st->policy = policy;
1823
- st->ndim = nd;
1824
- st->flags = flags;
1825
- st->bytes = src->bytes;
1826
- st->axes = axes;
1827
- st->naxes = naxes;
1828
-
1829
- /* Persist descs / parent_axis_dims in state; build parent row-major
1830
- pstrides so prefix_offset / inner offset share the same byte space. */
1831
- memcpy(st->descs, raw_descs, nd * sizeof(ca_axis_desc_t));
1832
- memcpy(st->parent_axis_dims, raw_pdims, nd * sizeof(ca_size_t));
1833
- st->desc_ndim = nd;
1408
+ case KI_L2_DESC_STRIDED: {
1409
+ /* The parent's memory is walked in place. Each slab axis is a
1410
+ STRIDE axis; an outer axis is STRIDE or INDEX (an outer SHIFT went
1411
+ to KI_L2_DESC_WHOLE). The start of every STRIDE axis is folded
1412
+ into composed_base, so a slab or STRIDE outer offset is step *
1413
+ index; an INDEX outer axis is looked up in its indices[] by
1414
+ next_slab_axes, which needs the descriptors kept. */
1415
+ CArray *parent = CAVIEW(src)->parent;
1416
+ ca_size_t base = 0;
1834
1417
  {
1835
1418
  ca_size_t s = src->bytes;
1836
- for ( k = nd - 1; k >= 0; k-- ) {
1419
+ for ( k = src->ndim - 1; k >= 0; k-- ) {
1837
1420
  st->pstrides[k] = s;
1838
- s *= raw_pdims[k];
1421
+ s *= plan->pdims[k];
1839
1422
  }
1840
1423
  }
1841
-
1842
- /* Inner slab parameters from innermost STRIDE descriptor axis. */
1843
- ca_size_t inner_start = raw_descs[nd - 1].start * st->pstrides[nd - 1];
1844
- ca_size_t inner_stride = raw_descs[nd - 1].step * st->pstrides[nd - 1];
1845
- ca_size_t inner_n = raw_descs[nd - 1].count;
1846
-
1847
- /* Outer prefix classify (skipped for 1-D source since no outer). */
1848
- if ( nd > 1 ) {
1849
- ca_axis_dispatch_classify_prefix(st->descs, st->pstrides, nd - 1, st->prefix);
1850
- }
1851
-
1852
- /* Total slabs = Π descs[0..nd-2].count (= src->elements / inner_n). */
1853
- ca_size_t outer_total = 1;
1854
- for ( k = 0; k < nd - 1; k++ ) outer_total *= raw_descs[k].count;
1855
- st->total_slabs = outer_total;
1856
- st->slab_n = inner_n;
1857
- st->slabs_emitted = 0;
1858
- st->chunk_size = inner_n;
1859
- st->total_elements = src->elements;
1860
-
1861
- /* composed_base holds inner_byte_start; composed_strides[nd-1] holds
1862
- inner stride so next_slab_strided's existing inner_st extraction
1863
- (= composed_strides[nd-1]) works without per-iter recomputation.
1864
- Outer slots of composed_strides are unused by the descriptor
1865
- branch in next_slab_strided (= prefix[] drives the offset). */
1866
- st->composed_base = inner_start;
1867
- st->composed_strides[nd - 1] = inner_stride;
1868
-
1869
- if ( nd > 1 ) {
1870
- CA_ASSUME(nd <= CA_RANK_MAX); /* with nd > 1: nd-1 in [1, CA_RANK_MAX-1] */
1871
- st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
1872
- for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
1873
- } else {
1874
- st->outer_idx = NULL;
1424
+ for ( k = 0; k < src->ndim; k++ ) {
1425
+ const ca_axis_desc_t *d = &plan->descs[k];
1426
+ lay.dims[k] = d->count;
1427
+ if ( plan->in_slab[k] ) {
1428
+ base += d->start * st->pstrides[k];
1429
+ lay.slab_strides[k] = d->step * st->pstrides[k];
1430
+ }
1431
+ else if ( d->kind == CA_AXIS_KIND_STRIDE ) {
1432
+ base += d->start * st->pstrides[k];
1433
+ lay.outer_strides[k] = d->step * st->pstrides[k];
1434
+ }
1435
+ else {
1436
+ lay.outer_strides[k] = st->pstrides[k];
1437
+ }
1875
1438
  }
1876
-
1877
- CArray *parent = CAVIEW(src)->parent;
1878
- ca_attach(parent);
1439
+ st->composed_base = base;
1440
+ memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
1441
+ memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
1442
+ st->desc_ndim = src->ndim;
1443
+ ca_attach(parent); /* window: finish detaches st->root */
1879
1444
  st->root = parent;
1880
1445
  st->alias_mode = CA_ITER_ALIAS_STRIDED;
1881
1446
  st->alias_ptr = (char *) parent->ptr;
1447
+ break;
1448
+ }
1882
1449
 
1883
- /* Mask: gather into scratch_mask (same as SRC_DESCRIPTOR path).
1884
- Data alias + mask materialise is per rev6 §4.4 deferred to F-4.c. */
1885
- if ( ca_has_mask(src) ) {
1886
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1887
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1888
- if ( src->elements > 0 ) {
1889
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1450
+ case KI_L2_STACK: {
1451
+ /* The K parents are attached and read directly; the slab (the K cells
1452
+ along the stack axis) is cut into a slab-sized scratch, refilled
1453
+ from a tile cache of TILE fibers that the parents fill in one pass.
1454
+ CAStack guarantees one shape across its parents, so parents[0]
1455
+ gives the native strides of all of them. */
1456
+ CAStack *stack = (CAStack *) src;
1457
+ int8_t parent_ndim = src->ndim - 1;
1458
+ ca_size_t K = (ca_size_t) stack->n_parents;
1459
+ ca_size_t bytes = (ca_size_t) src->bytes;
1460
+ ca_size_t tile;
1461
+ lay.slab_strides[axes[0]] = src->bytes;
1462
+ lay.mask_strides[axes[0]] = 1;
1463
+ ki_attach_stack_parents(st, stack);
1464
+ {
1465
+ ca_size_t s = src->bytes;
1466
+ for ( k = parent_ndim - 1; k >= 0; k-- ) {
1467
+ st->stack_parent_strides[k] = s;
1468
+ s *= stack->parents[0]->dim[k];
1890
1469
  }
1891
- st->alias_mask = st->scratch_mask;
1892
1470
  }
1893
- return CA_ITER_OK;
1471
+ st->scratch_cap = (ca_size_t) src->dim[axes[0]] * src->bytes;
1472
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1473
+ /* TILE budget = ~32 KB (half of M2 L1d), clamped to [8, 64] to keep
1474
+ the refill loop tight. */
1475
+ tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
1476
+ if ( tile < 8 ) tile = 8;
1477
+ if ( tile > 64 ) tile = 64;
1478
+ st->stack_tile_cap = tile;
1479
+ st->stack_tile_pos = 0;
1480
+ st->stack_tile_have = 0; /* force refill on first call */
1481
+ st->stack_tile_cache = (char *) ki_data_alloc(src->data_type, K * tile * bytes);
1482
+ st->alias_mode = CA_ITER_ALIAS_STACK;
1483
+ st->alias_ptr = NULL; /* no whole-view buffer */
1484
+ break;
1894
1485
  }
1895
1486
 
1896
- /* Sub-step 5.3: L2 dispatch over descriptor sources. The view-family
1897
- surface prioritises delivering the cells over avoiding a copy: always
1898
- materialise into a
1899
- scratch buffer via ca_axis_dispatch_attach and yield a single
1900
- strided slab (stride = bytes). CASelect/CAMapping always reach
1901
- here, CSA/CAGrid/CAWindow/CAShift when INDEX/SHIFT axes are
1902
- present. F-2 (rev6) routes innermost-STRIDE descriptor cases to
1903
- the L2 alias path above; this block now handles remaining mixed
1904
- cases (innermost INDEX, outer SHIFT temporarily, etc.). */
1905
- if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
1906
- /* raw_descs / raw_pdims / raw_ndim were populated by route_source
1907
- above so we skip the local describe_axes call. Kept locals
1908
- named raw_* to match the original code. */
1909
- st->src = src;
1910
- st->src_kind = CA_ITER_SRC_DESCRIPTOR;
1911
- st->level = 2;
1912
- st->policy = policy;
1913
- st->ndim = src->ndim;
1914
- st->flags = flags;
1915
- st->bytes = src->bytes;
1916
- st->axes = axes;
1917
- st->naxes = naxes;
1918
-
1919
- memcpy(st->parent_axis_dims, raw_pdims, raw_ndim * sizeof(ca_size_t));
1920
- ca_axis_dispatch_prepare(st->parent_axis_dims, raw_descs, raw_ndim,
1921
- st->bytes, st->descs, st->pstrides,
1922
- st->mdim, &st->desc_ndim);
1923
- ca_axis_dispatch_layout(st->descs, st->pstrides, st->mdim,
1924
- st->desc_ndim, st->bytes,
1925
- &st->slab_start, &st->slab_bytes_desc,
1926
- &st->slab_base);
1927
- if ( st->slab_start > 0 ) {
1928
- ca_axis_dispatch_classify_prefix(st->descs, st->pstrides,
1929
- st->slab_start, st->prefix);
1487
+ case KI_L2_STACK_OUTER_K: {
1488
+ /* Each slab is a region inside ONE parent, picked by
1489
+ outer_idx[stack_k_outer_pos]; next_slab_axes points into
1490
+ parents[k]->ptr + parent_off. A slab axis s of the stack is parent
1491
+ axis s, or s - 1 past the stack axis. */
1492
+ CAStack *stack = (CAStack *) src;
1493
+ int8_t k_axis = stack->k_axis;
1494
+ int8_t parent_ndim = src->ndim - 1;
1495
+ {
1496
+ ca_size_t sb = src->bytes, se = 1;
1497
+ for ( k = parent_ndim - 1; k >= 0; k-- ) {
1498
+ st->stack_parent_strides[k] = sb;
1499
+ st->stack_parent_mask_strides[k] = se;
1500
+ sb *= stack->parents[0]->dim[k];
1501
+ se *= stack->parents[0]->dim[k];
1502
+ }
1930
1503
  }
1931
- st->total_elements = src->elements;
1932
-
1933
- /* Materialise via the engine into a scratch buffer (contig
1934
- layout = single strided run, stride = bytes). */
1935
- CArray *parent = CAVIEW(src)->parent;
1936
- ca_attach(parent);
1937
- st->root = parent;
1938
- st->scratch_cap = src->elements * src->bytes;
1939
- const void *bound_fill = NULL;
1940
- if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
1941
- bound_fill = ((CAWindow *) src)->fill;
1504
+ st->stack_k_outer_pos = 0;
1505
+ for ( k = 0; k < src->ndim; k++ ) {
1506
+ if ( plan->in_slab[k] ) {
1507
+ int8_t parent_ax = (k < k_axis) ? k : (k - 1);
1508
+ lay.slab_strides[k] = st->stack_parent_strides[parent_ax];
1509
+ lay.mask_strides[k] = st->stack_parent_mask_strides[parent_ax];
1510
+ }
1511
+ else if ( k < k_axis ) {
1512
+ st->stack_k_outer_pos++;
1513
+ }
1942
1514
  }
1943
- st->scratch_ptr = ca_axis_dispatch_attach(parent,
1944
- st->parent_axis_dims,
1945
- raw_descs, raw_ndim,
1946
- src->bytes,
1947
- st->total_elements,
1948
- bound_fill);
1949
-
1950
- /* Lay out as a single 1-D L2 strided slab: ptr = scratch,
1951
- n = total_elements, stride = bytes. next_slab_strided reads
1952
- inner_stride from composed_strides[ndim - 1] and skips the
1953
- outer loop when outer_idx == NULL, so a 1-D logical layout
1954
- (st->ndim = 1, composed_strides[0] = bytes) yields exactly one
1955
- contig run. Note st->ndim diverges from src->ndim here — the
1956
- iterator's logical ndim is 1, but the view itself can be N-D
1957
- (the materialised buffer is flat). */
1958
- st->ndim = 1;
1959
- st->alias_mode = CA_ITER_ALIAS_NONE;
1960
- st->alias_ptr = st->scratch_ptr;
1961
- st->composed_strides[0] = src->bytes;
1962
- st->composed_base = 0;
1963
- st->slab_n = src->elements;
1964
- st->total_slabs = 1;
1965
- st->slabs_emitted = 0;
1966
- st->chunk_size = st->slab_n;
1967
- st->outer_idx = NULL;
1968
-
1969
- /* Step 6: mask gather, same as L1 descriptor branch. */
1515
+ ki_attach_stack_parents(st, stack);
1516
+ /* A masked stack: copy the K parent masks as they are at the start
1517
+ of the walk, and next_slab_axes points into the copy of parent k
1518
+ (promise 2). The stack's mask may still read stand-ins for
1519
+ parents that have gained a mask since; read through to those. */
1970
1520
  if ( ca_has_mask(src) ) {
1971
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
1972
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
1973
- if ( src->elements > 0 ) {
1974
- ca_copy_data(src->mask, (char *) st->scratch_mask);
1521
+ ca_multi_parent_mask_refresh(src->mask, stack->parents,
1522
+ stack->mask_standins,
1523
+ stack->n_parents);
1524
+ /* Zero-filled: a slot is set once its copy is made, and finish
1525
+ frees the copies whose slot is set. */
1526
+ st->stack_parent_mask_ptrs =
1527
+ (boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
1528
+ memset(st->stack_parent_mask_ptrs, 0,
1529
+ stack->n_parents * sizeof(boolean8_t *));
1530
+ for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
1531
+ CArray *pm = ((CAStack *) src->mask)->parents[kk];
1532
+ ca_size_t n = pm->elements;
1533
+ boolean8_t *copy = (boolean8_t *) xmalloc(n > 0 ? n : 1);
1534
+ st->stack_parent_mask_ptrs[kk] = copy;
1535
+ ca_attach(pm); /* window: only a memcpy before the detach */
1536
+ memcpy(copy, pm->ptr, n);
1537
+ ca_detach(pm);
1975
1538
  }
1976
- st->alias_mask = st->scratch_mask;
1977
1539
  }
1978
- return CA_ITER_OK;
1540
+ st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
1541
+ st->alias_ptr = NULL;
1542
+ break;
1979
1543
  }
1980
1544
 
1981
- /* === CAStride / entity L2 path === */
1982
- st->src = src;
1983
- st->src_kind = CA_ITER_SRC_CASTRIDE;
1984
- st->level = 2;
1985
- st->policy = policy;
1986
- st->ndim = src->ndim;
1987
- st->flags = flags;
1988
- st->bytes = src->bytes;
1989
- st->axes = axes;
1990
- st->naxes = naxes;
1991
-
1992
- /* Compute composed strides + base. For entity / CAStride contig we
1993
- synthesise row-major byte strides so next_slab_strided's offset
1994
- math is uniform across alias modes; for CAStride non-contig we
1995
- compose leaf strides up to the root entity via the substrate. */
1996
- int8_t nd = src->ndim;
1997
- int use_strided = 0;
1998
- int use_view_scratch = 0;
1999
- CArray *root = NULL;
1545
+ case KI_L2_TRANSFER:
1546
+ ki_gather_view(st, src, flags, 0);
1547
+ st->alias_mode = CA_ITER_ALIAS_NONE;
1548
+ st->alias_ptr = st->scratch_ptr;
1549
+ break;
2000
1550
 
2001
- if ( ca_iter_can_alias(src, 1) ) {
2002
- /* Entity or CAStride contig: stride = row-major bytes, base = 0. */
1551
+ case KI_L2_ROOT_STRIDED:
1552
+ memcpy(st->composed_strides, plan->composed, src->ndim * sizeof(ca_size_t));
1553
+ st->composed_base = plan->base;
1554
+ memcpy(lay.slab_strides, plan->composed, src->ndim * sizeof(ca_size_t));
1555
+ memcpy(lay.outer_strides, plan->composed, src->ndim * sizeof(ca_size_t));
1556
+ ca_attach(plan->root); /* window: finish detaches st->root */
1557
+ st->root = plan->root; /* recorded once attached */
1558
+ st->alias_mode = CA_ITER_ALIAS_STRIDED;
1559
+ st->alias_ptr = (char *) plan->root->ptr;
1560
+ break;
1561
+
1562
+ case KI_L2_VIEW_SCRATCH:
2003
1563
  ca_iter_build_rowmajor_strides(st->composed_strides,
2004
- src->dim, nd, src->bytes);
2005
- st->composed_base = 0;
2006
- } else {
2007
- /* CAStride non-contig: leaf->root compose. */
2008
- ca_size_t cs[CA_RANK_MAX];
2009
- ca_size_t base;
2010
- ca_stride_compose_to_root((CAStride *) src, &root, cs, &base);
2011
- if ( !ca_root_lends_no_memory(root) ) {
2012
- memcpy(st->composed_strides, cs, nd * sizeof(ca_size_t));
2013
- st->composed_base = base;
2014
- use_strided = 1;
2015
- } else {
2016
- /* Root holds nothing to read through: reaching root->ptr costs one
2017
- whole-root materialise no matter how few cells the view touches —
2018
- the same cliff the xfer path avoids by asking for a region. Take
2019
- the view's own region protocol instead: ca_copy_data walks the
2020
- leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
2021
- CAObject asks for exactly that block. The gathered buffer is view
2022
- row-major, so the slab arithmetic below is the contig-alias case
2023
- unchanged. */
2024
- root = NULL;
2025
- ca_iter_build_rowmajor_strides(st->composed_strides,
2026
- src->dim, nd, src->bytes);
2027
- st->composed_base = 0;
2028
- use_view_scratch = 1;
1564
+ src->dim, src->ndim, src->bytes);
1565
+ ki_gather_view(st, src, flags, 1);
1566
+ st->alias_mode = CA_ITER_ALIAS_NONE;
1567
+ st->alias_ptr = st->scratch_ptr;
1568
+ break;
1569
+
1570
+ case KI_L2_CONTIG:
1571
+ ca_iter_build_rowmajor_strides(st->composed_strides,
1572
+ src->dim, src->ndim, src->bytes);
1573
+ ca_attach(src);
1574
+ st->alias_mode = CA_ITER_ALIAS_CONTIG;
1575
+ st->alias_ptr = (char *) src->ptr;
1576
+ break;
1577
+ }
1578
+
1579
+ ki_lay_out_slabs(st, src, plan->in_slab, &lay);
1580
+
1581
+ if ( plan->route == KI_L2_PER_SLAB ) {
1582
+ /* One max-slab buffer refilled per outer index, sized now that the
1583
+ slab is known.
1584
+
1585
+ Shortcut PER_SLAB_HOIST.
1586
+ Layer data; the fiber layer sits on top.
1587
+ Speeds up per-slab gathers whose innermost slab axis is STRIDE
1588
+ and with no SHIFT axis in the view: the outer and
1589
+ non-innermost slab offsets are hoisted and the inner
1590
+ run is copied with memcpy.
1591
+ Why no difference measured on a row-selected 2-D read.
1592
+ Enters at least one slab axis, the innermost slab axis
1593
+ STRIDE, no SHIFT axis.
1594
+ Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
1595
+ only when the view has one).
1596
+ Switch ca_fast_paths_enabled(): off, the PER_SLAB gather.
1597
+ Checked by the two-slab-axes rows of the grid parent in the
1598
+ delivery matrix. */
1599
+ ca_size_t slab_bytes = st->slab_elements * src->bytes;
1600
+ int hoist = (st->slab_ndim > 0) && ca_fast_paths_enabled()
1601
+ && plan->descs[st->slab_axes_buf[st->slab_ndim - 1]].kind
1602
+ == CA_AXIS_KIND_STRIDE;
1603
+ for ( k = 0; hoist && k < src->ndim; k++ ) {
1604
+ if ( plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) hoist = 0;
2029
1605
  }
1606
+ st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
1607
+ st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
1608
+ st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
1609
+ st->alias_mode = hoist ? CA_ITER_ALIAS_PER_SLAB_HOIST
1610
+ : CA_ITER_ALIAS_PER_SLAB;
2030
1611
  }
2031
1612
 
2032
- /* Phase A: CA_SLAB_AXES branch. Partition axes into slab vs outer,
2033
- populate per-axis dims / strides (both data byte strides from
2034
- composed_strides[] and mask element strides from view row-major
2035
- dim products), allocate outer_idx if outer_ndim > 0, attach data
2036
- buffer (alias or strided), gather mask if present.
1613
+ /* The mask cursor (promise 2): fiber by fiber through fiber_mask_scratch
1614
+ on PER_FIBER_FUSED, the parents' copies on STACK_OUTER_K, none on
1615
+ STACK (an unmasked stack), and the walk's copy of the whole view's
1616
+ mask everywhere else. */
1617
+ if ( plan->route == KI_L2_FIBER_FUSED || plan->route == KI_L2_STACK
1618
+ || plan->route == KI_L2_STACK_OUTER_K ) {
1619
+ st->alias_mask = NULL;
1620
+ }
1621
+ else {
1622
+ ki_take_mask_copy(st, src);
1623
+ }
1624
+ }
2037
1625
 
2038
- Layout invariant: slab_axes_buf and outer_axes are both stored
2039
- sort-ascending (in source-axis order), so multi-axis CA_SLAB_AXES
2040
- is canonical regardless of the user's input order. */
2041
- if ( policy == CA_SLAB_AXES ) {
2042
- /* Validate axes input. */
2043
- if ( axes == NULL || naxes <= 0 || naxes > nd ) {
2044
- return CA_ITER_ERR_POLICY;
2045
- }
2046
- int8_t in_slab[CA_RANK_MAX];
2047
- int8_t k;
2048
- for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
2049
- for ( k = 0; k < naxes; k++ ) {
2050
- int8_t ax = axes[k];
2051
- if ( ax < 0 || ax >= nd ) return CA_ITER_ERR_POLICY;
2052
- if ( in_slab[ax] ) return CA_ITER_ERR_POLICY; /* duplicate */
2053
- in_slab[ax] = 1;
2054
- }
1626
+ static int
1627
+ ca_iter_state_init_l2_unprotected (ca_iter_state *st,
1628
+ struct _CArray *src,
1629
+ ca_slab_policy_t policy,
1630
+ int8_t *axes,
1631
+ int8_t naxes,
1632
+ uint32_t flags)
1633
+ {
1634
+ ki_l2_plan plan;
1635
+ int rc;
2055
1636
 
2056
- /* View row-major element strides (= mask scratch layout strides). */
2057
- ca_size_t row_elem_strides[CA_RANK_MAX];
2058
- {
2059
- ca_size_t s = 1;
2060
- for ( k = nd - 1; k >= 0; k-- ) {
2061
- row_elem_strides[k] = s;
2062
- s *= src->dim[k];
2063
- }
2064
- }
1637
+ /* Leave the state defined before anything can return. Every error
1638
+ return below hands control back to a caller that may not look at the
1639
+ code, so what it is left holding has to be a state that iterates zero
1640
+ times rather than whatever its stack frame happened to contain. */
1641
+ if ( st == NULL ) return CA_ITER_ERR_FLAGS;
1642
+ memset(st, 0, sizeof(*st));
1643
+ if ( src == NULL ) return CA_ITER_ERR_FLAGS;
2065
1644
 
2066
- /* Partition axes (ascending order). */
2067
- int8_t sp = 0, op = 0;
2068
- st->slab_elements = 1;
2069
- for ( k = 0; k < nd; k++ ) {
2070
- if ( in_slab[k] ) {
2071
- st->slab_axes_buf[sp] = k;
2072
- st->slab_dims[sp] = src->dim[k];
2073
- st->slab_strides[sp] = st->composed_strides[k];
2074
- st->slab_mask_strides[sp] = row_elem_strides[k];
2075
- st->slab_elements *= src->dim[k];
2076
- sp++;
2077
- } else {
2078
- st->outer_axes[op] = k;
2079
- st->outer_dims[op] = src->dim[k];
2080
- st->outer_strides[op] = st->composed_strides[k];
2081
- st->outer_mask_strides[op] = row_elem_strides[k];
2082
- op++;
2083
- }
2084
- }
2085
- st->slab_ndim = sp;
2086
- st->outer_ndim = op;
2087
-
2088
- /* total_slabs = Π outer_dims. Empty product (all-axes case) = 1
2089
- → single slab = whole array (D1.4 WHOLE-equivalent). */
2090
- ca_size_t total = 1;
2091
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
2092
- st->total_slabs = total;
2093
- st->slab_n = st->slab_elements; /* mirror to legacy field */
2094
- st->slabs_emitted = 0;
2095
- st->chunk_size = st->slab_n;
2096
-
2097
- if ( st->outer_ndim > 0 ) {
2098
- st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
2099
- for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
2100
- } else {
2101
- st->outer_idx = NULL;
2102
- }
1645
+ /* A Face only layers a semantic identifier and a CALazyMarker only
1646
+ layers "leaf of a lazy chain"; storage is identical to the parent
1647
+ either way, so strip at the kernel-compute entry and descend.
1648
+ Routing and alias decisions belong to the parent. Strip before
1649
+ validate_inputs: left on, both would classify as SRC_ATTACH (they
1650
+ have attach slots) and be materialised whole (see
1651
+ ca_iter_strip_storage_wrapper).
2103
1652
 
2104
- if ( use_strided ) {
2105
- st->root = root;
2106
- ca_attach(root);
2107
- st->alias_mode = CA_ITER_ALIAS_STRIDED;
2108
- st->alias_ptr = (char *) root->ptr;
2109
- } else if ( use_view_scratch ) {
2110
- st->root = NULL;
2111
- st->scratch_cap = (ca_size_t) src->elements * src->bytes;
2112
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0
2113
- ? st->scratch_cap : 1);
2114
- if ( src->elements > 0 ) {
2115
- ca_copy_data(src, st->scratch_ptr);
2116
- }
2117
- st->alias_mode = CA_ITER_ALIAS_NONE;
2118
- st->alias_ptr = st->scratch_ptr;
2119
- } else {
2120
- st->root = NULL;
2121
- ca_attach(src);
2122
- st->alias_mode = CA_ITER_ALIAS_CONTIG;
2123
- st->alias_ptr = (char *) src->ptr;
2124
- }
1653
+ The Face identifier is re-wrapped onto the result by the caller's lift
1654
+ hook (= primary operators / reductions / etc.). The marker is not:
1655
+ a reduction over a lazy marker yields a plain entity.
2125
1656
 
2126
- /* Mask: gather to scratch_mask in view row-major order (= same as
2127
- L1 path). The mask layout matches view->dim row-major, which is
2128
- what slab_mask_strides / outer_mask_strides walk. */
2129
- if ( ca_has_mask(src) ) {
2130
- ca_size_t mcap = src->elements > 0 ? src->elements : 1;
2131
- st->scratch_mask = (boolean8_t *) xmalloc(mcap);
2132
- if ( src->elements > 0 ) {
2133
- ca_copy_data(src->mask, (char *) st->scratch_mask);
2134
- }
2135
- st->alias_mask = st->scratch_mask;
2136
- }
2137
- return CA_ITER_OK;
1657
+ WRITE is rejected against the wrapper, before the strip — a marker is
1658
+ read-only while its parent is not. validate_inputs re-checks the
1659
+ stripped source, which is harmless. */
1660
+ if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
1661
+ return CA_ITER_ERR_READONLY;
2138
1662
  }
2139
-
2140
- /* Outer prefix axes [0..ndim-2] drive total_slabs; innermost axis is
2141
- the slab. 0-d / 1-d sources collapse to a single yield. */
2142
- if ( nd <= 1 ) {
2143
- st->total_slabs = 1;
2144
- st->slab_n = src->elements;
2145
- st->outer_idx = NULL;
2146
- } else {
2147
- ca_size_t total = 1;
2148
- int8_t k;
2149
- for ( k = 0; k < nd - 1; k++ ) total *= src->dim[k];
2150
- st->total_slabs = total;
2151
- st->slab_n = src->dim[nd - 1];
2152
- st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
2153
- for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
1663
+ if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
1664
+ src = ca_iter_strip_storage_wrapper(src);
2154
1665
  }
2155
- st->slabs_emitted = 0;
2156
- st->chunk_size = st->slab_n;
2157
1666
 
2158
- if ( use_strided ) {
2159
- st->root = root;
2160
- ca_attach(root);
2161
- st->alias_mode = CA_ITER_ALIAS_STRIDED;
2162
- st->alias_ptr = (char *) root->ptr;
2163
- } else if ( use_view_scratch ) {
2164
- st->root = NULL;
2165
- st->scratch_cap = (ca_size_t) src->elements * src->bytes;
2166
- st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0
2167
- ? st->scratch_cap : 1);
2168
- if ( src->elements > 0 ) {
2169
- ca_copy_data(src, st->scratch_ptr);
2170
- }
2171
- st->alias_mode = CA_ITER_ALIAS_NONE;
2172
- st->alias_ptr = st->scratch_ptr;
2173
- } else {
2174
- st->root = NULL;
2175
- ca_attach(src);
2176
- st->alias_mode = CA_ITER_ALIAS_CONTIG;
2177
- st->alias_ptr = (char *) src->ptr;
1667
+ rc = ca_iter_validate_inputs(st, src, policy, flags);
1668
+ if ( rc != CA_ITER_OK ) return rc;
1669
+ /* L2 walks a slab of named axes. The whole-array walk is L1
1670
+ (init_l1 with CA_SLAB_WHOLE); one strided run per innermost row is
1671
+ CA_SLAB_AXES with the last axis. */
1672
+ if ( policy != CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
1673
+
1674
+ /* The slab axes: in range, no repeats. */
1675
+ if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
1676
+ return CA_ITER_ERR_POLICY;
2178
1677
  }
1678
+ memset(plan.in_slab, 0, sizeof(plan.in_slab));
1679
+ for ( int8_t k = 0; k < naxes; k++ ) {
1680
+ int8_t ax = axes[k];
1681
+ if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
1682
+ if ( plan.in_slab[ax] ) return CA_ITER_ERR_POLICY;
1683
+ plan.in_slab[ax] = 1;
1684
+ }
1685
+
1686
+ /* route_source returns the refined src_kind and, for descriptor
1687
+ sources, the descriptors, so the choice below does not describe the
1688
+ axes again. */
1689
+ plan.ndesc = 0;
1690
+ plan.root = NULL;
1691
+ plan.base = 0;
1692
+ plan.src_kind = ca_iter_route_source(src, plan.descs, plan.pdims,
1693
+ &plan.ndesc);
1694
+ rc = ki_choose_l2_route(src, axes, naxes, flags, &plan);
1695
+ if ( rc != CA_ITER_OK ) return rc;
1696
+
1697
+ ki_open_l2(st, src, policy, axes, naxes, flags, &plan);
2179
1698
  return CA_ITER_OK;
2180
1699
  }
2181
1700
 
@@ -2202,73 +1721,54 @@ ca_iter_state_next_slab_unprotected (ca_iter_state *st,
2202
1721
  return 1;
2203
1722
  }
2204
1723
 
1724
+
1725
+ /* CA_KERNEL_FIBER_CONTIG: a fiber (a walk with one slab axis) reaches the
1726
+ kernel as a contiguous run of cells and of mask flags. Each path of
1727
+ next_slab_axes finds the fiber's first cell and the step along it, and
1728
+ hands both through these two functions, which copy a fiber with any
1729
+ other step into the walk's fiber scratch. Other walks get the pointer
1730
+ back unchanged. */
2205
1731
  static int
2206
- ca_iter_state_next_slab_strided_unprotected (ca_iter_state *st,
2207
- char **out_ptr,
2208
- boolean8_t **out_mask,
2209
- ca_size_t *out_n,
2210
- ca_size_t *out_stride_bytes)
1732
+ ki_fiber_contig_wanted (const ca_iter_state *st)
2211
1733
  {
2212
- if ( st == NULL || st->level != 2
2213
- || st->slabs_emitted >= st->total_slabs ) {
2214
- if ( out_ptr ) *out_ptr = NULL;
2215
- if ( out_mask ) *out_mask = NULL;
2216
- if ( out_n ) *out_n = 0;
2217
- if ( out_stride_bytes ) *out_stride_bytes = 0;
2218
- return 0;
2219
- }
1734
+ return (st->flags & CA_KERNEL_FIBER_CONTIG)
1735
+ && st->naxes == 1 && st->slab_ndim == 1;
1736
+ }
2220
1737
 
2221
- int8_t nd = st->ndim;
2222
- ca_size_t inner_st = (nd > 0)
2223
- ? st->composed_strides[nd - 1]
2224
- : st->bytes;
2225
- ca_size_t off = st->composed_base;
2226
- int8_t k;
1738
+ static char *
1739
+ ki_fiber_contig_data (ca_iter_state *st, char *data, ca_size_t step)
1740
+ {
1741
+ ca_size_t n = st->slab_dims[0];
1742
+ ca_size_t bytes = st->bytes;
1743
+ ca_size_t need = n * bytes;
1744
+ if ( ! ki_fiber_contig_wanted(st) || step == bytes ) {
1745
+ return data;
1746
+ }
1747
+ if ( st->fiber_data_scratch_cap < need ) {
1748
+ if ( st->fiber_data_scratch ) ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
1749
+ st->fiber_data_scratch = (char *) ki_data_alloc(st->src->data_type, need);
1750
+ st->fiber_data_scratch_cap = need;
1751
+ }
1752
+ ca_stride_gather_run(st->fiber_data_scratch, data, bytes, n, step);
1753
+ return st->fiber_data_scratch;
1754
+ }
2227
1755
 
2228
- /* Sum outer prefix offset. For F-2 descriptor L2 alias the prefix can
2229
- contain INDEX axes (and, once OOB fill_slab lands, SHIFT axes), so
2230
- we delegate to the pre-classified prefix[] engine. Otherwise (=
2231
- CAStride / entity outer = STRIDE-only by construction) use the
2232
- direct sum that has been the L2 inner loop since step 3. For
2233
- 0-d / 1-d sources, the loop / call is a no-op and off stays at
2234
- composed_base. */
2235
- if ( st->outer_idx != NULL ) {
2236
- if ( st->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
2237
- int oob = 0;
2238
- off += ca_axis_dispatch_prefix_offset(st->prefix, st->outer_idx,
2239
- nd - 1, &oob);
2240
- /* oob unreachable: outer SHIFT was downgraded in init_l2. */
2241
- } else {
2242
- for ( k = 0; k < nd - 1; k++ ) {
2243
- off += st->outer_idx[k] * st->composed_strides[k];
2244
- }
2245
- }
1756
+ static boolean8_t *
1757
+ ki_fiber_contig_mask (ca_iter_state *st, boolean8_t *mask, ca_size_t step)
1758
+ {
1759
+ ca_size_t n = st->slab_dims[0];
1760
+ if ( mask == NULL || ! ki_fiber_contig_wanted(st) || step == 1 ) {
1761
+ return mask;
2246
1762
  }
2247
-
2248
- if ( out_ptr ) *out_ptr = st->alias_ptr + off;
2249
- /* mask layout mirrors value layout, but mask is contig boolean8_t
2250
- when alias_mask is a scratch buffer (step 6 baseline = always
2251
- gather mask into scratch_mask for uniformity). Per-slab mask
2252
- offset = i in the outer cursor (= slabs_emitted at this point). */
2253
- if ( out_mask ) {
2254
- *out_mask = st->alias_mask
2255
- ? st->alias_mask + st->slabs_emitted * st->slab_n
2256
- : NULL;
1763
+ if ( st->fiber_mask_scratch_cap < n ) {
1764
+ if ( st->fiber_mask_scratch ) xfree(st->fiber_mask_scratch);
1765
+ st->fiber_mask_scratch = (boolean8_t *) xmalloc(n > 0 ? n : 1);
1766
+ st->fiber_mask_scratch_cap = n;
2257
1767
  }
2258
- if ( out_n ) *out_n = st->slab_n;
2259
- if ( out_stride_bytes ) *out_stride_bytes = inner_st;
2260
-
2261
- st->slabs_emitted += 1;
2262
-
2263
- /* Advance outer_idx row-major (least-significant axis innermost,
2264
- so we tick outer_idx[ndim-2] first). */
2265
- if ( st->outer_idx != NULL ) {
2266
- for ( k = nd - 2; k >= 0; k-- ) {
2267
- if ( ++st->outer_idx[k] < st->src->dim[k] ) break;
2268
- st->outer_idx[k] = 0;
2269
- }
1768
+ for ( ca_size_t i = 0; i < n; i++ ) {
1769
+ st->fiber_mask_scratch[i] = mask[i * step];
2270
1770
  }
2271
- return 1;
1771
+ return st->fiber_mask_scratch;
2272
1772
  }
2273
1773
 
2274
1774
  static int
@@ -2283,12 +1783,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2283
1783
  return 0;
2284
1784
  }
2285
1785
 
2286
- /* Phase C T3 specialised path (B-1b, C.1b, 2026-05-27): innermost
2287
- slab axis is STRIDE, no SHIFT axes anywhere. Hoist outer + non-
2288
- innermost-slab axes (= per-cell switch evaluated once per slab row),
2289
- inner = pure STRIDE linear memcpy (= SIMD-friendly contig run, no
2290
- engine per-cell dispatch). Target: 1.5-1.8x win for INDEX slab
2291
- with innermost STRIDE (= grid / select sparse projection use cases). */
1786
+ /* PER_SLAB_HOIST (see its name tag in ki_open_l2): the innermost slab
1787
+ axis is STRIDE and no axis is SHIFT. The outer and non-innermost
1788
+ slab offsets are computed once per slab row, and the inner run is a
1789
+ plain memcpy instead of the descriptor engine's per-cell walk. */
2292
1790
  if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB_HOIST ) {
2293
1791
  int8_t inner_view_ax = st->slab_axes_buf[st->slab_ndim - 1];
2294
1792
  ca_size_t inner_count = st->descs[inner_view_ax].count;
@@ -2370,7 +1868,11 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2370
1868
  }
2371
1869
 
2372
1870
  if ( out_ptr ) *out_ptr = st->scratch_ptr;
2373
- if ( out_mask ) *out_mask = st->scratch_mask ? st->scratch_mask + mask_off : NULL;
1871
+ if ( out_mask ) {
1872
+ *out_mask = ki_fiber_contig_mask(st,
1873
+ st->scratch_mask ? st->scratch_mask + mask_off : NULL,
1874
+ st->slab_mask_strides[0]);
1875
+ }
2374
1876
 
2375
1877
  st->slabs_emitted += 1;
2376
1878
  for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
@@ -2380,17 +1882,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2380
1882
  return 1;
2381
1883
  }
2382
1884
 
2383
- /* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18): CAStack
2384
- source + K-axis (k_axis) in outer iter. Each slab aliases a region
2385
- inside parents[k]->ptr where k = outer_idx[K_outer_pos]. Parent
2386
- inner byte offset = Σ outer_idx[m] * stack_parent_strides[parent_ax]
2387
- over all outer axes except the K-axis itself. Mask: parallel alias
2388
- into parents[k]->mask->ptr + mask_off. Zero copy / zero scratch /
2389
- parent entity bandwidth.
2390
-
2391
- K.3 (2026-06-20): parent_ax derivation generalised for arbitrary
2392
- k_axis -- stack axis s != k_axis maps to parent axis s if
2393
- s < k_axis else s - 1. */
1885
+ /* CAStack source with the stack axis k_axis in the outer iteration.
1886
+ Each slab is a region inside parents[k]->ptr where k =
1887
+ outer_idx[K_outer_pos]. Parent inner byte offset = Σ outer_idx[m] *
1888
+ stack_parent_strides[parent_ax] over all outer axes except the K-axis
1889
+ itself. Mask: a pointer into the copy of parent k's mask that init
1890
+ took (stack_parent_mask_ptrs[k] + mask_off); a write through it
1891
+ reaches nothing. The data is yielded in place unless FIBER_CONTIG
1892
+ needs a strided fiber gathered (ki_fiber_contig_data). A stack axis
1893
+ s other than k_axis is parent axis s, or s - 1 past k_axis. */
2394
1894
  if ( st->alias_mode == CA_ITER_ALIAS_STACK_OUTER_K ) {
2395
1895
  int8_t kpos = st->stack_k_outer_pos;
2396
1896
  int32_t k = (int32_t) st->outer_idx[kpos];
@@ -2406,11 +1906,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2406
1906
  mask_off += st->outer_idx[m] * st->stack_parent_mask_strides[parent_ax];
2407
1907
  }
2408
1908
 
2409
- if ( out_ptr ) *out_ptr = st->stack_parent_ptrs[k] + parent_off;
1909
+ if ( out_ptr ) {
1910
+ *out_ptr = ki_fiber_contig_data(st, st->stack_parent_ptrs[k] + parent_off,
1911
+ st->slab_strides[0]);
1912
+ }
2410
1913
  if ( out_mask ) {
2411
- *out_mask = st->stack_parent_mask_ptrs
2412
- ? st->stack_parent_mask_ptrs[k] + mask_off
2413
- : NULL;
1914
+ *out_mask = ki_fiber_contig_mask(st,
1915
+ st->stack_parent_mask_ptrs
1916
+ ? st->stack_parent_mask_ptrs[k] + mask_off : NULL,
1917
+ st->slab_mask_strides[0]);
2414
1918
  }
2415
1919
 
2416
1920
  st->slabs_emitted += 1;
@@ -2421,20 +1925,19 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2421
1925
  return 1;
2422
1926
  }
2423
1927
 
2424
- /* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2: CAStack direct
2425
- per-parent ptr access path. For each outer iter, compute the
1928
+ /* CAStack read along its stack axis, from the parents' memory. For
1929
+ each outer index, compute the
2426
1930
  parent inner byte offset (= same across all K parents, uniform
2427
- shape), then K-fold direct memcpy gather from cached parent ptrs
2428
- into scratch. No ca_xfer_stride dispatch; per-cell cost = pointer
2429
- arith + memcpy(bytes). For f64 the memcpy(8) compiles to a single
2430
- mov, so the inner loop is tight.
2431
-
2432
- Note: scope-narrowed to slab_axes == [0] at init, so outer_axes are
2433
- stack axes 1..N-1 = parent axes 0..parent_ndim-1. outer_idx[m]
2434
- maps directly to parent axis (m) (since the m-th outer axis is
2435
- stack axis m+1 = parent axis m). */
1931
+ shape), and yield the K cells of that fiber out of stack_tile_cache,
1932
+ which is refilled TILE fibers at a time from the cached parent ptrs
1933
+ (below). No ca_xfer_stride dispatch. The scratch_ptr init
1934
+ allocates is not used by this path.
1935
+
1936
+ init engages this path only for slab_axes == [k_axis], so the outer
1937
+ axes are the stack axes other than k_axis, which are the parent axes
1938
+ in order: outer_idx[m] maps to parent axis m. */
2436
1939
  if ( st->alias_mode == CA_ITER_ALIAS_STACK ) {
2437
- /* pilot/castack-axis0-loop-interchange: tile cache. Outer iter
1940
+ /* Tile cache. Outer iter
2438
1941
  walks parent storage in row-major order so parent_off increments
2439
1942
  by `bytes` per call monotonically. Refill TILE fibers at once
2440
1943
  via K contig parent streams (= TILE consecutive cells from each
@@ -2504,11 +2007,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2504
2007
  return 1;
2505
2008
  }
2506
2009
 
2507
- /* Phase C T3 fallback (C.1, 2026-05-27): per-slab materialise via
2508
- ca_axis_dispatch_gather with a subset descriptor built by pinning
2509
- outer axes at the current outer_idx position (D1.2 (A): caller-side
2510
- subset construction, engine API unchanged). scratch_ptr is sized
2511
- for one max slab and reused across iters (D1.1 (B)). */
2010
+ /* PER_SLAB: one slab gathered by ca_axis_dispatch_gather, from a
2011
+ subset of the descriptors with the outer axes pinned at the current
2012
+ outer_idx position. scratch_ptr is sized for one slab and reused for
2013
+ every slab. */
2512
2014
  if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB ) {
2513
2015
  ca_axis_desc_t subset_descs[CA_RANK_MAX];
2514
2016
  memcpy(subset_descs, st->descs, st->ndim * sizeof(ca_axis_desc_t));
@@ -2543,7 +2045,7 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2543
2045
  }
2544
2046
  }
2545
2047
 
2546
- /* CAWindow fill value capture (= same lookup as B.1.5 init). */
2048
+ /* CAWindow fill value (the same lookup as in ki_open_l2). */
2547
2049
  const void *bound_fill = NULL;
2548
2050
  if ( ca_func[st->src->obj_type].attach == ca_window_func.attach ) {
2549
2051
  bound_fill = ((CAWindow *) st->src)->fill;
@@ -2553,15 +2055,19 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2553
2055
  st->ndim, st->bytes, st->slab_elements,
2554
2056
  bound_fill, st->scratch_ptr);
2555
2057
 
2556
- /* Mask offset into whole-view scratch_mask (T3 path keeps mask
2557
- layout view-row-major to match slab_mask_strides). */
2058
+ /* Mask offset into the whole-view mask copy, which is view
2059
+ row-major to match slab_mask_strides. */
2558
2060
  ca_size_t mask_off = 0;
2559
2061
  for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
2560
2062
  mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
2561
2063
  }
2562
2064
 
2563
2065
  if ( out_ptr ) *out_ptr = st->scratch_ptr;
2564
- if ( out_mask ) *out_mask = st->scratch_mask ? st->scratch_mask + mask_off : NULL;
2066
+ if ( out_mask ) {
2067
+ *out_mask = ki_fiber_contig_mask(st,
2068
+ st->scratch_mask ? st->scratch_mask + mask_off : NULL,
2069
+ st->slab_mask_strides[0]);
2070
+ }
2565
2071
 
2566
2072
  st->slabs_emitted += 1;
2567
2073
  /* Advance outer_idx row-major (innermost outer axis ticks first). */
@@ -2577,10 +2083,9 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2577
2083
  (element units, view row-major). Both are zero when outer_ndim
2578
2084
  == 0 (= all-axes WHOLE-equivalent case).
2579
2085
 
2580
- STRIDE outer axes (Phase A SRC_CASTRIDE, and Phase B
2581
- SRC_DESCRIPTOR STRIDE-kind axes) use outer_idx[m] directly as the
2582
- multiplier. INDEX outer axes (Phase B SRC_DESCRIPTOR INDEX-kind)
2583
- need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
2086
+ STRIDE outer axes (SRC_CASTRIDE, and the STRIDE axes of a descriptor
2087
+ view) use outer_idx[m] directly as the multiplier. INDEX outer axes
2088
+ of a descriptor view need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
2584
2089
  Mask offset uses outer_idx[m] directly in both cases (= mask
2585
2090
  scratch is in view row-major order, outer_idx walks view-axis
2586
2091
  positions). */
@@ -2600,13 +2105,12 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2600
2105
  mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
2601
2106
  }
2602
2107
 
2603
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1: per-fiber fused dispatch.
2604
- When init_l2 selected CA_ITER_ALIAS_PER_FIBER_FUSED, there is no
2605
- whole-view buffer to alias from. Build the fiber region from
2606
- outer_idx + fiber_axis and call ca_xfer_stride(src, ..., GET)
2607
- directly into fiber_data_scratch. For fused-aware views (X.1
2608
- OOB-fused / X.4 transform-fused) this routes to a 1-pass per-
2609
- region path.
2108
+ /* PER_FIBER_FUSED: there is no whole-view buffer to alias from. Build
2109
+ the fiber region from outer_idx + fiber_axis and call
2110
+ ca_xfer_stride(src, ..., GET) directly into fiber_data_scratch; the
2111
+ view's region transfer (out-of-bounds filling for CAWindow / CAShift,
2112
+ the conversion for a fake or byte-swap view) delivers it in one
2113
+ pass.
2610
2114
 
2611
2115
  last_data_off carries the fiber region's outer footprint so
2612
2116
  sync_slab can reconstruct the same region for WRITE PUT. Encode
@@ -2622,16 +2126,16 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2622
2126
  ca_size_t counts[CA_RANK_MAX];
2623
2127
 
2624
2128
  if ( st->fiber_data_scratch_cap < need ) {
2625
- if ( st->fiber_data_scratch ) xfree(st->fiber_data_scratch);
2626
- st->fiber_data_scratch = (char *) xmalloc(need > 0 ? need : 1);
2129
+ if ( st->fiber_data_scratch ) ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
2130
+ st->fiber_data_scratch = (char *) ki_data_alloc(st->src->data_type, need);
2627
2131
  st->fiber_data_scratch_cap = need;
2628
2132
  }
2629
2133
 
2630
2134
  /* Build fiber region: fiber_axis spans the full fiber, all other
2631
2135
  axes pinned to outer_idx position (count=1). Cache starts[] in
2632
2136
  state so sync_slab can rebuild the same region for WRITE PUT
2633
- (= captured BEFORE outer_idx advance below, same hazard pattern
2634
- as F.1a last_data_off). */
2137
+ (= captured BEFORE the outer_idx advance below, the same hazard as
2138
+ last_data_off; see sync_slab). */
2635
2139
  {
2636
2140
  int8_t op = 0;
2637
2141
  for ( int8_t k = 0; k < nd; k++ ) {
@@ -2691,82 +2195,23 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
2691
2195
  return 1;
2692
2196
  }
2693
2197
 
2694
- /* PROPOSAL_FIBER_DELIVERY F.1a: per-axis fiber contig delivery.
2695
- For naxes==1 (= single slab axis = fiber) the catalog contract
2198
+ /* Contiguous fiber delivery (promise 1). For naxes==1 (= single slab axis = fiber) the catalog contract
2696
2199
  CA_FOR_EACH_FIBER promises contig data delivery. When the fiber
2697
2200
  is not innermost-contig (= slab_strides[0] != bytes), the engine
2698
2201
  gathers the fiber into fiber_data_scratch before yielding so the
2699
2202
  author can write p[i] without stride math.
2700
2203
 
2701
2204
  Capture last_data_off BEFORE the outer_idx advance below; sync_slab
2702
- consumes it for WRITE scatter. See header field doc + PROPOSAL
2703
- §4.3.2 hazard comment in sync_slab. */
2704
- char *yield_ptr;
2705
- if ( (st->flags & CA_KERNEL_FIBER_CONTIG)
2706
- && st->naxes == 1 && st->slab_ndim == 1 ) {
2707
- ca_size_t n = st->slab_dims[0];
2708
- ca_size_t data_step = st->slab_strides[0];
2709
- ca_size_t bytes = st->bytes;
2710
- char *src_data = st->alias_ptr + data_off;
2711
-
2712
- st->last_data_off = data_off;
2713
-
2714
- if ( data_step == (ca_size_t) bytes ) {
2715
- /* Fast path: fiber is already contig (= innermost-axis or stride
2716
- coincidentally == bytes). No gather needed. */
2717
- yield_ptr = src_data;
2718
- } else {
2719
- /* Per-fiber gather via the typed-store inline helper
2720
- (ca_iter_substrate.h). For bytes in {1,2,4,8} this uses a
2721
- compiler-vectorize-friendly `*dp++ = v; sp += step` loop;
2722
- other sizes fall back to per-element memcpy. Lazy-alloc
2723
- scratch sized to max fiber bytes (slab_dims[0] constant per
2724
- walk → single alloc in practice). */
2725
- ca_size_t need = n * bytes;
2726
- if ( st->fiber_data_scratch_cap < need ) {
2727
- if ( st->fiber_data_scratch ) xfree(st->fiber_data_scratch);
2728
- st->fiber_data_scratch = (char *) xmalloc(need);
2729
- st->fiber_data_scratch_cap = need;
2730
- }
2731
- ca_stride_gather_run(st->fiber_data_scratch, src_data,
2732
- bytes, n, data_step);
2733
- yield_ptr = st->fiber_data_scratch;
2734
- }
2735
- } else {
2736
- yield_ptr = st->alias_ptr + data_off;
2737
- }
2738
-
2739
- /* PROPOSAL_FIBER_DELIVERY F.1b: per-fiber contig mask delivery.
2740
- Symmetric to F.1a data path above. When the source has a mask
2741
- (= alias_mask != NULL) and the fiber's mask is not innermost-contig
2742
- (= slab_mask_strides[0] != 1), gather it into fiber_mask_scratch
2743
- so the author can write m[i] without stride math. Mask is
2744
- read-only here (= L2 WRITE never propagates to mask state), so no
2745
- scatter is needed in sync_slab. */
2746
- boolean8_t *yield_mask;
2747
- if ( (st->flags & CA_KERNEL_FIBER_CONTIG)
2748
- && st->naxes == 1 && st->slab_ndim == 1
2749
- && st->alias_mask != NULL ) {
2750
- ca_size_t n = st->slab_dims[0];
2751
- ca_size_t mask_step = st->slab_mask_strides[0];
2752
- boolean8_t *src_mask = st->alias_mask + mask_off;
2753
-
2754
- if ( mask_step == 1 ) {
2755
- yield_mask = src_mask;
2756
- } else {
2757
- if ( st->fiber_mask_scratch_cap < (ca_size_t) n ) {
2758
- if ( st->fiber_mask_scratch ) xfree(st->fiber_mask_scratch);
2759
- st->fiber_mask_scratch = (boolean8_t *) xmalloc(n);
2760
- st->fiber_mask_scratch_cap = n;
2761
- }
2762
- for ( ca_size_t i = 0; i < n; i++ ) {
2763
- st->fiber_mask_scratch[i] = src_mask[i * mask_step];
2764
- }
2765
- yield_mask = st->fiber_mask_scratch;
2766
- }
2767
- } else {
2768
- yield_mask = st->alias_mask ? st->alias_mask + mask_off : NULL;
2769
- }
2205
+ consumes it for WRITE scatter. See the header field and the hazard
2206
+ comment in sync_slab. */
2207
+ /* sync_slab finds the slab here: the fiber it scatters back, and the
2208
+ cells of a whole-view scratch it compares. */
2209
+ st->last_data_off = data_off;
2210
+ char *yield_ptr = ki_fiber_contig_data(st, st->alias_ptr + data_off,
2211
+ st->slab_strides[0]);
2212
+ boolean8_t *yield_mask = ki_fiber_contig_mask(st,
2213
+ st->alias_mask ? st->alias_mask + mask_off : NULL,
2214
+ st->slab_mask_strides[0]);
2770
2215
 
2771
2216
  if ( out_ptr ) *out_ptr = yield_ptr;
2772
2217
  if ( out_mask ) *out_mask = yield_mask;
@@ -2805,8 +2250,6 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
2805
2250
  /* ======================================================================
2806
2251
  * !!! CORRECTNESS HAZARD - DO NOT MOVE !!!
2807
2252
  *
2808
- * (PROPOSAL_FIBER_DELIVERY F.1a)
2809
- *
2810
2253
  * Per-fiber scratch reuse + scatter correctness depends on the strict
2811
2254
  * evaluation order of the CA_FOR_EACH_FIBER_* macro sandwich:
2812
2255
  *
@@ -2833,14 +2276,13 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
2833
2276
  * instead of reusing -- separate phase, separate design.
2834
2277
  * ====================================================================== */
2835
2278
 
2836
- /* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1: per-fiber fused WRITE PUT.
2837
- When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, the author
2279
+ /* PER_FIBER_FUSED write: the author
2838
2280
  wrote into fiber_data_scratch and there is no whole-view buffer
2839
2281
  to xfer_all PUT. Rebuild the same fiber region from cached
2840
2282
  fiber_region_starts[] (= captured pre-advance in next_slab_axes,
2841
- same hazard pattern as F.1a last_data_off) and call
2842
- ca_xfer_stride(src, ..., PUT) which routes through the view's
2843
- fused PUT path (X.1 / X.4) for 1-pass scatter back. */
2283
+ the same hazard as last_data_off) and call ca_xfer_stride(src, ...,
2284
+ PUT), which puts the fiber back through the view's region transfer
2285
+ in one pass. */
2844
2286
  if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
2845
2287
  ca_size_t fiber_n = st->slab_dims[0];
2846
2288
  int8_t nd = st->ndim;
@@ -2855,7 +2297,7 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
2855
2297
  return;
2856
2298
  }
2857
2299
 
2858
- /* Per-fiber gather path scatter (PROPOSAL_FIBER_DELIVERY F.1a).
2300
+ /* Per-fiber gather path scatter.
2859
2301
  When next_slab_axes gathered the fiber into fiber_data_scratch
2860
2302
  (= naxes==1 + slab_strides[0] != bytes), scatter it back to the
2861
2303
  source layout via the strided write. When the fiber was the
@@ -2880,87 +2322,57 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
2880
2322
  it fires for any source whose fiber is not innermost-contig -- so
2881
2323
  for SRC_ATTACH this has just written the fiber back into the
2882
2324
  whole-view scratch, and the push-back below carries it to the
2883
- source. For the alias paths scratch_ptr is NULL and the switch
2884
- below early-returns. */
2885
- }
2886
-
2887
- /* SRC_ATTACH path (step 9 + 2026-05-31 refactor): kernel wrote into
2888
- iterator-owned scratch (= scratch_ptr). Push back via xfer_all PUT
2889
- which routes through the view's xfer_all slot -- handles CAFake
2890
- (cast back), CAByteSwap (swap back), CABitfield/CABitarray (bit
2891
- pack back), CAReduce (broadcast across reduce window). Inherits
2892
- transform-fused / partial materialise / etc. automatically.
2893
-
2894
- This has to stay below the two per-fiber blocks: consulting
2895
- src_kind first would push back the whole-view buffer while the
2896
- fiber the author actually wrote sat unscattered in
2897
- fiber_data_scratch, and would push back a NULL one for
2898
- PER_FIBER_FUSED, which owns no whole-view buffer at all. */
2899
- if ( st->src_kind == CA_ITER_SRC_ATTACH ) {
2900
- if ( st->src->elements > 0 ) {
2901
- ca_xfer_all(st->src, st->scratch_ptr, CA_XFER_PUT);
2902
- }
2903
- st->write_dirty = 0;
2904
- return;
2325
+ source. For the alias paths scratch_ptr is NULL and the push-back
2326
+ below is skipped. */
2905
2327
  }
2906
2328
 
2907
- /* alias path: kernel wrote through alias_ptr into parent directly
2908
- (case A semantics, PROPOSAL_T1_WRITE_SEMANTICS.md §(a)). No
2909
- scatter needed. Applies to all CAStride alias and to
2910
- STRIDE-only descriptor alias (= descriptor L2 alias future
2911
- optimisation, not yet enabled — but if it lands, scratch_ptr
2912
- stays NULL and we no-op correctly). */
2329
+ /* alias path: the kernel wrote through alias_ptr into the memory the
2330
+ walk borrowed, so nothing is scattered. Applies to all CAStride alias and to
2331
+ STRIDE-only descriptor alias (SRC_DESCRIPTOR_L2_ALIASABLE), whose
2332
+ scratch_ptr is NULL too. An alias walk over a view root is written
2333
+ back once, by finish (ca_sync_detach on the root). */
2913
2334
  if ( st->scratch_ptr == NULL ) return;
2914
2335
 
2915
- /* scratch path: scatter back the materialised buffer into the
2916
- source view. src_kind chooses the engine: */
2917
- switch ( st->src_kind ) {
2918
- case CA_ITER_SRC_CASTRIDE:
2919
- /* Two producers of CAStride + scratch: L1 non-contig, and the L2
2920
- non-entity-root path (init_l2's use_view_scratch). Both gathered
2921
- with ca_copy_data into a view row-major buffer, so both scatter
2922
- back the same way — ca_sync_data routes through the view's
2923
- xfer_all(PUT), which asks the root for the region it owns rather
2924
- than writing a whole-root materialise back. */
2925
- ca_sync_data(st->src, st->scratch_ptr);
2926
- break;
2927
-
2928
- case CA_ITER_SRC_DESCRIPTOR: {
2929
- /* Sub-step 5.4: descriptor framework scatter back.
2930
- ca_axis_dispatch_scatter is the P3-landed engine entry that
2931
- handles per-axis kind (STRIDE / INDEX / SHIFT) gather direction
2932
- in reverse — INDEX duplicates yield last-write-wins (R5 spec),
2933
- SHIFT OOB cells in CAWindow FILL policy are skipped (no parent
2934
- destination), CAShift WRAP/REFLECT bounds map back to interior
2935
- and are written normally. Iteration order is engine-defined;
2936
- user kernels must not rely on it.
2937
-
2938
- scatter wants the pre-merge raw descriptors (the engine re-runs
2939
- _prepare internally with whatever we give it). init cached
2940
- the post-merge axes in st->descs for next_slab_strided's
2941
- offset math; we re-call describe_axes here for the scatter
2942
- call rather than caching a second copy in the state struct. */
2943
- ca_axis_desc_t raw_descs[CA_RANK_MAX];
2944
- ca_size_t raw_pdims[CA_RANK_MAX];
2945
- int8_t raw_ndim = 0;
2946
- ca_iter_describe_axes(st->src, raw_descs, raw_pdims, &raw_ndim);
2947
- ca_axis_dispatch_scatter(st->root /* parent */,
2948
- raw_pdims,
2949
- raw_descs, raw_ndim,
2950
- st->bytes, st->total_elements,
2951
- st->scratch_ptr);
2952
- break;
2953
- }
2954
- }
2336
+ /* scratch path: the kernel wrote into a whole-view scratch, and the
2337
+ changed cells go back through the view's own transfer, which
2338
+ converts, packs or scatters them as the view requires (promise 3).
2339
+ Every write walk that reads a scratch gathered it with
2340
+ ki_gather_view, which keeps the copy the comparison needs: the
2341
+ transfer walk (SRC_ATTACH) and the CAStride paths that read a
2342
+ gathered copy. The descriptor engine's own gathers serve reads
2343
+ only; a write over a descriptor view is sent to the transfer walk
2344
+ at init.
2345
+
2346
+ This has to stay below the two per-fiber blocks: pushing back first
2347
+ would send the whole-view buffer while the fiber the author actually
2348
+ wrote sat unscattered in fiber_data_scratch. */
2349
+ if ( st->src_kind != CA_ITER_SRC_ATTACH
2350
+ && st->src_kind != CA_ITER_SRC_CASTRIDE ) {
2351
+ rb_bug("kernel iterator: a write walk over a scratch with nothing to "
2352
+ "compare it against (src_kind %d)", (int) st->src_kind);
2353
+ }
2354
+ ki_scratch_push_changed(st);
2955
2355
  st->write_dirty = 0;
2956
2356
  }
2957
2357
 
2358
+ static VALUE ca_iter_finish_sync (VALUE arg);
2359
+
2360
+ static VALUE
2361
+ ki_sync_detach_root (VALUE root)
2362
+ {
2363
+ ca_sync_detach((CArray *) root);
2364
+ return Qnil;
2365
+ }
2366
+
2958
2367
  void
2959
2368
  ca_iter_state_finish (ca_iter_state *st)
2960
2369
  {
2961
2370
  if ( st == NULL || st->src == NULL ) {
2962
2371
  return;
2963
2372
  }
2373
+ /* A slab still handed out is dropped: finish abandons it. The block
2374
+ macros close with ca_iter_state_close, which writes it back first. */
2375
+ st->write_dirty = 0;
2964
2376
  /* composed_strides is inline — no free needed. outer_idx is heap
2965
2377
  for L2 multi-d sources (NULL on L1 paths and on L2 0/1-d). */
2966
2378
  if ( st->outer_idx ) {
@@ -2970,14 +2382,14 @@ ca_iter_state_finish (ca_iter_state *st)
2970
2382
  /* Lifecycle cleanup — orders matter slightly (free scratch before
2971
2383
  detaching parent so the kernel iterator's resources are released
2972
2384
  symmetrically with init):
2973
- - scratch_ptr: owned by iter (CAStride non-contig L1, or
2974
- descriptor materialise via ca_axis_dispatch_attach). xfree.
2975
- - root: descriptor parent attached at init (sub-step 5.1+) or
2976
- CAStride L2 compose-fold root attached at init (step 3).
2977
- ca_detach.
2385
+ - scratch_ptr: owned by the iterator (every gather into scratch:
2386
+ CAStride non-contig, descriptor, SRC_ATTACH, per-slab, stack).
2387
+ ki_data_free.
2388
+ - root: descriptor parent or CAStride compose-fold root attached
2389
+ at init. ca_detach, or ca_sync_detach on a WRITE walk.
2978
2390
  - else (alias paths): src was attached at init, detach. */
2979
2391
  if ( st->scratch_ptr ) {
2980
- xfree(st->scratch_ptr);
2392
+ ki_data_free(st->scratch_ptr, st->scratch_cap);
2981
2393
  st->scratch_ptr = NULL;
2982
2394
  st->scratch_cap = 0;
2983
2395
  }
@@ -2985,9 +2397,9 @@ ca_iter_state_finish (ca_iter_state *st)
2985
2397
  xfree(st->scratch_mask);
2986
2398
  st->scratch_mask = NULL;
2987
2399
  }
2988
- /* PROPOSAL_FIBER_DELIVERY F.1a/F.1b: per-fiber scratch lifecycle. */
2400
+ /* The per-fiber scratch. */
2989
2401
  if ( st->fiber_data_scratch ) {
2990
- xfree(st->fiber_data_scratch);
2402
+ ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
2991
2403
  st->fiber_data_scratch = NULL;
2992
2404
  st->fiber_data_scratch_cap = 0;
2993
2405
  }
@@ -2996,9 +2408,10 @@ ca_iter_state_finish (ca_iter_state *st)
2996
2408
  st->fiber_mask_scratch = NULL;
2997
2409
  st->fiber_mask_scratch_cap = 0;
2998
2410
  }
2999
- /* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2 + P.2 Case A: detach
3000
- K parents (+ K parent masks if cached) and free the cached ptr
3001
- arrays. Symmetric with init_l2 per-parent ca_attach loop. */
2411
+ /* Detach the K parents init attached and free the cached ptr array
2412
+ (symmetric with init_l2's per-parent ca_attach loop). The parent
2413
+ mask copies are freed separately below; they were never attached
2414
+ for the walk. */
3002
2415
  if ( st->stack_parent_ptrs ) {
3003
2416
  CAStack *stack = (CAStack *) st->src;
3004
2417
  for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
@@ -3007,37 +2420,87 @@ ca_iter_state_finish (ca_iter_state *st)
3007
2420
  xfree(st->stack_parent_ptrs);
3008
2421
  st->stack_parent_ptrs = NULL;
3009
2422
  }
3010
- /* pilot/castack-axis0-loop-interchange: free tile cache. */
2423
+ /* The STACK tile cache. */
3011
2424
  if ( st->stack_tile_cache ) {
3012
- xfree(st->stack_tile_cache);
2425
+ ki_data_free(st->stack_tile_cache, 0);
3013
2426
  st->stack_tile_cache = NULL;
3014
2427
  st->stack_tile_cap = 0;
3015
2428
  st->stack_tile_pos = 0;
3016
2429
  st->stack_tile_have = 0;
3017
2430
  }
3018
2431
  if ( st->stack_parent_mask_ptrs ) {
3019
- CAStack *stack = (CAStack *) st->src;
3020
2432
  for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
3021
- ca_detach(stack->parents[kk]->mask);
2433
+ if ( st->stack_parent_mask_ptrs[kk] ) {
2434
+ xfree(st->stack_parent_mask_ptrs[kk]);
2435
+ }
3022
2436
  }
3023
2437
  xfree(st->stack_parent_mask_ptrs);
3024
2438
  st->stack_parent_mask_ptrs = NULL;
3025
2439
  }
3026
2440
  st->stack_n_parents = 0;
3027
- if ( st->root ) {
3028
- ca_detach(st->root);
3029
- st->root = NULL;
3030
- } else if ( st->alias_mode == CA_ITER_ALIAS_CONTIG
3031
- || st->alias_mode == CA_ITER_ALIAS_STRIDED ) {
3032
- /* alias path (CONTIG/STRIDED for CAStride family): we attached
3033
- src directly at init, so detach it here.
3034
- Note: SRC_ATTACH used to be in this list (ALIAS_ATTACH); after
3035
- the 2026-05-31 refactor it owns its own scratch (alias_mode =
3036
- NONE, scratch_ptr xfree'd above), so no src.detach needed. */
3037
- ca_detach(st->src);
3038
- }
3039
- st->src = NULL;
3040
- st->alias_ptr = NULL;
2441
+ {
2442
+ /* The state is emptied before anything here can raise: syncing a
2443
+ write walk's root goes through the root's slots. A finish that
2444
+ raised is then finished already, and a second call returns at once
2445
+ instead of detaching src again. */
2446
+ CArray *root = st->root;
2447
+ CArray *src = st->src;
2448
+ CArray *own = (st->flags & CA_ITER_FLAG_OWNS_SRC) ? src : NULL;
2449
+ int write = (st->flags & CA_KERNEL_WRITE) != 0;
2450
+ int detach_src = root == NULL
2451
+ && ( st->alias_mode == CA_ITER_ALIAS_CONTIG
2452
+ || st->alias_mode == CA_ITER_ALIAS_STRIDED );
2453
+ int tag = 0;
2454
+ st->root = NULL;
2455
+ st->src = NULL;
2456
+ st->alias_ptr = NULL;
2457
+ st->flags &= ~CA_ITER_FLAG_OWNS_SRC;
2458
+ if ( root ) {
2459
+ /* A write walk wrote into the root's attach buffer, which is the root
2460
+ itself only for an entity: a view root (a transpose, a non-contig
2461
+ block, a view lending no memory) gave a buffer of its own. */
2462
+ if ( write ) {
2463
+ /* protect: frees the iterator's own copy before the raise passes on */
2464
+ rb_protect(ki_sync_detach_root, (VALUE) root, &tag);
2465
+ }
2466
+ else {
2467
+ ca_detach(root);
2468
+ }
2469
+ }
2470
+ else if ( detach_src ) {
2471
+ /* alias path (CONTIG/STRIDED for CAStride family): we attached
2472
+ src directly at init, so detach it here. A SRC_ATTACH walk owns
2473
+ its own scratch (alias_mode = NONE, freed above) and attached
2474
+ nothing to detach. */
2475
+ ca_detach(src);
2476
+ }
2477
+ if ( own ) {
2478
+ if ( ca_is_object_type(own) ) {
2479
+ ca_gc_release_buffer(own->ptr);
2480
+ }
2481
+ ca_free(own);
2482
+ }
2483
+ if ( tag ) {
2484
+ rb_jump_tag(tag);
2485
+ }
2486
+ }
2487
+ }
2488
+
2489
+ /* Finish a walk the way the block macros leave it: a slab of a write walk
2490
+ still handed out -- the body left with `break` -- is written back
2491
+ first. A raise from the write-back is passed on once the walk is
2492
+ finished. */
2493
+ void
2494
+ ca_iter_state_close (ca_iter_state *st)
2495
+ {
2496
+ int tag = 0;
2497
+ if ( st != NULL && st->src != NULL && st->write_dirty ) {
2498
+ rb_protect(ca_iter_finish_sync, (VALUE) st, &tag);
2499
+ }
2500
+ ca_iter_state_finish(st);
2501
+ if ( tag ) {
2502
+ rb_jump_tag(tag);
2503
+ }
3041
2504
  }
3042
2505
 
3043
2506
  /* ---- Raise-safe entry points ----------------------------------------
@@ -3053,10 +2516,9 @@ ca_iter_state_finish (ca_iter_state *st)
3053
2516
  * and ca_iter_state_finish is idempotent (it clears st->src), so the
3054
2517
  * caller's own finish after an ordinary walk stays correct.
3055
2518
  *
3056
- * Not covered: a kernel body that raises. It runs in the caller's
2519
+ * Not covered here: a kernel body that raises. It runs in the caller's
3057
2520
  * frame, between two calls of these functions, where the engine has no
3058
- * hold on it. An object-lane kernel calling back into Ruby is the case
3059
- * to watch.
2521
+ * hold on it. A body that can raise runs through ca_iter_ensure below.
3060
2522
  *
3061
2523
  * init runs protected always -- once per walk, next to a materialise.
3062
2524
  * next_slab / sync_slab are called once per slab, in walks whose whole
@@ -3072,6 +2534,7 @@ ca_iter_state_finish (ca_iter_state *st)
3072
2534
  header. */
3073
2535
  #define CA_ITER_FLAG_WALK_MAY_RAISE 0x80000000u
3074
2536
 
2537
+
3075
2538
  typedef struct {
3076
2539
  ca_iter_state *st;
3077
2540
  CArray *src;
@@ -3082,15 +2545,31 @@ typedef struct {
3082
2545
  char **out_ptr;
3083
2546
  boolean8_t **out_mask;
3084
2547
  ca_size_t *out_n;
3085
- ca_size_t *out_stride_bytes;
3086
2548
  int rc;
3087
2549
  } ca_iter_call_t;
3088
2550
 
3089
- /* Whether next_slab / sync_slab on this walk need the protection. */
2551
+ /* Whether next_slab / sync_slab on this walk need the protection.
2552
+
2553
+ Shortcut PROTECT_ONLY_MAY_RAISE.
2554
+ Layer the exits (promise 4): which walks run next_slab /
2555
+ sync_slab under rb_protect.
2556
+ Speeds up walks that cannot raise there: they skip rb_protect.
2557
+ Why about 1.5x on a per-axis sum over 200k short fibers.
2558
+ Enters the route is not PER_FIBER_FUSED and the walk does not
2559
+ write back out of scratch.
2560
+ Promises 4 (a walk left out cannot raise between init and finish).
2561
+ Switch ca_fast_paths_enabled(): off, every walk is protected.
2562
+ Checked by test_iter_raise_cleanup.rb; the raise rows of
2563
+ test_iter_delivery_detectors.rb. */
3090
2564
  static void
3091
2565
  ca_iter_mark_walk_may_raise (ca_iter_state *st)
3092
2566
  {
3093
2567
  if ( st == NULL ) return;
2568
+ if ( ! ca_fast_paths_enabled() ) {
2569
+ /* the general path protects every walk */
2570
+ st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
2571
+ return;
2572
+ }
3094
2573
  if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
3095
2574
  /* gathers each fiber through the source's xfer_stride slot */
3096
2575
  st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
@@ -3099,19 +2578,35 @@ ca_iter_mark_walk_may_raise (ca_iter_state *st)
3099
2578
  && (st->scratch_ptr != NULL
3100
2579
  || st->src_kind == CA_ITER_SRC_ATTACH) ) {
3101
2580
  /* writes back out of iterator-owned scratch, through the source's
3102
- xfer_all / sync_data slot. A write into aliased memory needs no
2581
+ transfer (ki_scratch_push_changed: ca_xfer_addrs / ca_xfer_all PUT)
2582
+ or the descriptor scatter. A write into aliased memory needs no
3103
2583
  push-back at all: sync_slab returns at once. */
3104
2584
  st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
3105
2585
  }
3106
2586
  }
3107
2587
 
2588
+ /* When body raises, finish st -- and the state it is paired with (see
2589
+ ca_iter_state_init_l2_paired), which is open in the same frame and
2590
+ would otherwise be jumped over. */
2591
+ static VALUE ca_iter_ensure_finish_one (VALUE arg);
2592
+
3108
2593
  static void
3109
2594
  ca_iter_protect (ca_iter_state *st, VALUE (*body)(VALUE), VALUE arg)
3110
2595
  {
3111
2596
  int tag = 0;
3112
2597
  rb_protect(body, arg, &tag);
3113
2598
  if ( tag ) {
3114
- ca_iter_state_finish(st);
2599
+ ca_iter_state *pair = (ca_iter_state *) st->pair;
2600
+ volatile VALUE error = rb_errinfo();
2601
+ int ignored = 0;
2602
+ /* protect: finishes the pair even when finishing st raises; the
2603
+ body's exception is the one passed on */
2604
+ rb_protect(ca_iter_ensure_finish_one, (VALUE) st, &ignored);
2605
+ if ( pair != NULL ) {
2606
+ /* protect: as above */
2607
+ rb_protect(ca_iter_ensure_finish_one, (VALUE) pair, &ignored);
2608
+ }
2609
+ rb_set_errinfo(error);
3115
2610
  rb_jump_tag(tag);
3116
2611
  }
3117
2612
  }
@@ -3143,15 +2638,6 @@ ca_iter_call_next_slab (VALUE arg)
3143
2638
  return Qnil;
3144
2639
  }
3145
2640
 
3146
- static VALUE
3147
- ca_iter_call_next_slab_strided (VALUE arg)
3148
- {
3149
- ca_iter_call_t *c = (ca_iter_call_t *) arg;
3150
- c->rc = ca_iter_state_next_slab_strided_unprotected(c->st, c->out_ptr,
3151
- c->out_mask, c->out_n,
3152
- c->out_stride_bytes);
3153
- return Qnil;
3154
- }
3155
2641
 
3156
2642
  static VALUE
3157
2643
  ca_iter_call_next_slab_axes (VALUE arg)
@@ -3181,6 +2667,9 @@ ca_iter_state_init_l1 (ca_iter_state *st,
3181
2667
  ca_iter_call_t c;
3182
2668
  c.st = st; c.src = src; c.policy = policy;
3183
2669
  c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
2670
+ if ( st != NULL ) {
2671
+ st->pair = NULL; /* read by ca_iter_protect if the init raises */
2672
+ }
3184
2673
  ca_iter_protect(st, ca_iter_call_init_l1, (VALUE) &c);
3185
2674
  if ( c.rc == CA_ITER_OK ) ca_iter_mark_walk_may_raise(st);
3186
2675
  return c.rc;
@@ -3197,11 +2686,32 @@ ca_iter_state_init_l2 (ca_iter_state *st,
3197
2686
  ca_iter_call_t c;
3198
2687
  c.st = st; c.src = src; c.policy = policy;
3199
2688
  c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
2689
+ if ( st != NULL ) {
2690
+ st->pair = NULL; /* read by ca_iter_protect if the init raises */
2691
+ }
3200
2692
  ca_iter_protect(st, ca_iter_call_init_l2, (VALUE) &c);
3201
2693
  if ( c.rc == CA_ITER_OK ) ca_iter_mark_walk_may_raise(st);
3202
2694
  return c.rc;
3203
2695
  }
3204
2696
 
2697
+ /* A slab handed out by a write walk is dirty until sync_slab writes it
2698
+ back; finish writes back one the caller left with `break`. */
2699
+ static int
2700
+ ca_iter_mark_dirty (ca_iter_state *st, int rc)
2701
+ {
2702
+ if ( rc && st != NULL && (st->flags & CA_KERNEL_WRITE) ) {
2703
+ st->write_dirty = 1;
2704
+ }
2705
+ return rc;
2706
+ }
2707
+
2708
+ static VALUE
2709
+ ca_iter_finish_sync (VALUE arg)
2710
+ {
2711
+ ca_iter_state_sync_slab_unprotected((ca_iter_state *) arg);
2712
+ return Qnil;
2713
+ }
2714
+
3205
2715
  int
3206
2716
  ca_iter_state_next_slab (ca_iter_state *st,
3207
2717
  char **out_ptr,
@@ -3210,31 +2720,15 @@ ca_iter_state_next_slab (ca_iter_state *st,
3210
2720
  {
3211
2721
  ca_iter_call_t c;
3212
2722
  if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
3213
- return ca_iter_state_next_slab_unprotected(st, out_ptr, out_mask, out_n);
2723
+ return ca_iter_mark_dirty(st,
2724
+ ca_iter_state_next_slab_unprotected(st, out_ptr, out_mask, out_n));
3214
2725
  }
3215
2726
  c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.out_n = out_n;
3216
2727
  c.rc = 0;
3217
2728
  ca_iter_protect(st, ca_iter_call_next_slab, (VALUE) &c);
3218
- return c.rc;
2729
+ return ca_iter_mark_dirty(st, c.rc);
3219
2730
  }
3220
2731
 
3221
- int
3222
- ca_iter_state_next_slab_strided (ca_iter_state *st,
3223
- char **out_ptr,
3224
- boolean8_t **out_mask,
3225
- ca_size_t *out_n,
3226
- ca_size_t *out_stride_bytes)
3227
- {
3228
- ca_iter_call_t c;
3229
- if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
3230
- return ca_iter_state_next_slab_strided_unprotected(st, out_ptr, out_mask,
3231
- out_n, out_stride_bytes);
3232
- }
3233
- c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.out_n = out_n;
3234
- c.out_stride_bytes = out_stride_bytes; c.rc = 0;
3235
- ca_iter_protect(st, ca_iter_call_next_slab_strided, (VALUE) &c);
3236
- return c.rc;
3237
- }
3238
2732
 
3239
2733
  int
3240
2734
  ca_iter_state_next_slab_axes (ca_iter_state *st,
@@ -3243,11 +2737,12 @@ ca_iter_state_next_slab_axes (ca_iter_state *st,
3243
2737
  {
3244
2738
  ca_iter_call_t c;
3245
2739
  if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
3246
- return ca_iter_state_next_slab_axes_unprotected(st, out_ptr, out_mask);
2740
+ return ca_iter_mark_dirty(st,
2741
+ ca_iter_state_next_slab_axes_unprotected(st, out_ptr, out_mask));
3247
2742
  }
3248
2743
  c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.rc = 0;
3249
2744
  ca_iter_protect(st, ca_iter_call_next_slab_axes, (VALUE) &c);
3250
- return c.rc;
2745
+ return ca_iter_mark_dirty(st, c.rc);
3251
2746
  }
3252
2747
 
3253
2748
  void
@@ -3262,6 +2757,168 @@ ca_iter_state_sync_slab (ca_iter_state *st)
3262
2757
  ca_iter_protect(st, ca_iter_call_sync_slab, (VALUE) &c);
3263
2758
  }
3264
2759
 
2760
+ typedef struct {
2761
+ ca_iter_state *st;
2762
+ CArray *src;
2763
+ CArray *copy;
2764
+ } ca_iter_reopen_t;
2765
+
2766
+ static VALUE
2767
+ ca_iter_reopen_copy (VALUE arg)
2768
+ {
2769
+ ca_iter_reopen_t *r = (ca_iter_reopen_t *) arg;
2770
+ r->copy = ca_copy(r->src);
2771
+ return Qnil;
2772
+ }
2773
+
2774
+ /* A read walk whose source lives in the root a paired write walk writes
2775
+ into would read cells the kernel has already written: an aliased fiber
2776
+ is the written memory itself, and a gathered one is read from the root
2777
+ after earlier fibers were written back. Such a read walk is reopened
2778
+ on a copy of its source, taken before anything is written and owned by
2779
+ the walk. Returns the init's rc; a raise finishes `first` first. */
2780
+ static int
2781
+ ca_iter_reopen_on_copy (ca_iter_state *first)
2782
+ {
2783
+ ca_iter_reopen_t r;
2784
+ ca_iter_call_t c;
2785
+ int8_t axes[CA_RANK_MAX];
2786
+ int8_t naxes = first->naxes;
2787
+ uint32_t flags = first->flags & ~(CA_ITER_FLAG_WALK_MAY_RAISE
2788
+ | CA_ITER_FLAG_OWNS_SRC);
2789
+ ca_slab_policy_t policy = first->policy;
2790
+ int tag = 0;
2791
+
2792
+ memcpy(axes, first->slab_axes_buf, (size_t) naxes);
2793
+ r.st = first; r.src = first->src; r.copy = NULL;
2794
+ rb_protect(ca_iter_reopen_copy, (VALUE) &r, &tag);
2795
+ if ( tag ) {
2796
+ ca_iter_state_finish(first);
2797
+ rb_jump_tag(tag);
2798
+ }
2799
+ if ( ca_is_object_type(r.copy) ) {
2800
+ /* the walk overwrites the cells that held these VALUEs */
2801
+ ca_gc_hold_buffer(r.copy->ptr, r.copy->elements);
2802
+ }
2803
+ ca_iter_state_finish(first);
2804
+
2805
+ c.st = first; c.src = r.copy; c.policy = policy;
2806
+ c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
2807
+ rb_protect(ca_iter_call_init_l2, (VALUE) &c, &tag);
2808
+ if ( tag || c.rc != CA_ITER_OK ) {
2809
+ ca_iter_state_finish(first);
2810
+ if ( ca_is_object_type(r.copy) ) {
2811
+ ca_gc_release_buffer(r.copy->ptr);
2812
+ }
2813
+ ca_free(r.copy);
2814
+ if ( tag ) {
2815
+ rb_jump_tag(tag);
2816
+ }
2817
+ return c.rc;
2818
+ }
2819
+ first->flags |= CA_ITER_FLAG_OWNS_SRC;
2820
+ ca_iter_mark_walk_may_raise(first);
2821
+ return CA_ITER_OK;
2822
+ }
2823
+
2824
+ int
2825
+ ca_iter_state_init_l2_paired (ca_iter_state *st,
2826
+ ca_iter_state *first,
2827
+ struct _CArray *src,
2828
+ ca_slab_policy_t policy,
2829
+ int8_t *axes,
2830
+ int8_t naxes,
2831
+ uint32_t flags)
2832
+ {
2833
+ ca_iter_call_t c;
2834
+ int tag = 0;
2835
+ if ( (flags & CA_KERNEL_WRITE) && first->src != NULL &&
2836
+ ! (first->flags & CA_KERNEL_WRITE) &&
2837
+ policy == CA_SLAB_AXES && first->policy == CA_SLAB_AXES &&
2838
+ ca_shares_root(first->src, src) ) {
2839
+ int rc = ca_iter_reopen_on_copy(first);
2840
+ if ( rc != CA_ITER_OK ) {
2841
+ return rc;
2842
+ }
2843
+ }
2844
+ c.st = st; c.src = src; c.policy = policy;
2845
+ c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
2846
+ rb_protect(ca_iter_call_init_l2, (VALUE) &c, &tag);
2847
+ if ( tag ) {
2848
+ ca_iter_state_finish(st);
2849
+ ca_iter_state_finish(first);
2850
+ rb_jump_tag(tag);
2851
+ }
2852
+ if ( c.rc != CA_ITER_OK ) {
2853
+ ca_iter_state_finish(first);
2854
+ return c.rc;
2855
+ }
2856
+ ca_iter_mark_walk_may_raise(st);
2857
+ st->pair = first;
2858
+ first->pair = st;
2859
+ return c.rc;
2860
+ }
2861
+
2862
+ typedef struct {
2863
+ int32_t n;
2864
+ ca_iter_state *states;
2865
+ } ca_iter_ensure_t;
2866
+
2867
+ static VALUE
2868
+ ca_iter_ensure_finish_one (VALUE arg)
2869
+ {
2870
+ ca_iter_state_finish((ca_iter_state *) arg);
2871
+ return Qnil;
2872
+ }
2873
+
2874
+ /* Finishes the walks in reverse order of their slots, as nested windows
2875
+ close. A finish can raise: on a WRITE walk whose root is a view, it
2876
+ syncs the root (ca_sync_detach) through the view's slots. Each finish
2877
+ is therefore run under rb_protect, so a raise from one does not leave
2878
+ the states before it unfinished; the first raise is re-raised once
2879
+ every state is closed. */
2880
+ static VALUE
2881
+ ca_iter_ensure_close (VALUE arg)
2882
+ {
2883
+ ca_iter_ensure_t *e = (ca_iter_ensure_t *) arg;
2884
+ int first = 0;
2885
+ volatile VALUE first_err = Qnil;
2886
+ for ( int32_t i = e->n - 1; i >= 0; i-- ) {
2887
+ int status = 0;
2888
+ rb_protect(ca_iter_ensure_finish_one, (VALUE) &e->states[i], &status);
2889
+ if ( status ) {
2890
+ if ( ! first ) {
2891
+ first = status;
2892
+ first_err = rb_errinfo();
2893
+ }
2894
+ rb_set_errinfo(Qnil);
2895
+ }
2896
+ }
2897
+ if ( first ) {
2898
+ rb_set_errinfo(first_err);
2899
+ rb_jump_tag(first);
2900
+ }
2901
+ return Qnil;
2902
+ }
2903
+
2904
+ VALUE
2905
+ ca_iter_ensure (int32_t n, ca_iter_state *states,
2906
+ VALUE (*body)(VALUE), VALUE arg)
2907
+ {
2908
+ ca_iter_ensure_t e;
2909
+ if ( n < 0 ) {
2910
+ rb_raise(rb_eArgError, "ca_iter_ensure: negative state count %d", (int) n);
2911
+ }
2912
+ /* Unopened: finish returns at once on a state whose src is NULL, and
2913
+ init clears the whole state before it takes anything. */
2914
+ for ( int32_t i = 0; i < n; i++ ) {
2915
+ states[i].src = NULL;
2916
+ }
2917
+ e.n = n;
2918
+ e.states = states;
2919
+ return rb_ensure(body, arg, ca_iter_ensure_close, (VALUE) &e);
2920
+ }
2921
+
3265
2922
  #ifdef CARRAY_DEV_BUILD
3266
2923
  /* ============================================================
3267
2924
  * smoke surface (dev-only, stripped in release)
@@ -3270,10 +2927,27 @@ ca_iter_state_sync_slab (ca_iter_state *st)
3270
2927
  * or `CARRAY_DEV=1 rake build_ext`). These helpers expose internal
3271
2928
  * engine state to Ruby for spec_ai regression pins. Do not consume
3272
2929
  * from user code.
3273
- *
3274
- * See devel/PROPOSAL_SMOKE_DEV_BUILD_GATE.md
3275
2930
  * ============================================================ */
3276
2931
 
2932
+ /* The innermost-axis walk the L2 smokes run: one strided run per
2933
+ innermost row, handed out as (pointer, cell count, byte stride). */
2934
+ static int
2935
+ t1_smoke_open_rows (ca_iter_state *st, CArray *src, uint32_t flags)
2936
+ {
2937
+ int8_t ax = (int8_t) (src->ndim - 1);
2938
+ return ca_iter_state_init_l2(st, src, CA_SLAB_AXES, &ax, 1, flags);
2939
+ }
2940
+
2941
+ static int
2942
+ t1_smoke_next_row (ca_iter_state *st, char **p, ca_size_t *n,
2943
+ ca_size_t *stride)
2944
+ {
2945
+ if ( ! ca_iter_state_next_slab_axes(st, p, NULL) ) return 0;
2946
+ *n = st->slab_dims[0];
2947
+ *stride = st->slab_strides[0];
2948
+ return 1;
2949
+ }
2950
+
3277
2951
  /* ---- Ruby smoke surface --------------------------------------------- */
3278
2952
 
3279
2953
  /* L1 smoke (steps 1+2): CArray.t1_smoke(ca) -> Hash with
@@ -3318,7 +2992,7 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
3318
2992
  return result;
3319
2993
  }
3320
2994
 
3321
- /* L2 smoke (step 3): CArray.t1_smoke_strided(ca) -> Hash with
2995
+ /* L2 smoke: CArray.t1_smoke_strided(ca) -> Hash with
3322
2996
  rc => Integer
3323
2997
  slabs => Integer (= total_slabs)
3324
2998
  total_elems => Integer
@@ -3327,8 +3001,6 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
3327
3001
  reported (ptr, n, stride_bytes) tuples, in
3328
3002
  iteration order; should equal view.to_ca.dump_binary
3329
3003
  strides => Array<Integer> — stride_bytes per yielded slab
3330
- (constant across yields under WHOLE policy in
3331
- step 3; surfaced for inspection)
3332
3004
  On error rc != OK, the walk fields are 0 / empty. */
3333
3005
  static VALUE
3334
3006
  rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
@@ -3347,9 +3019,9 @@ rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
3347
3019
  data = rb_str_new(0, 0);
3348
3020
  strides_arr = rb_ary_new();
3349
3021
 
3350
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
3022
+ rc = t1_smoke_open_rows(&st, src, 0);
3351
3023
  if ( rc == CA_ITER_OK ) {
3352
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
3024
+ while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
3353
3025
  if ( p != NULL && n > 0 ) {
3354
3026
  ca_size_t i;
3355
3027
  for ( i = 0; i < n; i++ ) {
@@ -3396,15 +3068,15 @@ rb_t1_smoke_sum_strided_f64 (VALUE klass, VALUE vsrc)
3396
3068
  "t1_smoke_sum_strided_f64 expects a float64 source");
3397
3069
  }
3398
3070
 
3399
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
3071
+ rc = t1_smoke_open_rows(&st, src, 0);
3400
3072
  if ( rc != CA_ITER_OK ) {
3401
3073
  rb_raise(rb_eRuntimeError, "ca_iter_state_init L2 failed (rc=%d)", rc);
3402
3074
  }
3403
3075
 
3404
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
3405
- /* Step 8.1: macro picks contig fast path when stride == sizeof(double),
3406
- falls back to strided loop otherwise. Removes the SIMD inhibition
3407
- observed in step 5.5 §10.4.5 on descriptor materialise paths. */
3076
+ while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
3077
+ /* CA_L2_FOR_EACH takes a contiguous loop when stride ==
3078
+ sizeof(double), which the compiler can vectorise, and a strided
3079
+ loop otherwise. */
3408
3080
  CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
3409
3081
  acc += *dp;
3410
3082
  });
@@ -3445,9 +3117,8 @@ rb_t1_smoke_write_fill_f64 (VALUE klass, VALUE vsrc, VALUE vval)
3445
3117
  return INT2NUM(CA_ITER_OK);
3446
3118
  }
3447
3119
 
3448
- /* WRITE smoke: in-place fill via L2 strided dispatch. Used by the
3449
- step 5.5 aggregate bench to round out the matrix (L1 WRITE was
3450
- already in t1_smoke_write_fill_f64). Kernel walks the strided
3120
+ /* WRITE smoke: in-place fill via an L2 walk (the L1 form is
3121
+ t1_smoke_write_fill_f64). Kernel walks the strided
3451
3122
  slab and writes val at every position; sync_slab scatters back. */
3452
3123
  static VALUE
3453
3124
  rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
@@ -3464,14 +3135,12 @@ rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
3464
3135
  rb_raise(rb_eTypeError, "expects a float64 source");
3465
3136
  }
3466
3137
 
3467
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0,
3468
- CA_KERNEL_WRITE);
3138
+ rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
3469
3139
  if ( rc != CA_ITER_OK ) return INT2NUM(rc);
3470
3140
 
3471
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
3472
- /* Step 8.1: contig fast path via macro (stride == sizeof(double)
3473
- on materialise scratch, which is the descriptor L2 path that
3474
- hit +18-22% in step 5.5 — macro removes that overhead). */
3141
+ while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
3142
+ /* Contiguous loop when stride == sizeof(double) (a gathered
3143
+ scratch), strided otherwise. */
3475
3144
  CA_L2_FOR_EACH(double, p, n, stride, dp, {
3476
3145
  *dp = v;
3477
3146
  });
@@ -3539,13 +3208,11 @@ cmp_double (const void *a, const void *b)
3539
3208
 
3540
3209
  /* WRITE smoke: per-row sort via L2 strided dispatch. For a 2D
3541
3210
  float64 src of shape [m, n], sort each of the m rows in ascending
3542
- order. Uses next_slab_strided so each row is a strided slab; the
3211
+ order. Walks the innermost axis so each row is a strided slab; the
3543
3212
  kernel materialises into a tight contig scratch, qsorts, then
3544
3213
  writes back via the same stride. This exercises L2 WRITE
3545
3214
  mechanics (multi-slab walk, per-slab fill of strided cells)
3546
- without over-engineering a strided qsort itself — that would be a
3547
- Pattern H specialised op, out of step 4 scope (reviewer advice #3).
3548
- */
3215
+ without a strided qsort of its own. */
3549
3216
  static VALUE
3550
3217
  rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
3551
3218
  {
@@ -3561,17 +3228,16 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
3561
3228
  rb_raise(rb_eTypeError, "expects a float64 source");
3562
3229
  }
3563
3230
 
3564
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0,
3565
- CA_KERNEL_WRITE);
3231
+ rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
3566
3232
  if ( rc != CA_ITER_OK ) return INT2NUM(rc);
3567
3233
 
3568
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
3234
+ while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
3569
3235
  if ( n > (ca_size_t) (sizeof(scratch) / sizeof(double)) ) {
3570
3236
  ca_iter_state_finish(&st);
3571
3237
  rb_raise(rb_eRuntimeError, "row too large for smoke scratch");
3572
3238
  }
3573
- /* strided -> contig scratch. Step 8.1: macro handles the
3574
- stride == bytes contig fast path automatically. */
3239
+ /* strided -> contig scratch (CA_L2_FOR_EACH takes the contiguous
3240
+ loop when stride == bytes). */
3575
3241
  ca_size_t _sc_i = 0;
3576
3242
  CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
3577
3243
  scratch[_sc_i++] = *dp;
@@ -3579,7 +3245,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
3579
3245
  /* in-place qsort on the contig scratch */
3580
3246
  qsort(scratch, n, sizeof(double), cmp_double);
3581
3247
  /* contig scratch -> strided (writes back to parent via alias_ptr,
3582
- case A direct write). Step 8.1: macro contig fast path. */
3248
+ a direct write). */
3583
3249
  ca_size_t _wb_i = 0;
3584
3250
  CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
3585
3251
  *dp = scratch[_wb_i++];
@@ -3590,7 +3256,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
3590
3256
  return INT2NUM(CA_ITER_OK);
3591
3257
  }
3592
3258
 
3593
- /* Step 6 smoke: L1 walk that exposes BOTH the value slab and the
3259
+ /* Mask smoke: L1 walk that exposes BOTH the value slab and the
3594
3260
  mask slab. Used to pin "mask is delivered to kernel" semantics.
3595
3261
  Returns rc + mask_seen (Boolean: was out_mask non-NULL at first
3596
3262
  yield) + mask_bytes (String: concatenated mask bytes). */
@@ -3666,9 +3332,9 @@ rb_t1_smoke_attach (VALUE klass, VALUE vsrc)
3666
3332
  return SIZE2NUM(total);
3667
3333
  }
3668
3334
 
3669
- /* L2 bench-grade smoke (sub-step 5.3+). Same shape as t1_smoke_attach
3670
- but uses init_l2 / next_slab_strided so descriptor sources can be
3671
- exercised through the L2 path. */
3335
+ /* L2 bench-grade smoke. Same shape as t1_smoke_attach
3336
+ but walks the innermost axis through init_l2 so descriptor sources
3337
+ can be exercised through the L2 path. */
3672
3338
  static VALUE
3673
3339
  rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
3674
3340
  {
@@ -3681,10 +3347,10 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
3681
3347
  volatile char sink = 0;
3682
3348
 
3683
3349
  TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
3684
- rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
3350
+ rc = t1_smoke_open_rows(&st, src, 0);
3685
3351
  if ( rc != CA_ITER_OK ) return INT2NUM(rc);
3686
3352
 
3687
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
3353
+ while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
3688
3354
  if ( p != NULL && n > 0 ) sink ^= p[0];
3689
3355
  total += n;
3690
3356
  }
@@ -3693,7 +3359,7 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
3693
3359
  return SIZE2NUM(total);
3694
3360
  }
3695
3361
 
3696
- /* step 7: minimal smoke that exposes the `flags` argument to Ruby
3362
+ /* Flags smoke: minimal smoke that exposes the `flags` argument to Ruby
3697
3363
  tests. Just runs init_l1 with the requested flags and returns
3698
3364
  the rc — used to verify NO_MASK enforcement (and any future
3699
3365
  flag-gated rejection paths) without needing a full kernel walk. */
@@ -3703,6 +3369,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
3703
3369
  CArray *src;
3704
3370
  ca_iter_state st;
3705
3371
  int rc;
3372
+ /* fits: a development smoke argument, not a cell */
3706
3373
  uint32_t flags = (uint32_t) NUM2UINT(vflags);
3707
3374
 
3708
3375
  TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
@@ -3711,7 +3378,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
3711
3378
  return INT2NUM(rc);
3712
3379
  }
3713
3380
 
3714
- /* ---- Phase A capstone: CA_SLAB_AXES smoke -------------------------
3381
+ /* ---- CA_SLAB_AXES smoke ----------------------------------------------
3715
3382
  sum a float64 source over user-specified slab axes via init_l2 +
3716
3383
  next_slab_axes. K-D walk over slab dims using slab_strides for data
3717
3384
  and slab_mask_strides for mask (so non-contig CAStride sources work
@@ -3741,10 +3408,11 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
3741
3408
  rb_raise(rb_eTypeError, "t1_smoke_sum_axes_f64 expects a float64 source");
3742
3409
  }
3743
3410
  naxes = (int8_t) (argc - 1);
3744
- if ( naxes > CA_RANK_MAX ) {
3411
+ if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
3745
3412
  rb_raise(rb_eArgError, "too many axes");
3746
3413
  }
3747
3414
  for ( i = 0; i < naxes; i++ ) {
3415
+ /* fits: an axis number, checked by the engine's init */
3748
3416
  slab_axes[i] = (int8_t) NUM2INT(argv[i + 1]);
3749
3417
  }
3750
3418
 
@@ -3780,7 +3448,7 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
3780
3448
  return DBL2NUM(acc);
3781
3449
  }
3782
3450
 
3783
- /* ---- Phase B.3 helper smoke: rb_ca_parse_reduce_axes ---------------
3451
+ /* ---- helper smoke: rb_ca_parse_reduce_axes ---------------------------
3784
3452
  Calls the helper and returns the parsed axes as a Ruby Array of
3785
3453
  Integer. Lets tests inspect parsing behaviour directly.
3786
3454
  Signature: CArray.t1_test_parse_reduce_axes(ca, *args) */
@@ -3806,7 +3474,7 @@ rb_t1_test_parse_reduce_axes (int argc, VALUE *argv, VALUE klass)
3806
3474
  return arr;
3807
3475
  }
3808
3476
 
3809
- /* ---- API harmonisation A.1 smoke: rb_ca_parse_reduce_axes_kw -------
3477
+ /* ---- helper smoke: rb_ca_parse_reduce_axes_kw ------------------------
3810
3478
  Calls the kwarg helper and returns the parsed axes as a Ruby Array.
3811
3479
  Lets tests inspect kwarg-form parsing (Qnil / Integer / Array) +
3812
3480
  validation parity with the variadic entry.
@@ -3834,7 +3502,7 @@ rb_test_parse_reduce_axes_kw (int argc, VALUE *argv, VALUE klass)
3834
3502
  return arr;
3835
3503
  }
3836
3504
 
3837
- /* ---- Phase A.3 helper smoke: rb_ca_new_reduced ---------------------
3505
+ /* ---- helper smoke: rb_ca_new_reduced ---------------------------------
3838
3506
  Exposes rb_ca_new_reduced (carray_core.c) for unit testing. Returns
3839
3507
  the newly allocated output CArray (= tests verify shape + data_type).
3840
3508
  Signature: CArray.t1_test_new_reduced(ca, data_type, axis1, axis2, ...) */
@@ -3849,21 +3517,21 @@ rb_t1_test_new_reduced (int argc, VALUE *argv, VALUE klass)
3849
3517
  rb_raise(rb_eArgError,
3850
3518
  "t1_test_new_reduced expects (ca, data_type, axis...) with >= 1 axis");
3851
3519
  }
3852
- /* PROPOSAL_DTYPE_SYMBOL_FLIP: accept Symbol / Integer / Class / String
3853
- uniformly via rb_ca_guess_type so post-flip CA_* (Symbol) callers
3854
- and legacy Integer-code callers both work. */
3520
+ /* Accept Symbol / Integer / Class / String alike through
3521
+ rb_ca_guess_type. */
3855
3522
  data_type = (int32_t) rb_ca_guess_type(argv[1]);
3856
3523
  naxes = (int8_t) (argc - 2);
3857
- if ( naxes > CA_RANK_MAX ) {
3524
+ if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
3858
3525
  rb_raise(rb_eArgError, "too many axes");
3859
3526
  }
3860
3527
  for ( int i = 0; i < naxes; i++ ) {
3528
+ /* fits: an axis number, checked by the engine's init */
3861
3529
  axes[i] = (int8_t) NUM2INT(argv[2 + i]);
3862
3530
  }
3863
3531
  return rb_ca_new_reduced(argv[0], axes, naxes, data_type, 0);
3864
3532
  }
3865
3533
 
3866
- /* ---- step 9.3: CAReduce-specific smoke -----------------------------
3534
+ /* ---- CAReduce smoke --------------------------------------------------
3867
3535
  CAReduce has no public Ruby ctor (it's an internal class used by
3868
3536
  CARefer mask handling). Tests need to drive kernel_iterator over
3869
3537
  a CAReduce, so we expose a thin construction helper plus dedicated
@@ -3928,8 +3596,9 @@ rb_t1_smoke_reduce_read (VALUE klass, VALUE vparent, VALUE vcount, VALUE voffset
3928
3596
  }
3929
3597
 
3930
3598
  /* WRITE broadcast smoke for CAReduce: fills the reduce view with
3931
- `val` (boolean / 0 or 1) and lets sync_slab → ca_sync run the
3932
- broadcast scatter back to parent. After the call parent's
3599
+ `val` (boolean / 0 or 1) and lets sync_slab send the changed cells
3600
+ back through the view (ki_scratch_push_changed), which broadcasts them
3601
+ to the parent. After the call parent's
3933
3602
  [offset..offset+elems*count) bytes should all equal val. Returns
3934
3603
  rc; on rc != OK parent is not modified. */
3935
3604
  static VALUE
@@ -3942,6 +3611,7 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
3942
3611
  ca_size_t n, i;
3943
3612
  int rc;
3944
3613
  CArray *reduce;
3614
+ /* fits: a development smoke argument, not a cell */
3945
3615
  uint8_t v = (uint8_t) NUM2UINT(vval);
3946
3616
 
3947
3617
  TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
@@ -3957,15 +3627,15 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
3957
3627
  }
3958
3628
  while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
3959
3629
  for ( i = 0; i < n; i++ ) p[i] = v;
3960
- ca_iter_state_sync_slab(&st); /* triggers ca_sync → broadcast */
3630
+ ca_iter_state_sync_slab(&st); /* sends the changed cells back */
3961
3631
  }
3962
3632
  ca_iter_state_finish(&st);
3963
3633
  ca_free(reduce);
3964
3634
  return INT2NUM(CA_ITER_OK);
3965
3635
  }
3966
3636
 
3967
- /* ---- M.6: CARemap kernel_iterator smoke helpers --------------------
3968
- CARemap has no public Ruby ctor (internal-only per §2.2), so we
3637
+ /* ---- CARemap smoke helpers --------------------------------------------
3638
+ CARemap has no public Ruby constructor, so we
3969
3639
  construct it from C using the (ref, idx) args. Pattern mirrors
3970
3640
  rb_t1_smoke_reduce_*: build the view, run the iterator state
3971
3641
  machine, return the standard hash (rc / slabs / total_elems /
@@ -4032,10 +3702,10 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
4032
3702
  view = ca_remap_new(ref, idx);
4033
3703
  data = rb_str_new(0, 0);
4034
3704
 
4035
- rc = ca_iter_state_init_l2(&st, view, CA_SLAB_WHOLE, NULL, 0, 0);
3705
+ rc = t1_smoke_open_rows(&st, view, 0);
4036
3706
  if ( rc == CA_ITER_OK ) {
4037
3707
  ca_size_t stride;
4038
- while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
3708
+ while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
4039
3709
  ca_size_t i;
4040
3710
  for ( i = 0; i < n; i++ ) {
4041
3711
  rb_str_cat(data, p + i * stride, st.bytes);
@@ -4058,7 +3728,8 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
4058
3728
  }
4059
3729
 
4060
3730
  /* WRITE smoke: fill view's slab buffer with `val` (float64), let
4061
- sync_slab → ca_sync run the scatter back to ref via xfer_all PUT. */
3731
+ sync_slab send the changed cells back to ref through the view
3732
+ (ki_scratch_push_changed: ca_xfer_addrs or ca_xfer_all PUT). */
4062
3733
  static VALUE
4063
3734
  rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
4064
3735
  VALUE vval)
@@ -4091,17 +3762,17 @@ rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
4091
3762
  return INT2NUM(CA_ITER_OK);
4092
3763
  }
4093
3764
 
4094
- /* ---- Phase C C.3: CA_FOR_EACH_SLAB macro smoke surfaces -------------
3765
+ /* ---- CA_FOR_EACH_SLAB macro smoke surfaces ---------------------------
4095
3766
  Smoke kernels that exercise the block macros end-to-end. Used by
4096
3767
  spec_ai/test_ca_for_each_slab_macros.rb to pin behavioural
4097
3768
  correctness of the macro expansion (= same byte-parity result as
4098
3769
  the raw API equivalent). */
4099
3770
 
4100
- /* sum reduction using CA_FOR_EACH_SLAB. innermost slab axis only,
4101
- float64 source, accumulator = scalar (single full-reduction slab
4102
- when naxes == src->ndim, otherwise per-outer-slab accumulator
4103
- written to a row-major output buffer). Mirrors carray_kernel_sum
4104
- structurally but uses the macro for lifecycle. */
3771
+ /* sum reduction using CA_FOR_EACH_SLAB over the slab axes given in
3772
+ argv[1..] (K-D slabs). Reads the source as float64 without checking
3773
+ its data_type. Accumulator = scalar (single full-reduction slab when
3774
+ naxes == src->ndim, otherwise per-outer-slab accumulator written to a
3775
+ row-major output buffer). */
4105
3776
  static VALUE
4106
3777
  rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
4107
3778
  {
@@ -4120,6 +3791,7 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
4120
3791
  rb_raise(rb_eArgError, "bad axes count");
4121
3792
  }
4122
3793
  for ( int8_t k = 0; k < naxes; k++ ) {
3794
+ /* fits: an axis number, checked by the engine's init */
4123
3795
  slab_axes[k] = (int8_t) NUM2INT(argv[1 + k]);
4124
3796
  if ( slab_axes[k] < 0 ) slab_axes[k] += ca->ndim;
4125
3797
  }
@@ -4189,8 +3861,8 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
4189
3861
 
4190
3862
  /* map kernel using CA_FOR_EACH_SLAB_INOUT. Doubles each element of
4191
3863
  a float64 source into a fresh same-shape output. Slab axis is
4192
- always the innermost (= argv[1] is unused for simplicity, axis
4193
- fixed to ca->ndim - 1). Demonstrates the 2-iter pattern. */
3864
+ always the innermost (ca->ndim - 1); there is no axis argument.
3865
+ Demonstrates the 2-iter pattern. */
4194
3866
  static VALUE
4195
3867
  rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
4196
3868
  {
@@ -4234,11 +3906,11 @@ rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
4234
3906
  return vout;
4235
3907
  }
4236
3908
 
4237
- /* ---- PROPOSAL_FIBER_DELIVERY F.2: catalog macro smokes -------------
3909
+ /* ---- CA_FOR_EACH_FIBER macro smokes ----------------------------------
4238
3910
  Exercise each of the 4 catalog forms end-to-end with float64 fibers,
4239
3911
  verifying contig delivery for both innermost (= stride==bytes) and
4240
3912
  non-innermost (= stride>bytes, gather path) axis positions. Used by
4241
- spec_ai/test_fiber_delivery.rb (= F.3). */
3913
+ spec_ai/test_fiber_delivery.rb. */
4242
3914
 
4243
3915
  /* form 1 (NO_MASK single): per-axis sum, returns total sum.
4244
3916
  Catalog: CA_FOR_EACH_FIBER. */
@@ -4264,8 +3936,7 @@ rb_caf_fiber_smoke_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4264
3936
  }
4265
3937
 
4266
3938
  /* form 2 (NO_MASK INOUT): per-axis fiber copy * scalar.
4267
- Catalog: CA_FOR_EACH_FIBER_INOUT. Sorts each fiber for non-trivial
4268
- gather/scatter behavior. */
3939
+ Catalog: CA_FOR_EACH_FIBER_INOUT. Doubles each cell of each fiber. */
4269
3940
  static VALUE
4270
3941
  rb_caf_fiber_smoke_double_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4271
3942
  {
@@ -4349,9 +4020,9 @@ rb_caf_fiber_smoke_zero_masked_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4349
4020
  return vout;
4350
4021
  }
4351
4022
 
4352
- /* F.5 bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
4353
- Functional equivalent of ext/carray_order.c::rb_ca_sort_copy_axis_*
4354
- (= hand-rolled gather/scatter with explicit slab_strides[0] loops).
4023
+ /* Bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
4024
+ Functional equivalent of the per-axis sort_copy (rb_ca_sort_copy in
4025
+ ext/carray_sort.c).
4355
4026
  The macro form lets us A/B the author-side overhead of stride math
4356
4027
  vs the catalog-macro-driven engine gather/scatter. */
4357
4028
  static int
@@ -4394,7 +4065,7 @@ rb_caf_fiber_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4394
4065
  return vout;
4395
4066
  }
4396
4067
 
4397
- /* F.5 bench helper variant: sort_copy via SLAB macro + manual
4068
+ /* Bench helper variant: sort_copy via SLAB macro + manual
4398
4069
  gather/scatter (= what hand-rolled sort_copy effectively does, but
4399
4070
  driven through the public CA_FOR_EACH_SLAB_INOUT macro instead of
4400
4071
  the raw next_slab_axes API). Lets us isolate macro overhead from
@@ -4418,9 +4089,13 @@ rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4418
4089
  TypedData_Get_Struct(vout, CArray, &carray_data_type, out);
4419
4090
  slab_axes[0] = (int8_t) axis;
4420
4091
 
4092
+ if ( axis < 0 || axis >= src->ndim ) {
4093
+ rb_raise(rb_eArgError, "caf_slab_bench_sort_copy_f64: axis out of range");
4094
+ }
4421
4095
  ca_size_t fiber_n = src->dim[axis];
4422
4096
  ca_size_t bytes = sizeof(double);
4423
- char *buf = (char *) xmalloc(fiber_n * bytes);
4097
+ volatile VALUE holder; /* the walk can raise while buf is held */
4098
+ char *buf = ALLOCV_N(char, holder, fiber_n * bytes);
4424
4099
 
4425
4100
  CA_FOR_EACH_SLAB_INOUT(st_in, st_out, src, out,
4426
4101
  slab_axes, 1,
@@ -4452,11 +4127,11 @@ rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4452
4127
  }
4453
4128
  }
4454
4129
  }
4455
- xfree(buf);
4130
+ ALLOCV_END(holder);
4456
4131
  return vout;
4457
4132
  }
4458
4133
 
4459
- /* F.5 follow-up bench: per-fiber fused xfer_stride direct into a contig
4134
+ /* Bench helper: per-fiber fused xfer_stride direct into a contig
4460
4135
  scratch, skipping the kernel_iterator SRC_ATTACH whole-view materialise.
4461
4136
  Tests user's hypothesis: for transform views (CAFake/CAByteSwap etc.),
4462
4137
  does bypassing the whole-view scratch_ptr and calling ca_xfer_stride
@@ -4502,7 +4177,8 @@ rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4502
4177
  }
4503
4178
 
4504
4179
  /* Fiber-sized contig scratch. */
4505
- char *buf = (char *) xmalloc(fiber_n * src->bytes);
4180
+ volatile VALUE holder;
4181
+ char *buf = ALLOCV_N(char, holder, fiber_n * src->bytes);
4506
4182
 
4507
4183
  for ( ca_size_t f = 0; f < total_fibers; f++ ) {
4508
4184
  /* Build region: counts = 1 on all non-axis, fiber_n on axis;
@@ -4530,21 +4206,33 @@ rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
4530
4206
  }
4531
4207
  }
4532
4208
 
4533
- xfree(buf);
4209
+ ALLOCV_END(holder);
4534
4210
  return rb_float_new(total);
4535
4211
  }
4536
4212
 
4213
+ /* CArray.__iter_buffers__: see ki_buffers_out. */
4214
+ static VALUE
4215
+ rb_ki_buffers_out (VALUE klass)
4216
+ {
4217
+ (void) klass;
4218
+ return LONG2NUM(__atomic_load_n(&ki_buffers_out, __ATOMIC_RELAXED));
4219
+ }
4220
+
4537
4221
  #endif /* CARRAY_DEV_BUILD — end smoke surface fence */
4538
4222
 
4539
4223
  void
4540
4224
  Init_ca_kernel_iterator (void)
4541
4225
  {
4226
+ ki_kept_key = rb_ractor_local_storage_ptr_newkey(&ki_kept_type);
4227
+
4542
4228
  #ifdef CARRAY_DEV_BUILD
4543
4229
  /* ==== smoke surface registrations (dev-only, stripped in release) ====
4544
- * See PROPOSAL_SMOKE_DEV_BUILD_GATE.md. All `t1_smoke_*`, `caf_smoke_*`,
4230
+ * All `t1_smoke_*`, `caf_smoke_*`,
4545
4231
  * `caf_fiber_smoke_*`, `caf_*_bench_*`, helper smokes, and `T1_*` test
4546
4232
  * constants are gated here together — none of them are consumed by
4547
4233
  * production code (lib/ / other ext/), only by spec_ai regression pins. */
4234
+ rb_define_singleton_method(rb_cCArray, "__iter_buffers__",
4235
+ rb_ki_buffers_out, 0);
4548
4236
  rb_define_singleton_method(rb_cCArray, "t1_smoke",
4549
4237
  rb_t1_smoke, 1);
4550
4238
  rb_define_singleton_method(rb_cCArray, "t1_smoke_strided",
@@ -4567,20 +4255,20 @@ Init_ca_kernel_iterator (void)
4567
4255
  rb_t1_smoke_with_mask, 1);
4568
4256
  rb_define_singleton_method(rb_cCArray, "t1_smoke_init_rc",
4569
4257
  rb_t1_smoke_init_rc, 2);
4570
- /* Phase A capstone: CA_SLAB_AXES smoke (variadic axes) */
4258
+ /* CA_SLAB_AXES smoke (variadic axes) */
4571
4259
  rb_define_singleton_method(rb_cCArray, "t1_smoke_sum_axes_f64",
4572
4260
  rb_t1_smoke_sum_axes_f64, -1);
4573
- /* Phase A.3: rb_ca_new_reduced helper smoke */
4261
+ /* rb_ca_new_reduced helper smoke */
4574
4262
  rb_define_singleton_method(rb_cCArray, "t1_test_new_reduced",
4575
4263
  rb_t1_test_new_reduced, -1);
4576
- /* Phase B.3: rb_ca_parse_reduce_axes helper smoke */
4264
+ /* rb_ca_parse_reduce_axes helper smoke */
4577
4265
  rb_define_singleton_method(rb_cCArray, "t1_test_parse_reduce_axes",
4578
4266
  rb_t1_test_parse_reduce_axes, -1);
4579
4267
 
4580
- /* API harmonisation A.1: rb_ca_parse_reduce_axes_kw helper smoke */
4268
+ /* rb_ca_parse_reduce_axes_kw helper smoke */
4581
4269
  rb_define_singleton_method(rb_cCArray, "test_parse_reduce_axes_kw",
4582
4270
  rb_test_parse_reduce_axes_kw, -1);
4583
- /* CAReduce-specific (step 9.3): no public Ruby API */
4271
+ /* CAReduce: no public Ruby API */
4584
4272
  rb_define_singleton_method(rb_cCArray, "t1_make_reduce",
4585
4273
  rb_t1_make_reduce, 3);
4586
4274
  rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_read",
@@ -4588,19 +4276,19 @@ Init_ca_kernel_iterator (void)
4588
4276
  rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_write_broadcast",
4589
4277
  rb_t1_smoke_reduce_write_broadcast, 4);
4590
4278
 
4591
- /* M.6 CARemap kernel_iterator smokes (test-only). */
4279
+ /* CARemap smokes (test-only). */
4592
4280
  rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read",
4593
4281
  rb_t1_smoke_remap_read, 2);
4594
4282
  rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read_strided",
4595
4283
  rb_t1_smoke_remap_read_strided, 2);
4596
4284
  rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_write_fill_f64",
4597
4285
  rb_t1_smoke_remap_write_fill_f64, 3);
4598
- /* Phase C C.3 (2026-05-27): block macro smokes */
4286
+ /* block macro smokes */
4599
4287
  rb_define_singleton_method(rb_cCArray, "caf_smoke_sum_f64",
4600
4288
  rb_caf_smoke_sum_f64, -1);
4601
4289
  rb_define_singleton_method(rb_cCArray, "caf_smoke_double_f64",
4602
4290
  rb_caf_smoke_double_f64, 1);
4603
- /* PROPOSAL_FIBER_DELIVERY F.2: catalog macro smokes */
4291
+ /* fiber macro smokes */
4604
4292
  rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_sum_f64",
4605
4293
  rb_caf_fiber_smoke_sum_f64, 2);
4606
4294
  rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_double_f64",
@@ -4609,7 +4297,7 @@ Init_ca_kernel_iterator (void)
4609
4297
  rb_caf_fiber_smoke_unmasked_sum_f64, 2);
4610
4298
  rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_zero_masked_f64",
4611
4299
  rb_caf_fiber_smoke_zero_masked_f64, 2);
4612
- /* F.5 bench helper */
4300
+ /* bench helper */
4613
4301
  rb_define_singleton_method(rb_cCArray, "caf_fiber_bench_sort_copy_f64",
4614
4302
  rb_caf_fiber_bench_sort_copy_f64, 2);
4615
4303
  rb_define_singleton_method(rb_cCArray, "caf_slab_bench_sort_copy_f64",
@@ -4629,5 +4317,6 @@ Init_ca_kernel_iterator (void)
4629
4317
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_CONTIG", INT2NUM(CA_ITER_ALIAS_CONTIG));
4630
4318
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_STRIDED", INT2NUM(CA_ITER_ALIAS_STRIDED));
4631
4319
  rb_define_const(rb_cCArray, "T1_ITER_ALIAS_ATTACH", INT2NUM(CA_ITER_ALIAS_ATTACH));
4320
+ rb_define_const(rb_cCArray, "T1_ITER_ALIAS_PER_SLAB", INT2NUM(CA_ITER_ALIAS_PER_SLAB));
4632
4321
  #endif /* CARRAY_DEV_BUILD */
4633
4322
  }