carray 2.0.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +5 -25
- data/CHANGELOG.md +16 -0
- data/LICENSE +1 -1
- data/NEWS.md +3 -0
- data/README.md +128 -44
- data/carray.gemspec +22 -24
- data/ext/ca_array_pool.c +91 -0
- data/ext/ca_axis_descriptor.h +186 -0
- data/ext/ca_axis_dispatch.c +924 -0
- data/ext/ca_axis_group.c +1208 -0
- data/ext/ca_bincmp_dispatch.c +76 -0
- data/ext/ca_bincmp_dispatch.h +85 -0
- data/ext/ca_binop_dispatch.c +125 -0
- data/ext/ca_binop_dispatch.h +159 -0
- data/ext/ca_categorical_iterator.c +1375 -0
- data/ext/ca_compare.c +94 -0
- data/ext/ca_compare.h +26 -0
- data/ext/ca_composite_dispatch.c +414 -0
- data/ext/ca_composite_dispatch.h +116 -0
- data/ext/ca_for_buffer.h +96 -0
- data/ext/ca_for_each_element.h +241 -0
- data/ext/ca_group_iter.c +304 -0
- data/ext/ca_iter_substrate.h +325 -0
- data/ext/ca_kernel_iterator.c +4321 -0
- data/ext/ca_kernel_iterator.h +2603 -0
- data/ext/ca_moncmp_dispatch.c +37 -0
- data/ext/ca_moncmp_dispatch.h +62 -0
- data/ext/ca_monop_dispatch.c +200 -0
- data/ext/ca_monop_dispatch.h +235 -0
- data/ext/ca_obj_array.c +355 -359
- data/ext/ca_obj_bincmp.c +809 -0
- data/ext/ca_obj_binop.c +892 -0
- data/ext/ca_obj_bitarray.c +369 -164
- data/ext/ca_obj_bitfield.c +294 -234
- data/ext/ca_obj_block.c +189 -711
- data/ext/ca_obj_byte_swap.c +766 -0
- data/ext/ca_obj_const_string.c +965 -0
- data/ext/ca_obj_face.c +670 -0
- data/ext/ca_obj_face.h +247 -0
- data/ext/ca_obj_fake.c +228 -100
- data/ext/ca_obj_farray.c +54 -441
- data/ext/ca_obj_field.c +82 -529
- data/ext/ca_obj_fixlen_string.c +306 -0
- data/ext/ca_obj_grid.c +858 -440
- data/ext/ca_obj_meld.c +1034 -0
- data/ext/ca_obj_moncmp.c +569 -0
- data/ext/ca_obj_monop.c +1111 -0
- data/ext/ca_obj_object.c +772 -297
- data/ext/ca_obj_record.c +468 -0
- data/ext/ca_obj_reduce.c +97 -82
- data/ext/ca_obj_refer.c +569 -459
- data/ext/ca_obj_remap.c +475 -0
- data/ext/ca_obj_repeat.c +92 -477
- data/ext/ca_obj_roll.c +616 -0
- data/ext/ca_obj_select.c +344 -296
- data/ext/ca_obj_select_axis.c +1296 -0
- data/ext/ca_obj_shift.c +229 -791
- data/ext/ca_obj_source.c +78 -0
- data/ext/ca_obj_stack.c +1173 -0
- data/ext/ca_obj_stride.c +2501 -0
- data/ext/ca_obj_string.c +268 -0
- data/ext/ca_obj_tile.c +614 -0
- data/ext/ca_obj_time.c +546 -0
- data/ext/ca_obj_timedelta.c +435 -0
- data/ext/ca_obj_transpose.c +62 -516
- data/ext/ca_obj_triop.c +746 -0
- data/ext/ca_obj_unbound_repeat.c +207 -240
- data/ext/ca_obj_window.c +1131 -563
- data/ext/ca_op_byte_swap.c +175 -0
- data/ext/ca_op_ipower.c +319 -0
- data/ext/ca_op_powi.h +88 -0
- data/ext/ca_sort_kernels.h +132 -0
- data/ext/ca_sweep_engine.c +430 -0
- data/ext/ca_sweep_engine.h +157 -0
- data/ext/ca_transform_common.c +228 -0
- data/ext/ca_triop_dispatch.c +55 -0
- data/ext/ca_triop_dispatch.h +62 -0
- data/ext/carray.h +795 -402
- data/ext/carray_access.c +831 -711
- data/ext/carray_attribute.c +98 -329
- data/ext/carray_bincount.c +255 -0
- data/ext/carray_broadcast.c +283 -0
- data/ext/carray_call_cfunc.c +1360 -828
- data/ext/carray_call_cfunc.h +160 -0
- data/ext/carray_cast.c +1212 -301
- data/ext/carray_cast_func.rb +81 -40
- data/ext/carray_class.c +53 -63
- data/ext/carray_config.h +28 -0
- data/ext/carray_conversion.c +350 -346
- data/ext/carray_copy.c +156 -268
- data/ext/carray_core.c +1341 -198
- data/ext/carray_count.c +312 -0
- data/ext/carray_data_type.c +43 -19
- data/ext/carray_element.c +585 -213
- data/ext/carray_factorize.c +2542 -0
- data/ext/carray_generate.c +230 -559
- data/ext/carray_histogram.c +490 -0
- data/ext/carray_hold.c +228 -0
- data/ext/carray_index_classifier.c +1035 -0
- data/ext/carray_index_classifier.h +27 -0
- data/ext/carray_internal.h +120 -0
- data/ext/carray_kernels_bincmp.c +4445 -0
- data/ext/carray_kernels_binop.c +10979 -0
- data/ext/carray_kernels_init.c +36 -0
- data/ext/carray_kernels_map.c +3466 -0
- data/ext/carray_kernels_moncmp.c +2096 -0
- data/ext/carray_kernels_monop.c +18312 -0
- data/ext/carray_kernels_reduce_aggregate.c +25836 -0
- data/ext/carray_kernels_reduce_boolean.c +329 -0
- data/ext/carray_kernels_reduce_cumulative.c +14592 -0
- data/ext/carray_kernels_reduce_extreme.c +16947 -0
- data/ext/carray_kernels_reduce_variance.c +3909 -0
- data/ext/carray_kernels_scan.c +3692 -0
- data/ext/carray_kernels_search.c +32137 -0
- data/ext/carray_kernels_sort.c +10625 -0
- data/ext/carray_kernels_triop.c +1391 -0
- data/ext/carray_lazy.c +567 -0
- data/ext/carray_loop.c +88 -200
- data/ext/carray_mask.c +848 -154
- data/ext/carray_math_kernel.h +120 -0
- data/ext/carray_mathfunc.c +10 -241
- data/ext/carray_median_percentile.c +1257 -0
- data/ext/carray_memory_view.c +1625 -0
- data/ext/carray_operator.c +1526 -318
- data/ext/carray_order.c +664 -1394
- data/ext/carray_partition.c +416 -0
- data/ext/carray_random.c +518 -0
- data/ext/carray_scatter.c +357 -0
- data/ext/carray_slab.c +1219 -0
- data/ext/carray_slab.h +84 -0
- data/ext/carray_sort.c +829 -0
- data/ext/carray_sort_kernel.c +620 -0
- data/ext/carray_struct.c +695 -0
- data/ext/carray_test.c +343 -229
- data/ext/carray_undef.c +34 -17
- data/ext/carray_utils.c +175 -74
- data/ext/extconf.rb +216 -55
- data/ext/mk_call_cfunc.rb +480 -0
- data/ext/mkkernel.rb +8842 -0
- data/ext/ruby_carray.c +202 -101
- data/ext/version.h +5 -15
- data/ext/version.rb +5 -13
- data/lib/carray/arrow_tensor.rb +401 -0
- data/lib/carray/attribute.rb +166 -0
- data/lib/carray/autoload_carray.rb +220 -0
- data/lib/carray/autoload_method_extension.rb +44 -0
- data/lib/carray/axis_group.rb +711 -0
- data/lib/carray/basics.rb +481 -0
- data/lib/carray/bincount_nd.rb +358 -0
- data/lib/carray/block_iterator.rb +604 -0
- data/lib/carray/boolean_reduce.rb +109 -0
- data/lib/carray/categorical.rb +561 -0
- data/lib/carray/categorical_iterator.rb +1062 -0
- data/lib/carray/complex.rb +150 -0
- data/lib/carray/conditional.rb +216 -0
- data/lib/carray/const_string.rb +228 -0
- data/lib/carray/construct.rb +139 -328
- data/lib/carray/core_extensions.rb +240 -0
- data/lib/carray/data_type_extension.rb +233 -0
- data/lib/carray/fixlen_string.rb +95 -0
- data/lib/carray/frame/concat.rb +132 -0
- data/lib/carray/frame/convert.rb +95 -0
- data/lib/carray/frame/csv_parser.rb +211 -0
- data/lib/carray/frame/frame.rb +649 -0
- data/lib/carray/frame/group.rb +186 -0
- data/lib/carray/frame/io.rb +164 -0
- data/lib/carray/frame/join.rb +248 -0
- data/lib/carray/frame/records.rb +99 -0
- data/lib/carray/frame/sort.rb +113 -0
- data/lib/carray/frame/verbs.rb +299 -0
- data/lib/carray/frame.rb +16 -0
- data/lib/carray/histogram.rb +512 -0
- data/lib/carray/inspect.rb +37 -20
- data/lib/carray/iterator.rb +57 -349
- data/lib/carray/lazy.rb +889 -0
- data/lib/carray/mask_gap_fill.rb +200 -0
- data/lib/carray/math.rb +78 -342
- data/lib/carray/meld_reduce.rb +289 -0
- data/lib/carray/methods/align_addr.rb +116 -0
- data/lib/carray/methods/bin.rb +128 -0
- data/lib/carray/methods/bincount.rb +87 -0
- data/lib/carray/methods/bit_string.rb +92 -0
- data/lib/carray/methods/broadcast.rb +63 -0
- data/lib/carray/methods/choose.rb +39 -0
- data/lib/carray/methods/composition.rb +280 -0
- data/lib/carray/methods/gather_nd.rb +206 -0
- data/lib/carray/methods/index.rb +39 -0
- data/lib/carray/methods/insert_block.rb +99 -0
- data/lib/carray/methods/is_in.rb +141 -0
- data/lib/carray/methods/join.rb +90 -0
- data/lib/carray/methods/locate_addr.rb +47 -0
- data/lib/carray/methods/mask_duplicates.rb +41 -0
- data/lib/carray/methods/meshgrid.rb +91 -0
- data/lib/carray/methods/mode.rb +126 -0
- data/lib/carray/methods/nunique.rb +46 -0
- data/lib/carray/methods/resize.rb +56 -0
- data/lib/carray/methods/snap.rb +156 -0
- data/lib/carray/methods/string_format.rb +57 -0
- data/lib/carray/methods/unique.rb +47 -0
- data/lib/carray/methods/value_counts.rb +71 -0
- data/lib/carray/mkmf.rb +124 -101
- data/lib/carray/runtime.rb +108 -0
- data/lib/carray/serialize.rb +478 -167
- data/lib/carray/slab_iterator.rb +292 -0
- data/lib/carray/stack.rb +291 -0
- data/lib/carray/string.rb +56 -180
- data/lib/carray/string_operation_extension.rb +289 -0
- data/lib/carray/struct.rb +335 -323
- data/lib/carray/struct_builder.rb +697 -0
- data/lib/carray/table.rb +41 -2
- data/lib/carray/time.rb +2255 -38
- data/lib/carray/window_iterator.rb +655 -0
- data/lib/carray.rb +55 -57
- metadata +163 -130
- data/Rakefile +0 -51
- data/TODO.md +0 -18
- data/ext/ca_iter_block.c +0 -257
- data/ext/ca_iter_dimension.c +0 -299
- data/ext/ca_iter_window.c +0 -214
- data/ext/ca_obj_mapping.c +0 -644
- data/ext/carray_iterator.c +0 -641
- data/ext/carray_math.rb +0 -850
- data/ext/carray_numeric.c +0 -259
- data/ext/carray_sort_addr.c +0 -254
- data/ext/carray_stat.c +0 -2100
- data/ext/carray_stat_proc.rb +0 -1999
- data/ext/mkmath.rb +0 -741
- data/ext/ruby_ccomplex.c +0 -509
- data/ext/ruby_float_func.c +0 -86
- data/lib/carray/array.rb +0 -8
- data/lib/carray/autoload/autoload_base.rb +0 -19
- data/lib/carray/autoload/autoload_gem_cairo.rb +0 -9
- data/lib/carray/autoload/autoload_gem_ffi.rb +0 -9
- data/lib/carray/autoload/autoload_gem_gnuplot.rb +0 -2
- data/lib/carray/autoload/autoload_gem_io_csv.rb +0 -14
- data/lib/carray/autoload/autoload_gem_io_pg.rb +0 -6
- data/lib/carray/autoload/autoload_gem_io_sqlite3.rb +0 -12
- data/lib/carray/autoload/autoload_gem_narray.rb +0 -10
- data/lib/carray/autoload/autoload_gem_numo_narray.rb +0 -15
- data/lib/carray/autoload/autoload_gem_opencv.rb +0 -16
- data/lib/carray/autoload/autoload_gem_random.rb +0 -8
- data/lib/carray/autoload/autoload_gem_rmagick.rb +0 -23
- data/lib/carray/autoload/autoload_gem_zimg.rb +0 -3
- data/lib/carray/autoload/autoload_io_imagemagick.rb +0 -6
- data/lib/carray/autoload/autoload_math_histogram.rb +0 -5
- data/lib/carray/autoload/autoload_math_recurrence.rb +0 -6
- data/lib/carray/autoload/autoload_object_iterator.rb +0 -1
- data/lib/carray/autoload/autoload_object_link.rb +0 -1
- data/lib/carray/autoload/autoload_object_pack.rb +0 -2
- data/lib/carray/autoload.rb +0 -141
- data/lib/carray/basic.rb +0 -191
- data/lib/carray/broadcast.rb +0 -101
- data/lib/carray/compose.rb +0 -315
- data/lib/carray/convert.rb +0 -115
- data/lib/carray/info.rb +0 -110
- data/lib/carray/io/imagemagick.rb +0 -235
- data/lib/carray/mask.rb +0 -102
- data/lib/carray/math/histogram.rb +0 -177
- data/lib/carray/math/recurrence.rb +0 -93
- data/lib/carray/object/ca_obj_iterator.rb +0 -50
- data/lib/carray/object/ca_obj_link.rb +0 -50
- data/lib/carray/object/ca_obj_pack.rb +0 -99
- data/lib/carray/obsolete.rb +0 -256
- data/lib/carray/ordering.rb +0 -181
- data/lib/carray/testing.rb +0 -51
- data/lib/carray/transform.rb +0 -109
- data/mailmap +0 -1
- data/misc/Methods.ja.md +0 -182
- data/misc/NOTE +0 -51
- data/spec/Classes/CABitfield_spec.rb +0 -58
- data/spec/Classes/CABlockIterator_spec.rb +0 -114
- data/spec/Classes/CABlock_spec.rb +0 -205
- data/spec/Classes/CAField_spec.rb +0 -39
- data/spec/Classes/CAGrid_spec.rb +0 -75
- data/spec/Classes/CAMap_spec.rb +0 -0
- data/spec/Classes/CAMapping_spec.rb +0 -105
- data/spec/Classes/CAObject_attribute_spec.rb +0 -33
- data/spec/Classes/CAObject_spec.rb +0 -33
- data/spec/Classes/CARefer_spec.rb +0 -93
- data/spec/Classes/CARepeat_spec.rb +0 -65
- data/spec/Classes/CASelect_spec.rb +0 -22
- data/spec/Classes/CAShift_spec.rb +0 -16
- data/spec/Classes/CAStruct_spec.rb +0 -71
- data/spec/Classes/CATranspose_spec.rb +0 -60
- data/spec/Classes/CAUnboudRepeat_spec.rb +0 -102
- data/spec/Classes/CAWindow_spec.rb +0 -54
- data/spec/Classes/CAWrap_spec.rb +0 -8
- data/spec/Classes/CArray_spec.rb +0 -184
- data/spec/Classes/CScalar_spec.rb +0 -55
- data/spec/Features/feature_130_spec.rb +0 -19
- data/spec/Features/feature_attributes_spec.rb +0 -280
- data/spec/Features/feature_boolean_spec.rb +0 -98
- data/spec/Features/feature_broadcast.rb +0 -116
- data/spec/Features/feature_cast_function.rb +0 -19
- data/spec/Features/feature_cast_spec.rb +0 -33
- data/spec/Features/feature_class_spec.rb +0 -84
- data/spec/Features/feature_complex_spec.rb +0 -42
- data/spec/Features/feature_composite_spec.rb +0 -124
- data/spec/Features/feature_convert_spec.rb +0 -46
- data/spec/Features/feature_copy_spec.rb +0 -123
- data/spec/Features/feature_creation_spec.rb +0 -84
- data/spec/Features/feature_element_spec.rb +0 -144
- data/spec/Features/feature_extream_spec.rb +0 -54
- data/spec/Features/feature_generate_spec.rb +0 -74
- data/spec/Features/feature_index_spec.rb +0 -69
- data/spec/Features/feature_mask_spec.rb +0 -580
- data/spec/Features/feature_math_spec.rb +0 -97
- data/spec/Features/feature_order_spec.rb +0 -146
- data/spec/Features/feature_ref_store_spec.rb +0 -209
- data/spec/Features/feature_serialization_spec.rb +0 -125
- data/spec/Features/feature_stat_spec.rb +0 -397
- data/spec/Features/feature_virtual_spec.rb +0 -48
- data/spec/Features/method_eq_spec.rb +0 -81
- data/spec/Features/method_is_nan_spec.rb +0 -12
- data/spec/Features/method_map_spec.rb +0 -54
- data/spec/Features/method_max_with.rb +0 -20
- data/spec/Features/method_min_with.rb +0 -19
- data/spec/Features/method_ne_spec.rb +0 -18
- data/spec/Features/method_project_spec.rb +0 -188
- data/spec/Features/method_ref_spec.rb +0 -27
- data/spec/Features/method_round_spec.rb +0 -11
- data/spec/Features/method_s_linspace_spec.rb +0 -48
- data/spec/Features/method_s_span_spec.rb +0 -14
- data/spec/Features/method_seq_spec.rb +0 -47
- data/spec/Features/method_sort_with.rb +0 -43
- data/spec/Features/method_sorted_with.rb +0 -29
- data/spec/Features/method_span_spec.rb +0 -42
- data/spec/Features/method_wrap_readonly_spec.rb +0 -43
- data/spec/UnitTest/test_CAVirtual.rb +0 -214
- data/spec/spec_all.rb +0 -10
- data/utils/ca_ase.rb +0 -21
- data/utils/ca_methods.rb +0 -15
- data/utils/cast_checker.rb +0 -30
- data/utils/convert_test.rb +0 -73
- data/utils/extract_yard.rb +0 -22
- data/utils/guess_shape.rb +0 -76
- data/utils/monkey_patch_methods.rb +0 -62
- data/utils/remove_resource_fork.sh +0 -5
|
@@ -0,0 +1,4321 @@
|
|
|
1
|
+
/* ---------------------------------------------------------------------------
|
|
2
|
+
|
|
3
|
+
T1 kernel_iterator MVP — Phase 1 step 1 + step 2 + step 3 implementation.
|
|
4
|
+
|
|
5
|
+
Scope after step 3 (PROPOSAL_T1_KERNEL_ITERATOR.md §10.3):
|
|
6
|
+
- ca_iter_state struct + init / next_slab / next_slab_strided /
|
|
7
|
+
finish state machine
|
|
8
|
+
- CA_SLAB_WHOLE policy only (step 5+ adds AXES / FREE)
|
|
9
|
+
- level == 1 (L1, contig kernel):
|
|
10
|
+
- entity / CAStride contig: alias path (single slab, alias_ptr
|
|
11
|
+
= parent->ptr, stride implicit = bytes)
|
|
12
|
+
- CAStride family non-contig: scratch path (ca_copy_data
|
|
13
|
+
compose-fold gather into a malloc'd buffer)
|
|
14
|
+
- other sources: CA_ITER_ERR_NOT_CHEAP
|
|
15
|
+
- level == 2 (L2, strided kernel):
|
|
16
|
+
- entity / CAStride contig: alias path (single slab,
|
|
17
|
+
stride_bytes = bytes)
|
|
18
|
+
- CAStride family non-contig: alias_strided path — no scratch,
|
|
19
|
+
per-outer-prefix yield with native inner stride_bytes
|
|
20
|
+
- other sources: CA_ITER_ERR_NOT_CHEAP
|
|
21
|
+
- READ-only (flags == 0). WRITE = step 4, NO_MASK = step 7.
|
|
22
|
+
|
|
23
|
+
---------------------------------------------------------------------------- */
|
|
24
|
+
|
|
25
|
+
#include "carray.h"
|
|
26
|
+
#include "carray_internal.h" /* per-obj_type view constructors */
|
|
27
|
+
#include "ca_kernel_iterator.h"
|
|
28
|
+
#include "ca_monop_dispatch.h" /* P.6.2.d: ca_monop_view_is_single_cast for F.6.2 gate */
|
|
29
|
+
#include "ca_obj_face.h" /* PROPOSAL_CAFACE_PHASE_2 F.2.6 — ca_strip_face for SRC_* entry */
|
|
30
|
+
|
|
31
|
+
#include <assert.h>
|
|
32
|
+
#include <string.h>
|
|
33
|
+
|
|
34
|
+
/* Defined in carray_core.c. (ca_is_readonly / ca_has_mask /
|
|
35
|
+
ca_sync_data are already declared in carray.h.) */
|
|
36
|
+
extern int ca_attach_is_alias (void *ap);
|
|
37
|
+
extern int ca_root_lends_no_memory (void *ap);
|
|
38
|
+
|
|
39
|
+
/* CAStride family attach-fn marker (step 1-4). */
|
|
40
|
+
extern ca_operation_function_t ca_stride_func;
|
|
41
|
+
|
|
42
|
+
/* Descriptor framework view ops (step 5+). Each view's describe_axes
|
|
43
|
+
emits the per-axis descriptor + parent dim snapshot consumed by
|
|
44
|
+
the P3 ca_axis_dispatch_* substrate. CAShift uses ca_window_func
|
|
45
|
+
(Phase G typedef pattern, ca_shift_func is a copy with only free /
|
|
46
|
+
clone / create_mask overridden) so it routes via attach pointer
|
|
47
|
+
match against ca_window_func.attach. */
|
|
48
|
+
extern ca_operation_function_t ca_select_axis_func;
|
|
49
|
+
extern ca_operation_function_t ca_grid_func;
|
|
50
|
+
extern ca_operation_function_t ca_select_func;
|
|
51
|
+
extern ca_operation_function_t ca_window_func;
|
|
52
|
+
/* ca_mapping_func retired in R.3 (PROPOSAL_CAMAPPING_REMOVAL). */
|
|
53
|
+
|
|
54
|
+
/* SRC_ATTACH 5 view (step 9). Each defines its own attach that
|
|
55
|
+
materialises src->ptr via view-specific transform (cast / swap /
|
|
56
|
+
bit unpack / reduction). kernel_iterator treats them uniformly:
|
|
57
|
+
ca_attach(src) → kernel sees src->ptr as contig → on WRITE,
|
|
58
|
+
ca_sync(src) lets the view's sync_data scatter back. */
|
|
59
|
+
extern ca_operation_function_t ca_fake_func;
|
|
60
|
+
extern ca_operation_function_t ca_byte_swap_func;
|
|
61
|
+
extern ca_operation_function_t ca_bitfield_func;
|
|
62
|
+
extern ca_operation_function_t ca_bitarray_func;
|
|
63
|
+
extern ca_operation_function_t ca_reduce_func;
|
|
64
|
+
extern ca_operation_function_t ca_object_func;
|
|
65
|
+
|
|
66
|
+
/* F-2 follow-up (2026-05-26): connect CATile / CARoll to kernel_iterator
|
|
67
|
+
via the SRC_ATTACH pattern. Per-cell modulo wrap does not fit the
|
|
68
|
+
descriptor framework's kind enum {STRIDE, INDEX, SHIFT}, so the
|
|
69
|
+
innermost-STRIDE L2 alias path is not used; but the view-specific
|
|
70
|
+
func_attach (embed-region gather) materialises src->ptr and func_sync
|
|
71
|
+
scatters back — structurally identical to SRC_ATTACH. Fills the gap
|
|
72
|
+
in the "deliver" principle (= these were previously rejected via
|
|
73
|
+
SRC_NONE). */
|
|
74
|
+
extern ca_operation_function_t ca_tile_func;
|
|
75
|
+
extern ca_operation_function_t ca_roll_func;
|
|
76
|
+
|
|
77
|
+
/* PROPOSAL_CASTACK.md Phase 3 (2026-06-18): CAStack via SRC_ATTACH.
|
|
78
|
+
func_attach materialises src->ptr via per-parent ca_xfer_all GET
|
|
79
|
+
(= K * parent.bytes alloc, caller responsibility per MEMO §3.4);
|
|
80
|
+
func_sync scatters back via xfer_all PUT. kernel sees a flat
|
|
81
|
+
contig slab over the stacked output of shape (K, *parent_shape). */
|
|
82
|
+
extern ca_operation_function_t ca_stack_func;
|
|
83
|
+
extern ca_operation_function_t ca_meld_func;
|
|
84
|
+
|
|
85
|
+
/* M.6 (PROPOSAL_CAREMAP_INTERNAL.md §5.2): CARemap is internal-only
|
|
86
|
+
(no Ruby class constant) but participates in kernel_iterator as an
|
|
87
|
+
SRC_ATTACH source. Per-element gather has no STRIDE structure to
|
|
88
|
+
preserve, so SRC_ATTACH (scratch materialise via ca_remap_func_attach)
|
|
89
|
+
is the natural acceptance path. func_attach allocates a fresh
|
|
90
|
+
buffer and runs xfer_all(GET); kernel sees a flat contig slab.
|
|
91
|
+
On WRITE, ca_sync routes through xfer_all(PUT). */
|
|
92
|
+
extern ca_operation_function_t ca_remap_func;
|
|
93
|
+
|
|
94
|
+
/* PROPOSAL_LAZY_ELEMENTWISE_VIEW Phase 4.5 P.4.5.1 (2026-06-07): lazy
|
|
95
|
+
element-wise view family. CAMonOp (Phase 1) / CABinOp (Phase 2) /
|
|
96
|
+
CABinCmp + CAMonCmp (Phase 4) all expose a func_attach that pulls
|
|
97
|
+
the lazy tree's materialise into a fresh contig buffer; kernel sees
|
|
98
|
+
a flat slab. Same SRC_ATTACH structural pattern as CAFake et al,
|
|
99
|
+
so a 4-line addition to the classify_source list opens all 22
|
|
100
|
+
mkkernel-generated reduction ops (sum / count / mean / variance /
|
|
101
|
+
argmin / ...) to lazy operands — `(a.lazy + b).sum` now works. */
|
|
102
|
+
extern ca_operation_function_t ca_monop_func;
|
|
103
|
+
extern ca_operation_function_t ca_binop_func;
|
|
104
|
+
extern ca_operation_function_t ca_bincmp_func;
|
|
105
|
+
extern ca_operation_function_t ca_moncmp_func;
|
|
106
|
+
extern CArray *ca_remap_new (CArray *ref, CArray *idx);
|
|
107
|
+
|
|
108
|
+
/* ca_reduce_new is declared in carray.h; rb_cCAReduce defined in
|
|
109
|
+
ca_obj_reduce.c — needed by 9.3 smoke helpers since CAReduce has
|
|
110
|
+
no public Ruby surface. */
|
|
111
|
+
extern VALUE rb_cCAReduce;
|
|
112
|
+
extern void ca_select_axis_describe_axes (void *ap, ca_axis_desc_t *out,
|
|
113
|
+
ca_size_t *out_parent_dims);
|
|
114
|
+
extern void ca_grid_describe_axes (void *ap, ca_axis_desc_t *out,
|
|
115
|
+
ca_size_t *out_parent_dims);
|
|
116
|
+
extern void ca_select_describe_axes (void *ap, ca_axis_desc_t *out,
|
|
117
|
+
ca_size_t *out_parent_dims);
|
|
118
|
+
extern void ca_window_describe_axes (void *ap, ca_axis_desc_t *out,
|
|
119
|
+
ca_size_t *out_parent_dims);
|
|
120
|
+
|
|
121
|
+
/* ---- helpers -------------------------------------------------------- */
|
|
122
|
+
|
|
123
|
+
/* True iff src is a CAStride-family view (contig or not). Entity
|
|
124
|
+
arrays are not CAStride family — they're tested separately. */
|
|
125
|
+
static int
|
|
126
|
+
ca_iter_is_castride_family (CArray *src)
|
|
127
|
+
{
|
|
128
|
+
if ( src == NULL ) return 0;
|
|
129
|
+
return ca_func[src->obj_type].attach == ca_stride_func.attach;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/* Source kinds declared by view classes installed from outside the core
|
|
133
|
+
(ca_install_obj_type). Sized to CA_OBJ_TYPE_MAX so any obj_type can be
|
|
134
|
+
indexed directly; file-scope zero-init leaves unregistered slots at
|
|
135
|
+
CA_ITER_SRC_NONE (= 0), which is exactly "the classifier decides".
|
|
136
|
+
See ca_kernel_iterator.h for the contract an external class accepts by
|
|
137
|
+
registering. */
|
|
138
|
+
static uint8_t ca_iter_registered_source_kind[CA_OBJ_TYPE_MAX];
|
|
139
|
+
|
|
140
|
+
void
|
|
141
|
+
ca_iter_register_source_kind (int obj_type, uint8_t kind)
|
|
142
|
+
{
|
|
143
|
+
if ( obj_type < 0 || obj_type >= CA_OBJ_TYPE_MAX ) {
|
|
144
|
+
rb_raise(rb_eArgError,
|
|
145
|
+
"ca_iter_register_source_kind: obj_type %d out of range",
|
|
146
|
+
obj_type);
|
|
147
|
+
}
|
|
148
|
+
if ( kind != CA_ITER_SRC_ATTACH ) {
|
|
149
|
+
rb_raise(rb_eArgError,
|
|
150
|
+
"ca_iter_register_source_kind: only CA_ITER_SRC_ATTACH (%d) "
|
|
151
|
+
"may be registered, got %d",
|
|
152
|
+
CA_ITER_SRC_ATTACH, (int) kind);
|
|
153
|
+
}
|
|
154
|
+
ca_iter_registered_source_kind[obj_type] = kind;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/* Classify the source by routing kind (proposal §1 strategy table).
|
|
158
|
+
Returns CA_ITER_SRC_NONE for sources not yet supported. */
|
|
159
|
+
static uint8_t
|
|
160
|
+
ca_iter_classify_source (CArray *src)
|
|
161
|
+
{
|
|
162
|
+
if ( src == NULL ) return CA_ITER_SRC_NONE;
|
|
163
|
+
if ( ca_is_entity(src) ) return CA_ITER_SRC_CASTRIDE;
|
|
164
|
+
if ( ca_iter_is_castride_family(src) ) return CA_ITER_SRC_CASTRIDE;
|
|
165
|
+
|
|
166
|
+
/* Externally installed obj_types declare their routing (2026-08-07).
|
|
167
|
+
Placed after the two structural cases and before the built-in list:
|
|
168
|
+
entity and CAStride-family sources are recognised from the struct
|
|
169
|
+
itself and are read directly, so a registration must not be able to
|
|
170
|
+
divert them onto a materialising path; everything below is a lookup
|
|
171
|
+
of one operation table against another, and a class that registered
|
|
172
|
+
is answering exactly that question about itself. */
|
|
173
|
+
{
|
|
174
|
+
uint8_t kind = ca_iter_registered_source_kind[src->obj_type];
|
|
175
|
+
if ( kind != CA_ITER_SRC_NONE ) return kind;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/* Descriptor framework views (step 5.1: CSA + CAGrid; 5.2: + CASelect /
|
|
179
|
+
CAMapping / CAWindow / CAShift). CAShift uses ca_window_func
|
|
180
|
+
for attach (Phase G typedef pattern) so it matches the CAWindow
|
|
181
|
+
check below. */
|
|
182
|
+
void *attach = ca_func[src->obj_type].attach;
|
|
183
|
+
if ( attach == ca_select_axis_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
184
|
+
if ( attach == ca_grid_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
185
|
+
if ( attach == ca_select_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
186
|
+
if ( attach == ca_window_func.attach ) return CA_ITER_SRC_DESCRIPTOR; /* + CAShift */
|
|
187
|
+
|
|
188
|
+
/* Step 9: SRC_ATTACH 5 view. Each view-specific attach materialises
|
|
189
|
+
src->ptr via per-element transform; kernel sees a flat contig slab.
|
|
190
|
+
CAUnboundRepeat shares ca_stride_func.attach so it was already
|
|
191
|
+
classified as SRC_CASTRIDE above (prep doc §2.6). */
|
|
192
|
+
if ( attach == ca_fake_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
193
|
+
if ( attach == ca_byte_swap_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
194
|
+
if ( attach == ca_bitfield_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
195
|
+
if ( attach == ca_bitarray_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
196
|
+
if ( attach == ca_reduce_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
197
|
+
/* Step 11: CAObject — Ruby callback per-element bridge. Same
|
|
198
|
+
SRC_ATTACH structural pattern (func_attach materialises via Ruby
|
|
199
|
+
copy_data, func_sync scatters back via Ruby sync_data, CA_FLAG_
|
|
200
|
+
READ_ONLY auto-rejects WRITE via ca_is_readonly). Bench gate
|
|
201
|
+
n/a (Ruby callback overhead structurally dominant). */
|
|
202
|
+
if ( attach == ca_object_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
203
|
+
|
|
204
|
+
/* F-2 follow-up: CATile / CARoll via SRC_ATTACH (embed-region gather
|
|
205
|
+
materialises src->ptr; ca_sync scatters back per view-specific
|
|
206
|
+
semantics — CATile tile decomposition, CARoll cyclic permutation). */
|
|
207
|
+
if ( attach == ca_tile_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
208
|
+
if ( attach == ca_roll_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
209
|
+
|
|
210
|
+
/* Phase 3 (PROPOSAL_CASTACK.md): CAStack via SRC_ATTACH. func_attach
|
|
211
|
+
materialises src->ptr via per-parent ca_xfer_all GET (= K * parent
|
|
212
|
+
bytes alloc, caller responsibility per MEMO §3.4); func_sync
|
|
213
|
+
scatters back via xfer_all PUT. kernel sees a flat contig slab
|
|
214
|
+
over the stacked output. Routine ndim mismatch is irrelevant
|
|
215
|
+
here (= attach delivers a self-owned buffer of shape (K, *parent
|
|
216
|
+
shape), kernel reads/writes that buffer). */
|
|
217
|
+
if ( attach == ca_stack_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
218
|
+
|
|
219
|
+
/* CAMeld — ragged concatenate along an existing axis. func_attach
|
|
220
|
+
materialises via K per-parent xfer_all GET into a contig buffer at
|
|
221
|
+
seg_offset[k] * tail_bytes offsets; func_sync scatters back. Same
|
|
222
|
+
SRC_ATTACH structural pattern as CAStack; reduce hot paths bypass
|
|
223
|
+
this via the per-parent decompose in lib/carray/meld_reduce.rb. */
|
|
224
|
+
if ( attach == ca_meld_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
225
|
+
|
|
226
|
+
/* M.6: CARemap — per-element gather, internal-only. Same SRC_ATTACH
|
|
227
|
+
structural pattern (func_attach materialises via xfer_all GET,
|
|
228
|
+
func_sync via xfer_all PUT). */
|
|
229
|
+
if ( attach == ca_remap_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
230
|
+
|
|
231
|
+
/* Phase 4.5 P.4.5.1: lazy view family — CAMonOp / CABinOp / CABinCmp /
|
|
232
|
+
CAMonCmp. func_attach pulls the lazy chain materialise (arena-pooled
|
|
233
|
+
scratches under the hood from Phase 3); kernel sees a flat contig
|
|
234
|
+
slab. Read-only (= CA_FLAG_READ_ONLY) so WRITE auto-rejects at
|
|
235
|
+
the ca_is_readonly check. Opens 22 mkkernel-generated reduction
|
|
236
|
+
ops to lazy operands. */
|
|
237
|
+
if ( attach == ca_monop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
238
|
+
if ( attach == ca_binop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
239
|
+
if ( attach == ca_bincmp_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
240
|
+
if ( attach == ca_moncmp_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
241
|
+
|
|
242
|
+
return CA_ITER_SRC_NONE;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/* F-2 (PROPOSAL_F2_KERNEL_ITERATOR_ALIAS rev6): route a source by
|
|
246
|
+
running classify_source and, for descriptor-routed views, also calling
|
|
247
|
+
describe_axes to inspect the innermost axis kind. Returns the refined
|
|
248
|
+
src_kind (= SRC_DESCRIPTOR_L2_ALIASABLE iff innermost axis is STRIDE,
|
|
249
|
+
else SRC_DESCRIPTOR). out_descs / out_parent_dims / out_ndim are
|
|
250
|
+
populated for descriptor sources so the caller (init_l1 / init_l2)
|
|
251
|
+
does not have to re-call describe_axes. For non-descriptor sources
|
|
252
|
+
(CASTRIDE / ATTACH / NONE) the out_* arguments are not touched and
|
|
253
|
+
classify_source's verdict is returned as-is.
|
|
254
|
+
|
|
255
|
+
Cost analysis: classify_source is O(1) pointer compares. For
|
|
256
|
+
SRC_DESCRIPTOR candidates we add one describe_axes call (O(ndim) with
|
|
257
|
+
ndim ≤ CA_RANK_MAX = 16) plus one innermost-axis kind compare.
|
|
258
|
+
Per-walk overhead = 1 describe_axes call (descriptor sources only),
|
|
259
|
+
which init was going to do anyway -- so the routing is net zero cost
|
|
260
|
+
compared to the pre-rev6 path where classify_source + init both
|
|
261
|
+
computed describe_axes redundantly. See prep doc rev2 §4.1.1. */
|
|
262
|
+
static uint8_t ca_iter_classify_source (CArray *src);
|
|
263
|
+
static void ca_iter_describe_axes (CArray *src, ca_axis_desc_t *,
|
|
264
|
+
ca_size_t *, int8_t *);
|
|
265
|
+
|
|
266
|
+
static uint8_t
|
|
267
|
+
ca_iter_route_source (CArray *src,
|
|
268
|
+
ca_axis_desc_t *out_descs,
|
|
269
|
+
ca_size_t *out_parent_dims,
|
|
270
|
+
int8_t *out_ndim)
|
|
271
|
+
{
|
|
272
|
+
uint8_t kind = ca_iter_classify_source(src);
|
|
273
|
+
if ( kind != CA_ITER_SRC_DESCRIPTOR ) return kind;
|
|
274
|
+
|
|
275
|
+
/* Descriptor source: describe_axes + inspect innermost. */
|
|
276
|
+
ca_iter_describe_axes(src, out_descs, out_parent_dims, out_ndim);
|
|
277
|
+
if ( ca_axis_dispatch_is_innermost_stride(out_descs, *out_ndim) ) {
|
|
278
|
+
return CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE;
|
|
279
|
+
}
|
|
280
|
+
return CA_ITER_SRC_DESCRIPTOR;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/* Dispatch to the view's describe_axes. Routing keyed on the
|
|
284
|
+
shared attach pointer (CAShift shares with CAWindow per Phase G
|
|
285
|
+
typedef). */
|
|
286
|
+
static void
|
|
287
|
+
ca_iter_describe_axes (CArray *src,
|
|
288
|
+
ca_axis_desc_t *out_descs,
|
|
289
|
+
ca_size_t *out_parent_dims,
|
|
290
|
+
int8_t *out_ndim)
|
|
291
|
+
{
|
|
292
|
+
void *attach = ca_func[src->obj_type].attach;
|
|
293
|
+
if ( attach == ca_select_axis_func.attach ) {
|
|
294
|
+
ca_select_axis_describe_axes(src, out_descs, out_parent_dims);
|
|
295
|
+
*out_ndim = src->ndim;
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
if ( attach == ca_grid_func.attach ) {
|
|
299
|
+
ca_grid_describe_axes(src, out_descs, out_parent_dims);
|
|
300
|
+
*out_ndim = src->ndim;
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
if ( attach == ca_select_func.attach ) {
|
|
304
|
+
ca_select_describe_axes(src, out_descs, out_parent_dims);
|
|
305
|
+
/* CASelect emits 1-D INDEX descriptor per its describe_axes
|
|
306
|
+
contract. src->ndim should reflect the selector-flattened
|
|
307
|
+
view shape; engine reads ndim from this argument. */
|
|
308
|
+
*out_ndim = src->ndim;
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
if ( attach == ca_window_func.attach ) {
|
|
312
|
+
ca_window_describe_axes(src, out_descs, out_parent_dims);
|
|
313
|
+
*out_ndim = src->ndim;
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
/* unreachable: validate_inputs already gated through ca_iter_classify_source */
|
|
317
|
+
*out_ndim = 0;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/* Public alias eligibility predicate (proposal §11.3 case (a)). */
|
|
321
|
+
int
|
|
322
|
+
ca_iter_can_alias (void *ap, int level)
|
|
323
|
+
{
|
|
324
|
+
CArray *ca = (CArray *) ap;
|
|
325
|
+
if ( ca == NULL ) return 0;
|
|
326
|
+
|
|
327
|
+
switch ( level ) {
|
|
328
|
+
case 1:
|
|
329
|
+
return ca_attach_is_alias(ca);
|
|
330
|
+
|
|
331
|
+
case 2:
|
|
332
|
+
if ( ca_is_entity(ca) ) return 1;
|
|
333
|
+
return ca_iter_is_castride_family(ca);
|
|
334
|
+
|
|
335
|
+
default:
|
|
336
|
+
/* L3 (and any future level) — not implemented in Phase 1; L1
|
|
337
|
+
fallback so the predicate stays well-defined for callers that
|
|
338
|
+
probe ahead. */
|
|
339
|
+
return ca_attach_is_alias(ca);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/* Build row-major byte strides for an entity-shape (CA_RANK_MAX). */
|
|
344
|
+
static void
|
|
345
|
+
ca_iter_build_rowmajor_strides (ca_size_t *strides,
|
|
346
|
+
const ca_size_t *dim,
|
|
347
|
+
int8_t ndim,
|
|
348
|
+
ca_size_t bytes)
|
|
349
|
+
{
|
|
350
|
+
ca_size_t s = bytes;
|
|
351
|
+
int8_t k;
|
|
352
|
+
for ( k = ndim - 1; k >= 0; k-- ) {
|
|
353
|
+
strides[k] = s;
|
|
354
|
+
s *= dim[k];
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/* ---- PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 dispatch predicate -------- */
|
|
359
|
+
|
|
360
|
+
/* Hybrid (rev2 Q2): coarse src_kind branch + view-specific func pointer
|
|
361
|
+
probe + fiber-axis effective stride check (Q3).
|
|
362
|
+
F.6.1 substrate: returns 0 (= disabled) for all sources. F.6.2+
|
|
363
|
+
(CAFake / CAByteSwap / CAShift / CAWindow / etc.) progressively enable
|
|
364
|
+
specific source kinds with bench-driven justification.
|
|
365
|
+
|
|
366
|
+
The fiber_axis_stride argument is the effective byte stride of the
|
|
367
|
+
fiber axis on the view (= what next_slab_axes would yield as
|
|
368
|
+
slab_strides[0]). Q3-equivalence: fiber_axis_stride == src->bytes
|
|
369
|
+
means the fiber is parent-memory contig, which is the precondition
|
|
370
|
+
for X.1/X.4 per-region fused paths to deliver a 1-pass result.
|
|
371
|
+
|
|
372
|
+
Called from init_l2 SRC_ATTACH branch right before the whole-view
|
|
373
|
+
ca_xfer_all GET. When returning 1, caller skips xfer_all GET and
|
|
374
|
+
sets alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED with fiber dispatch
|
|
375
|
+
state populated. */
|
|
376
|
+
static int
|
|
377
|
+
ca_iter_should_per_fiber_fused (CArray *src,
|
|
378
|
+
int src_kind,
|
|
379
|
+
int8_t fiber_axis,
|
|
380
|
+
ca_size_t fiber_axis_stride,
|
|
381
|
+
uint32_t flags)
|
|
382
|
+
{
|
|
383
|
+
(void) fiber_axis;
|
|
384
|
+
(void) flags;
|
|
385
|
+
|
|
386
|
+
/* F.6.2: CAFake / CAByteSwap (transform-fused, X.4 per-region).
|
|
387
|
+
Enable when fiber-axis effective stride == view cell bytes
|
|
388
|
+
(= Q3: fiber is parent-memory contig run, so ca_xfer_stride
|
|
389
|
+
routes into ca_xfer_stride_transform_fused inner-contig fast
|
|
390
|
+
path for 1-pass per-fiber delivery).
|
|
391
|
+
|
|
392
|
+
F.5 bench: innermost-axis fiber 30-35% faster than whole-view
|
|
393
|
+
materialise (CAFake) / 16-18% (CAByteSwap). Non-innermost (=
|
|
394
|
+
fiber_axis_stride != bytes) was a loser, so the stride gate
|
|
395
|
+
keeps current path for that case.
|
|
396
|
+
|
|
397
|
+
Phase 6 P.6.2.d (Q13 α): single-cast CAMonOp (= post-migration
|
|
398
|
+
successor of CAFake numeric path) is recognised via
|
|
399
|
+
ca_monop_view_is_single_cast and routed through the same F.6.2
|
|
400
|
+
fast path. Chain CAMonOp (depth ≥ 2) is excluded — its
|
|
401
|
+
materialise path (= arena-pooled scratches from Phase 3) is
|
|
402
|
+
structurally different from the X.4 inner-contig fast path. */
|
|
403
|
+
void *attach = ca_func[src->obj_type].attach;
|
|
404
|
+
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
405
|
+
if ( attach == ca_fake_func.attach
|
|
406
|
+
|| attach == ca_byte_swap_func.attach
|
|
407
|
+
|| ca_monop_view_is_single_cast(src) ) {
|
|
408
|
+
return fiber_axis_stride == (ca_size_t) src->bytes;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/* F.6.3: CAShift / CAWindow with OOB-fused materialise path (X.1).
|
|
413
|
+
Per-fiber ca_xfer_stride wins decisively when the view would
|
|
414
|
+
otherwise fall into a per-slab gather / whole-view materialise
|
|
415
|
+
path (= SRC_DESCRIPTOR, e.g. CAShift always, CAWindow with SHIFT
|
|
416
|
+
axes). But when the view qualifies for the L2 alias fast path
|
|
417
|
+
(SRC_DESCRIPTOR_L2_ALIASABLE, = interior-only CAWindow with F-1
|
|
418
|
+
STRIDE promotion), the existing alias path is already optimal
|
|
419
|
+
(parent.ptr + composed offsets, zero materialise). Per-fiber
|
|
420
|
+
dispatch in that case adds ca_xfer_stride per-call overhead with
|
|
421
|
+
no payoff -- F.6.3 bench measured 0.35-0.82x (regression). Gate
|
|
422
|
+
on SRC_DESCRIPTOR only. */
|
|
423
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
424
|
+
/* Both CAWindow and CAShift match ca_window_func.attach (Phase G
|
|
425
|
+
typedef pattern: ca_shift_func = ca_window_func). */
|
|
426
|
+
if ( attach == ca_window_func.attach ) {
|
|
427
|
+
return 1;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/* F.6.4 audit (devel/bench_f6_4_audit.rb): CSA (CASelectAxis) /
|
|
432
|
+
CAGrid / CASelect / CAMapping all kept on the whole-view
|
|
433
|
+
materialise path. Per-fiber ca_xfer_stride is a LOSER for
|
|
434
|
+
these views (1.25-4.7x slower across all axis positions
|
|
435
|
+
measured) because the descriptor framework's
|
|
436
|
+
ca_axis_dispatch_gather is already a single batched per-axis
|
|
437
|
+
kind gather; splitting into N per-fiber calls loses the
|
|
438
|
+
batched throughput. Structurally different from
|
|
439
|
+
CAFake/CAByteSwap (transform-fused at byte level, X.4) and
|
|
440
|
+
CAShift/CAWindow OOB (per-region fused with bound check, X.1):
|
|
441
|
+
descriptor framework views do not have a per-region fused fast
|
|
442
|
+
path designed for per-fiber entry.
|
|
443
|
+
|
|
444
|
+
Note (rev4): bench split fiber-axis kind shows STRIDE fiber
|
|
445
|
+
1.25x slower vs INDEX fiber 4-5x slower. STRIDE-fiber case
|
|
446
|
+
is close to parity and could conceivably reach win with finer
|
|
447
|
+
gating (= bypass per-axis kind classification when the outer
|
|
448
|
+
prefix is homogeneous STRIDE). Not pursued at current 1.25x
|
|
449
|
+
deficit; recorded as a future possible refinement should an
|
|
450
|
+
application motivate it. See proposal §3.1 SRC_DESCRIPTOR
|
|
451
|
+
(CSA, CAGrid, ...) inline comment.
|
|
452
|
+
|
|
453
|
+
F.6.6 audit (devel/bench_f6_6_bit_audit.rb): CABitarray /
|
|
454
|
+
CABitfield kept on current path. Bench measured via
|
|
455
|
+
bits.fake(:float64) (= production CAFake-wrap hot path) shows
|
|
456
|
+
per-fiber 1.23-5.86x slower on the F.6.2-gate-off axes.
|
|
457
|
+
|
|
458
|
+
The CAFake wrap overhead applies equally to both per-fiber and
|
|
459
|
+
whole-view paths (same total cell-cast cost), so the 5x ratio
|
|
460
|
+
is essentially the CABitarray whole-view-vs-per-fiber ratio
|
|
461
|
+
itself. Phase A bench note "xfer_stride 0.98x parity" is a
|
|
462
|
+
single-call comparison: N per-fiber calls cumulatively cost
|
|
463
|
+
N x parity > 1 x xfer_all (+ N dispatch overhead + per-call
|
|
464
|
+
bit alignment math). This reasoning generalises to direct
|
|
465
|
+
CABitarray / CABitfield source (= when a non-f64 fiber smoke
|
|
466
|
+
is added in a future phase) -- no separate audit needed for
|
|
467
|
+
that case. Predicate stays unchanged.
|
|
468
|
+
|
|
469
|
+
F.6.7 will audit CAReduce. */
|
|
470
|
+
return 0;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/* ---- state machine -------------------------------------------------- */
|
|
474
|
+
|
|
475
|
+
/* Shared input validation for init_l1 / init_l2. Returns CA_ITER_OK
|
|
476
|
+
if the (src, policy, flags) tuple is acceptable, else the
|
|
477
|
+
corresponding CA_ITER_ERR_*. */
|
|
478
|
+
static int
|
|
479
|
+
ca_iter_validate_inputs (ca_iter_state *st,
|
|
480
|
+
struct _CArray *src,
|
|
481
|
+
ca_slab_policy_t policy,
|
|
482
|
+
uint32_t flags)
|
|
483
|
+
{
|
|
484
|
+
if ( st == NULL || src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
485
|
+
/* Phase A capstone: CA_SLAB_AXES accepted (init_l2 path only — init_l1
|
|
486
|
+
does not implement CA_SLAB_AXES yet, see init_l1's policy gate). */
|
|
487
|
+
if ( policy != CA_SLAB_WHOLE && policy != CA_SLAB_AXES )
|
|
488
|
+
return CA_ITER_ERR_POLICY;
|
|
489
|
+
|
|
490
|
+
/* Step 6: accept CA_KERNEL_WRITE and CA_KERNEL_NO_MASK.
|
|
491
|
+
CA_KERNEL_CHUNK_HINT is reserved for T2 — still rejected.
|
|
492
|
+
NO_MASK enforcement (= reject masked source if NO_MASK is set)
|
|
493
|
+
lands in step 7; step 6 accepts the flag but does not enforce.
|
|
494
|
+
CA_KERNEL_FIBER_CONTIG (PROPOSAL_FIBER_DELIVERY F.1a) accepted —
|
|
495
|
+
activates per-fiber contig delivery for naxes==1 in next_slab_axes. */
|
|
496
|
+
const uint32_t accepted = CA_KERNEL_WRITE | CA_KERNEL_NO_MASK
|
|
497
|
+
| CA_KERNEL_FIBER_CONTIG;
|
|
498
|
+
if ( flags & ~accepted ) return CA_ITER_ERR_FLAGS;
|
|
499
|
+
|
|
500
|
+
/* WRITE on a readonly view (CARepeat stride-0 / value_array /
|
|
501
|
+
CAWrap readonly) — explicit reject, would otherwise SEGV on
|
|
502
|
+
write. */
|
|
503
|
+
if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
|
|
504
|
+
return CA_ITER_ERR_READONLY;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/* Step 6: mask is default-borne (bakeoff #5). The step-4
|
|
508
|
+
CA_ITER_ERR_MASK gate is lifted (= masked sources are accepted).
|
|
509
|
+
Step 7: enforce CA_KERNEL_NO_MASK as an explicit kernel-character
|
|
510
|
+
declaration — if a kernel says "I cannot handle mask" and a
|
|
511
|
+
masked source is handed in, reject with a dedicated error code
|
|
512
|
+
so the caller can decide whether to peel via .value /
|
|
513
|
+
.strip_mask(fill) or pick a mask-aware kernel. */
|
|
514
|
+
if ( (flags & CA_KERNEL_NO_MASK) && ca_has_mask(src) ) {
|
|
515
|
+
return CA_ITER_ERR_MASK_NOT_ALLOWED;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/* Gate: classify source. sub-step 5.1 routing accepts entity +
|
|
519
|
+
CAStride family (step 1-4) and CSA + CAGrid (step 5.1). Other
|
|
520
|
+
descriptor views (CASelect / CAMapping / CAWindow / CAShift)
|
|
521
|
+
and overlay views (CAFake / ...) still reject — handled in
|
|
522
|
+
5.2 and Phase 2 respectively. */
|
|
523
|
+
if ( ca_iter_classify_source(src) == CA_ITER_SRC_NONE ) {
|
|
524
|
+
return CA_ITER_ERR_NOT_CHEAP;
|
|
525
|
+
}
|
|
526
|
+
return CA_ITER_OK;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
int
|
|
530
|
+
ca_iter_state_init_l1 (ca_iter_state *st,
|
|
531
|
+
struct _CArray *src,
|
|
532
|
+
ca_slab_policy_t policy,
|
|
533
|
+
int8_t *axes,
|
|
534
|
+
int8_t naxes,
|
|
535
|
+
uint32_t flags)
|
|
536
|
+
{
|
|
537
|
+
/* PROPOSAL_CAFACE_PHASE_2 F.2.6: Face strip at entry (= same rationale as
|
|
538
|
+
init_l2 above). Strip before validate_inputs. */
|
|
539
|
+
if ( ca_is_face(src) ) {
|
|
540
|
+
src = ca_strip_face(src);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
int rc = ca_iter_validate_inputs(st, src, policy, flags);
|
|
544
|
+
if ( rc != CA_ITER_OK ) return rc;
|
|
545
|
+
|
|
546
|
+
/* Phase A capstone: CA_SLAB_AXES is L2-only (kernels needing K-D slab
|
|
547
|
+
yield use init_l2). init_l1 rejects so callers don't get a silent
|
|
548
|
+
misdispatch. */
|
|
549
|
+
if ( policy == CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
|
|
550
|
+
|
|
551
|
+
uint8_t src_kind = ca_iter_classify_source(src);
|
|
552
|
+
|
|
553
|
+
memset(st, 0, sizeof(*st));
|
|
554
|
+
st->src = src;
|
|
555
|
+
st->src_kind = src_kind;
|
|
556
|
+
st->level = 1;
|
|
557
|
+
st->policy = policy;
|
|
558
|
+
st->ndim = src->ndim;
|
|
559
|
+
st->flags = flags;
|
|
560
|
+
st->bytes = src->bytes;
|
|
561
|
+
st->axes = axes;
|
|
562
|
+
st->naxes = naxes;
|
|
563
|
+
|
|
564
|
+
st->slab_n = src->elements;
|
|
565
|
+
st->total_slabs = 1;
|
|
566
|
+
st->slabs_emitted = 0;
|
|
567
|
+
st->chunk_size = st->slab_n;
|
|
568
|
+
|
|
569
|
+
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
570
|
+
/* === SRC_ATTACH path (step 9 + 2026-05-31 refactor): CAFake /
|
|
571
|
+
CAByteSwap / CABitfield / CABitarray / CAReduce =================
|
|
572
|
+
View's xfer_all delivers materialised data through the unified
|
|
573
|
+
dispatch surface. Previously used `ca_attach(src) + src->ptr`
|
|
574
|
+
(= view's attach slot) and `ca_sync(src)` for WRITE; now uses
|
|
575
|
+
`ca_xfer_all(src, scratch, GET/PUT)` with iterator-owned scratch.
|
|
576
|
+
Decouples kernel_iterator from per-view attach/sync lifecycle
|
|
577
|
+
and inherits xfer reform improvements (transform-fused, etc.)
|
|
578
|
+
automatically. */
|
|
579
|
+
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
580
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
581
|
+
if ( src->elements > 0 ) {
|
|
582
|
+
ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
|
|
583
|
+
}
|
|
584
|
+
st->alias_mode = CA_ITER_ALIAS_NONE; /* scratch-owned, no src.detach */
|
|
585
|
+
st->alias_ptr = st->scratch_ptr;
|
|
586
|
+
st->composed_strides[0] = src->bytes;
|
|
587
|
+
st->composed_base = 0;
|
|
588
|
+
st->outer_idx = NULL;
|
|
589
|
+
/* Step 6: mask gather (xfer_all on mask -- ca_copy_data is a thin
|
|
590
|
+
wrapper, this just removes the extra hop). */
|
|
591
|
+
if ( ca_has_mask(src) ) {
|
|
592
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
593
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
594
|
+
if ( src->elements > 0 ) {
|
|
595
|
+
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
596
|
+
}
|
|
597
|
+
st->alias_mask = st->scratch_mask;
|
|
598
|
+
}
|
|
599
|
+
return CA_ITER_OK;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
603
|
+
/* === descriptor path (step 5.1: CSA + CAGrid) ======================
|
|
604
|
+
Reuse P3 substrate: describe_axes → prepare → layout. For now
|
|
605
|
+
always materialise via ca_axis_dispatch_attach into scratch and
|
|
606
|
+
yield one contig slab (alias_mode = NONE). The view-transparency
|
|
607
|
+
principle (proposal §0): kernel sees a flat slab regardless of
|
|
608
|
+
per-axis kind; INDEX / SHIFT mix is handled inside the engine. */
|
|
609
|
+
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
610
|
+
int8_t raw_ndim = 0;
|
|
611
|
+
ca_iter_describe_axes(src, raw_descs, st->parent_axis_dims, &raw_ndim);
|
|
612
|
+
/* Cache the post-merge layout for next_slab / sync_slab. */
|
|
613
|
+
ca_axis_dispatch_prepare(st->parent_axis_dims, raw_descs, raw_ndim,
|
|
614
|
+
st->bytes, st->descs, st->pstrides,
|
|
615
|
+
st->mdim, &st->desc_ndim);
|
|
616
|
+
ca_axis_dispatch_layout(st->descs, st->pstrides, st->mdim,
|
|
617
|
+
st->desc_ndim, st->bytes,
|
|
618
|
+
&st->slab_start, &st->slab_bytes_desc,
|
|
619
|
+
&st->slab_base);
|
|
620
|
+
if ( st->slab_start > 0 ) {
|
|
621
|
+
ca_axis_dispatch_classify_prefix(st->descs, st->pstrides,
|
|
622
|
+
st->slab_start, st->prefix);
|
|
623
|
+
}
|
|
624
|
+
st->total_elements = src->elements;
|
|
625
|
+
|
|
626
|
+
/* Materialise via the engine. ca_axis_dispatch_attach handles
|
|
627
|
+
parent attach + scratch alloc + gather (INDEX axes included);
|
|
628
|
+
we own the resulting buffer. Descriptor views inherit from
|
|
629
|
+
CAView so parent is at the CAVIEW prefix slot. */
|
|
630
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
631
|
+
ca_attach(parent); /* engine reads parent->ptr */
|
|
632
|
+
st->root = parent; /* finish() detaches */
|
|
633
|
+
st->scratch_cap = src->elements * src->bytes;
|
|
634
|
+
/* Engine ndim = descriptor ndim from describe_axes (not view.ndim).
|
|
635
|
+
Critical for CAMapping where view.ndim > 1 but the descriptor is
|
|
636
|
+
1-D INDEX gather.
|
|
637
|
+
bound_fill = CAWindow/CAShift fill value (Tier 2.B SHIFT-axis
|
|
638
|
+
OOB cell), NULL for non-window views (CSA / CAGrid / CASelect /
|
|
639
|
+
CAMapping don't have OOB semantics — engine sees no SHIFT axes). */
|
|
640
|
+
const void *bound_fill = NULL;
|
|
641
|
+
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
642
|
+
bound_fill = ((CAWindow *) src)->fill;
|
|
643
|
+
}
|
|
644
|
+
st->scratch_ptr = ca_axis_dispatch_attach(parent,
|
|
645
|
+
st->parent_axis_dims,
|
|
646
|
+
raw_descs, raw_ndim,
|
|
647
|
+
src->bytes,
|
|
648
|
+
st->total_elements,
|
|
649
|
+
bound_fill);
|
|
650
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
651
|
+
st->alias_ptr = st->scratch_ptr;
|
|
652
|
+
|
|
653
|
+
/* Step 6: gather mask via ca_copy_data on src->mask (works for
|
|
654
|
+
all descriptor views — mask propagates through their
|
|
655
|
+
func_copy_data via the descriptor framework's own mask
|
|
656
|
+
handling). */
|
|
657
|
+
if ( ca_has_mask(src) ) {
|
|
658
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
659
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
660
|
+
if ( src->elements > 0 ) {
|
|
661
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
662
|
+
}
|
|
663
|
+
st->alias_mask = st->scratch_mask;
|
|
664
|
+
}
|
|
665
|
+
return CA_ITER_OK;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/* === CAStride path (step 1-4) ============================== */
|
|
669
|
+
if ( ca_iter_can_alias(src, 1) ) {
|
|
670
|
+
/* alias path: entity or CAStride contig */
|
|
671
|
+
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
672
|
+
ca_attach(src);
|
|
673
|
+
st->alias_ptr = (char *) src->ptr;
|
|
674
|
+
} else {
|
|
675
|
+
/* scratch path: CAStride family non-contig. ca_copy_data routes
|
|
676
|
+
to ca_stride_func_copy_data which composes leaf strides up to
|
|
677
|
+
the root entity and gathers in one pass — no per-intermediate
|
|
678
|
+
view materialise. */
|
|
679
|
+
ca_size_t cap = src->elements * src->bytes;
|
|
680
|
+
st->scratch_cap = cap;
|
|
681
|
+
st->scratch_ptr = xmalloc(cap > 0 ? cap : 1);
|
|
682
|
+
if ( cap > 0 ) {
|
|
683
|
+
ca_copy_data(src, st->scratch_ptr);
|
|
684
|
+
}
|
|
685
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
686
|
+
st->alias_ptr = st->scratch_ptr;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/* Step 6: gather mask into scratch_mask if source carries one.
|
|
690
|
+
We always gather into a contig boolean8_t buffer for uniformity
|
|
691
|
+
across alias / scratch / descriptor paths — the kernel sees a
|
|
692
|
+
simple `boolean8_t *` aligned with the value slab. Future
|
|
693
|
+
optimisation: alias mask directly for CAStride contig. Mask is
|
|
694
|
+
informational; kernel uses CA_FOR_EACH_UNMASKED macros to skip
|
|
695
|
+
masked cells. */
|
|
696
|
+
if ( ca_has_mask(src) ) {
|
|
697
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
698
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
699
|
+
if ( src->elements > 0 ) {
|
|
700
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
701
|
+
}
|
|
702
|
+
st->alias_mask = st->scratch_mask;
|
|
703
|
+
}
|
|
704
|
+
return CA_ITER_OK;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
int
|
|
708
|
+
ca_iter_state_init_l2 (ca_iter_state *st,
|
|
709
|
+
struct _CArray *src,
|
|
710
|
+
ca_slab_policy_t policy,
|
|
711
|
+
int8_t *axes,
|
|
712
|
+
int8_t naxes,
|
|
713
|
+
uint32_t flags)
|
|
714
|
+
{
|
|
715
|
+
/* PROPOSAL_CAFACE_PHASE_2 F.2.6 (= MEMO §3.5 kernel_iterator entry strip):
|
|
716
|
+
Face only layers a semantic identifier; storage is identical to parent,
|
|
717
|
+
so strip at the kernel-compute entry and descend to parent. The Face
|
|
718
|
+
identifier is re-wrapped onto the result by the caller's lift hook
|
|
719
|
+
(= primary operators / reductions / etc.). Because Face is an identity
|
|
720
|
+
mask, routing / alias decisions should be based on parent.
|
|
721
|
+
Strip before validate_inputs — `classify_source` would reject Face as
|
|
722
|
+
knows-no. */
|
|
723
|
+
if ( ca_is_face(src) ) {
|
|
724
|
+
src = ca_strip_face(src);
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
int rc = ca_iter_validate_inputs(st, src, policy, flags);
|
|
728
|
+
if ( rc != CA_ITER_OK ) return rc;
|
|
729
|
+
|
|
730
|
+
/* F-2 (rev6): route_source returns the refined src_kind and (for
|
|
731
|
+
descriptor sources) populates raw_descs / raw_pdims / raw_ndim so we
|
|
732
|
+
don't re-call describe_axes when SRC_DESCRIPTOR_L2_ALIASABLE upgrades
|
|
733
|
+
into the alias branch. Non-descriptor sources leave the out_*
|
|
734
|
+
buffers untouched. */
|
|
735
|
+
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
736
|
+
ca_size_t raw_pdims[CA_RANK_MAX];
|
|
737
|
+
int8_t raw_ndim = 0;
|
|
738
|
+
uint8_t src_kind = ca_iter_route_source(src, raw_descs, raw_pdims, &raw_ndim);
|
|
739
|
+
|
|
740
|
+
/* Phase B capstone (T2): CA_SLAB_AXES + descriptor view (= SRC_DESCRIPTOR
|
|
741
|
+
and SRC_DESCRIPTOR_L2_ALIASABLE), accept when slab axes are all-STRIDE
|
|
742
|
+
kind and no outer SHIFT. Other combinations:
|
|
743
|
+
- slab has INDEX/SHIFT kind → reject (Phase C T3 scope)
|
|
744
|
+
- outer has SHIFT axis → reject (B.1.5 materialise downgrade scope)
|
|
745
|
+
- SRC_ATTACH → reject (overlay views; no kind structure)
|
|
746
|
+
SRC_CASTRIDE still goes through the Phase A branch below. */
|
|
747
|
+
if ( policy == CA_SLAB_AXES ) {
|
|
748
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR
|
|
749
|
+
|| src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
750
|
+
/* Validate axes early (range / duplicate). */
|
|
751
|
+
if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
|
|
752
|
+
return CA_ITER_ERR_POLICY;
|
|
753
|
+
}
|
|
754
|
+
int8_t in_slab[CA_RANK_MAX];
|
|
755
|
+
int8_t k;
|
|
756
|
+
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
757
|
+
for ( k = 0; k < naxes; k++ ) {
|
|
758
|
+
int8_t ax = axes[k];
|
|
759
|
+
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
760
|
+
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
761
|
+
in_slab[ax] = 1;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.3 hook (= parallel to
|
|
765
|
+
F.6.2 SRC_ATTACH hook): for descriptor views CAWindow /
|
|
766
|
+
CAShift (predicate returns 1 unconditionally per F.5 always-
|
|
767
|
+
win bench), skip materialise (alias / C.1 PER_SLAB / B.1.5)
|
|
768
|
+
and use per-fiber ca_xfer_stride which routes to the view's
|
|
769
|
+
X.1 OOB-fused per-region fast path. */
|
|
770
|
+
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
771
|
+
&& raw_ndim == src->ndim ) {
|
|
772
|
+
int8_t fiber_ax = axes[0];
|
|
773
|
+
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
774
|
+
{
|
|
775
|
+
ca_size_t b = src->bytes;
|
|
776
|
+
for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
|
|
777
|
+
row_byte_strides[kk] = b;
|
|
778
|
+
b *= src->dim[kk];
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
|
|
782
|
+
row_byte_strides[fiber_ax],
|
|
783
|
+
flags) ) {
|
|
784
|
+
memset(st, 0, sizeof(*st));
|
|
785
|
+
st->src = src;
|
|
786
|
+
st->src_kind = src_kind;
|
|
787
|
+
st->level = 2;
|
|
788
|
+
st->policy = policy;
|
|
789
|
+
st->ndim = src->ndim;
|
|
790
|
+
st->flags = flags;
|
|
791
|
+
st->bytes = src->bytes;
|
|
792
|
+
st->axes = axes;
|
|
793
|
+
st->naxes = naxes;
|
|
794
|
+
|
|
795
|
+
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
796
|
+
st->alias_ptr = NULL;
|
|
797
|
+
st->fiber_axis = fiber_ax;
|
|
798
|
+
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
799
|
+
st->fiber_native_strides[kk] = row_byte_strides[kk];
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/* Row-major element strides for mask (= identity element
|
|
803
|
+
index strides over src->dim). */
|
|
804
|
+
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
805
|
+
{
|
|
806
|
+
ca_size_t e = 1;
|
|
807
|
+
for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
|
|
808
|
+
row_elem_strides[kk] = e;
|
|
809
|
+
e *= src->dim[kk];
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
int8_t sp = 0, op = 0;
|
|
814
|
+
st->slab_elements = 1;
|
|
815
|
+
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
816
|
+
if ( in_slab[kk] ) {
|
|
817
|
+
st->slab_axes_buf[sp] = kk;
|
|
818
|
+
st->slab_dims[sp] = src->dim[kk];
|
|
819
|
+
st->slab_strides[sp] = row_byte_strides[kk];
|
|
820
|
+
st->slab_mask_strides[sp] = row_elem_strides[kk];
|
|
821
|
+
st->slab_elements *= src->dim[kk];
|
|
822
|
+
sp++;
|
|
823
|
+
} else {
|
|
824
|
+
st->outer_axes[op] = kk;
|
|
825
|
+
st->outer_dims[op] = src->dim[kk];
|
|
826
|
+
st->outer_strides[op] = row_byte_strides[kk];
|
|
827
|
+
st->outer_mask_strides[op] = row_elem_strides[kk];
|
|
828
|
+
op++;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
st->slab_ndim = sp;
|
|
832
|
+
st->outer_ndim = op;
|
|
833
|
+
st->desc_ndim = 0;
|
|
834
|
+
|
|
835
|
+
ca_size_t total = 1;
|
|
836
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
837
|
+
st->total_slabs = total;
|
|
838
|
+
st->slab_n = st->slab_elements;
|
|
839
|
+
st->slabs_emitted = 0;
|
|
840
|
+
st->chunk_size = st->slab_n;
|
|
841
|
+
|
|
842
|
+
if ( st->outer_ndim > 0 ) {
|
|
843
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
844
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
845
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
846
|
+
} else {
|
|
847
|
+
st->outer_idx = NULL;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/* Mask: per-fiber gather via next_slab_axes; no whole-view
|
|
851
|
+
materialise (rev2 §3.3, (data,mask) pair travels together). */
|
|
852
|
+
st->alias_mask = NULL;
|
|
853
|
+
return CA_ITER_OK;
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
/* Phase C T3 (C.1): slab axis has INDEX or SHIFT kind → per-slab
|
|
858
|
+
materialise fallback path (D1.1 (B) + D1.2 (A)). CAMapping-style
|
|
859
|
+
views (raw_ndim != src->ndim) are out of C.1 scope — descriptor
|
|
860
|
+
exposes a flat 1-D INDEX gather that doesn't expose the view-
|
|
861
|
+
axis partition the user gave via slab_axes. Reject explicitly. */
|
|
862
|
+
if ( raw_ndim != src->ndim ) {
|
|
863
|
+
return CA_ITER_ERR_POLICY;
|
|
864
|
+
}
|
|
865
|
+
int slab_has_non_stride = 0;
|
|
866
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
867
|
+
if ( in_slab[k] && raw_descs[k].kind != CA_AXIS_KIND_STRIDE ) {
|
|
868
|
+
slab_has_non_stride = 1;
|
|
869
|
+
break;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
if ( slab_has_non_stride ) {
|
|
873
|
+
/* C.1 scope: READ-only. WRITE = future sub-step (C.1c). */
|
|
874
|
+
if ( flags & CA_KERNEL_WRITE ) {
|
|
875
|
+
return CA_ITER_ERR_FLAGS;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
memset(st, 0, sizeof(*st));
|
|
879
|
+
st->src = src;
|
|
880
|
+
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
881
|
+
st->level = 2;
|
|
882
|
+
st->policy = policy;
|
|
883
|
+
st->ndim = src->ndim;
|
|
884
|
+
st->flags = flags;
|
|
885
|
+
st->bytes = src->bytes;
|
|
886
|
+
st->axes = axes;
|
|
887
|
+
st->naxes = naxes;
|
|
888
|
+
|
|
889
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
890
|
+
ca_attach(parent);
|
|
891
|
+
st->root = parent;
|
|
892
|
+
|
|
893
|
+
/* Persist raw_descs / pdims / pstrides for per-slab subset
|
|
894
|
+
construction (PER_SLAB fallback) or for the hoisted manual
|
|
895
|
+
gather (PER_SLAB_HOIST specialised path). desc_ndim =
|
|
896
|
+
src->ndim so has_descs branch in next_slab_axes can read
|
|
897
|
+
st->descs (though T3 path takes its own branch before that
|
|
898
|
+
check). */
|
|
899
|
+
memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
900
|
+
memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
|
|
901
|
+
{
|
|
902
|
+
ca_size_t s = src->bytes;
|
|
903
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
904
|
+
st->pstrides[k] = s;
|
|
905
|
+
s *= raw_pdims[k];
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
st->desc_ndim = src->ndim;
|
|
909
|
+
|
|
910
|
+
/* View-row-major element strides (= mask scratch layout, since
|
|
911
|
+
mask is gathered whole-view once at init). */
|
|
912
|
+
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
913
|
+
{
|
|
914
|
+
ca_size_t e = 1;
|
|
915
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
916
|
+
row_elem_strides[k] = e;
|
|
917
|
+
e *= src->dim[k];
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/* Row-major byte strides over SLAB axes only (= scratch_ptr
|
|
922
|
+
layout, since scratch_ptr is sized slab_elements × bytes
|
|
923
|
+
and refilled per outer iter as a contig row-major slab). */
|
|
924
|
+
ca_size_t slab_data_strides[CA_RANK_MAX];
|
|
925
|
+
{
|
|
926
|
+
ca_size_t b = src->bytes;
|
|
927
|
+
ca_size_t e_slab = 1;
|
|
928
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
929
|
+
if ( in_slab[k] ) {
|
|
930
|
+
slab_data_strides[k] = b;
|
|
931
|
+
b *= raw_descs[k].count;
|
|
932
|
+
e_slab *= raw_descs[k].count;
|
|
933
|
+
} else {
|
|
934
|
+
slab_data_strides[k] = 0; /* unused for outer axes */
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
st->slab_elements = e_slab;
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
int8_t sp = 0, op = 0;
|
|
941
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
942
|
+
if ( in_slab[k] ) {
|
|
943
|
+
st->slab_axes_buf[sp] = k;
|
|
944
|
+
st->slab_dims[sp] = raw_descs[k].count;
|
|
945
|
+
st->slab_strides[sp] = slab_data_strides[k];
|
|
946
|
+
/* Slab mask stride is view-row-major elem stride along
|
|
947
|
+
this view axis (mask scratch is whole-view layout). */
|
|
948
|
+
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
949
|
+
sp++;
|
|
950
|
+
} else {
|
|
951
|
+
st->outer_axes[op] = k;
|
|
952
|
+
st->outer_dims[op] = raw_descs[k].count;
|
|
953
|
+
/* T3 path: outer data offset is always 0 (scratch refilled
|
|
954
|
+
per slab, ptr = scratch_ptr). */
|
|
955
|
+
st->outer_strides[op] = 0;
|
|
956
|
+
/* Outer mask offset uses view-row-major elem stride into
|
|
957
|
+
whole-view mask scratch. */
|
|
958
|
+
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
959
|
+
op++;
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
st->slab_ndim = sp;
|
|
963
|
+
st->outer_ndim = op;
|
|
964
|
+
st->composed_base = 0; /* unused for T3 fallback */
|
|
965
|
+
|
|
966
|
+
ca_size_t total = 1;
|
|
967
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
968
|
+
st->total_slabs = total;
|
|
969
|
+
st->slab_n = st->slab_elements;
|
|
970
|
+
st->slabs_emitted = 0;
|
|
971
|
+
st->chunk_size = st->slab_n;
|
|
972
|
+
|
|
973
|
+
if ( st->outer_ndim > 0 ) {
|
|
974
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
975
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
976
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
977
|
+
} else {
|
|
978
|
+
st->outer_idx = NULL;
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
/* Alloc per-slab data scratch (D1.1 (B): one max-slab buffer
|
|
982
|
+
refilled per outer iter, instead of per-iter alloc/free or
|
|
983
|
+
full-view materialise). */
|
|
984
|
+
ca_size_t slab_bytes = st->slab_elements * src->bytes;
|
|
985
|
+
st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
|
|
986
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap);
|
|
987
|
+
st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
|
|
988
|
+
|
|
989
|
+
/* Specialisation (B-1b, C.1b): if innermost slab axis is STRIDE
|
|
990
|
+
and no SHIFT axes anywhere in the view, use the hoisted
|
|
991
|
+
manual gather path (= outer + non-innermost-slab hoist,
|
|
992
|
+
inner pure STRIDE linear memcpy). Else (innermost INDEX /
|
|
993
|
+
SHIFT-anywhere): use the engine fallback (A). */
|
|
994
|
+
int specialised_eligible = (sp > 0);
|
|
995
|
+
if ( specialised_eligible ) {
|
|
996
|
+
int8_t inner_view_ax = st->slab_axes_buf[sp - 1];
|
|
997
|
+
if ( raw_descs[inner_view_ax].kind != CA_AXIS_KIND_STRIDE ) {
|
|
998
|
+
specialised_eligible = 0;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
if ( specialised_eligible ) {
|
|
1002
|
+
/* Any SHIFT axis (slab or outer) → fallback to engine for
|
|
1003
|
+
clean bound_fill semantics in C.1b scope. */
|
|
1004
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1005
|
+
if ( raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1006
|
+
specialised_eligible = 0;
|
|
1007
|
+
break;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
st->alias_mode = specialised_eligible
|
|
1012
|
+
? CA_ITER_ALIAS_PER_SLAB_HOIST
|
|
1013
|
+
: CA_ITER_ALIAS_PER_SLAB;
|
|
1014
|
+
|
|
1015
|
+
/* Mask: gather whole view mask once into scratch_mask (= same
|
|
1016
|
+
strategy as B.1.5). Per-slab mask offset is computed via
|
|
1017
|
+
outer_idx × outer_mask_strides in next_slab_axes. */
|
|
1018
|
+
if ( ca_has_mask(src) ) {
|
|
1019
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1020
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1021
|
+
if ( src->elements > 0 ) {
|
|
1022
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1023
|
+
}
|
|
1024
|
+
st->alias_mask = st->scratch_mask;
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
return CA_ITER_OK;
|
|
1028
|
+
}
|
|
1029
|
+
/* slab is all-STRIDE: fall through to B.1.5 / Phase B paths. */
|
|
1030
|
+
/* Phase B.1.5: outer SHIFT axis → materialise downgrade. Alias
|
|
1031
|
+
can't deliver OOB cells (= need fill_value), so per the
|
|
1032
|
+
delivery principle (CLAUDE.md §"deliver" + D1.3 confirmed),
|
|
1033
|
+
we materialise the entire view into a row-major scratch
|
|
1034
|
+
buffer via ca_axis_dispatch_attach (= same engine as the
|
|
1035
|
+
existing SRC_DESCRIPTOR L2 NONE path), then walk it with
|
|
1036
|
+
Phase A-style row-major K-D strides. Slab / outer partition
|
|
1037
|
+
applies to view-axes; the scratch IS the flat row-major view. */
|
|
1038
|
+
int outer_has_shift = 0;
|
|
1039
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1040
|
+
if ( !in_slab[k] && raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1041
|
+
outer_has_shift = 1;
|
|
1042
|
+
break;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
if ( outer_has_shift ) {
|
|
1047
|
+
memset(st, 0, sizeof(*st));
|
|
1048
|
+
st->src = src;
|
|
1049
|
+
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1050
|
+
st->level = 2;
|
|
1051
|
+
st->policy = policy;
|
|
1052
|
+
st->ndim = src->ndim;
|
|
1053
|
+
st->flags = flags;
|
|
1054
|
+
st->bytes = src->bytes;
|
|
1055
|
+
st->axes = axes;
|
|
1056
|
+
st->naxes = naxes;
|
|
1057
|
+
|
|
1058
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1059
|
+
ca_attach(parent);
|
|
1060
|
+
st->root = parent;
|
|
1061
|
+
|
|
1062
|
+
ca_axis_desc_t local_descs[CA_RANK_MAX];
|
|
1063
|
+
memcpy(local_descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1064
|
+
const void *bound_fill = NULL;
|
|
1065
|
+
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
1066
|
+
bound_fill = ((CAWindow *) src)->fill;
|
|
1067
|
+
}
|
|
1068
|
+
st->scratch_cap = src->elements * src->bytes;
|
|
1069
|
+
st->scratch_ptr = ca_axis_dispatch_attach(parent, raw_pdims,
|
|
1070
|
+
local_descs, src->ndim,
|
|
1071
|
+
src->bytes, src->elements,
|
|
1072
|
+
bound_fill);
|
|
1073
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1074
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1075
|
+
|
|
1076
|
+
/* Row-major byte / element strides on the scratch (= view layout). */
|
|
1077
|
+
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
1078
|
+
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1079
|
+
{
|
|
1080
|
+
ca_size_t b = src->bytes, e = 1;
|
|
1081
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1082
|
+
row_byte_strides[k] = b;
|
|
1083
|
+
row_elem_strides[k] = e;
|
|
1084
|
+
b *= src->dim[k];
|
|
1085
|
+
e *= src->dim[k];
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
int8_t sp = 0, op = 0;
|
|
1090
|
+
st->slab_elements = 1;
|
|
1091
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1092
|
+
if ( in_slab[k] ) {
|
|
1093
|
+
st->slab_axes_buf[sp] = k;
|
|
1094
|
+
st->slab_dims[sp] = src->dim[k];
|
|
1095
|
+
st->slab_strides[sp] = row_byte_strides[k];
|
|
1096
|
+
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1097
|
+
st->slab_elements *= src->dim[k];
|
|
1098
|
+
sp++;
|
|
1099
|
+
} else {
|
|
1100
|
+
st->outer_axes[op] = k;
|
|
1101
|
+
st->outer_dims[op] = src->dim[k];
|
|
1102
|
+
st->outer_strides[op] = row_byte_strides[k];
|
|
1103
|
+
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1104
|
+
op++;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
st->slab_ndim = sp;
|
|
1108
|
+
st->outer_ndim = op;
|
|
1109
|
+
st->composed_base = 0;
|
|
1110
|
+
/* desc_ndim = 0: next_slab_axes treats outer like Phase A
|
|
1111
|
+
(no INDEX lookup), correct because scratch is row-major
|
|
1112
|
+
contig (= STRIDE everywhere). */
|
|
1113
|
+
st->desc_ndim = 0;
|
|
1114
|
+
|
|
1115
|
+
ca_size_t total = 1;
|
|
1116
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1117
|
+
st->total_slabs = total;
|
|
1118
|
+
st->slab_n = st->slab_elements;
|
|
1119
|
+
st->slabs_emitted = 0;
|
|
1120
|
+
st->chunk_size = st->slab_n;
|
|
1121
|
+
|
|
1122
|
+
if ( st->outer_ndim > 0 ) {
|
|
1123
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1124
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1125
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1126
|
+
} else {
|
|
1127
|
+
st->outer_idx = NULL;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
if ( ca_has_mask(src) ) {
|
|
1131
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1132
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1133
|
+
if ( src->elements > 0 ) {
|
|
1134
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1135
|
+
}
|
|
1136
|
+
st->alias_mask = st->scratch_mask;
|
|
1137
|
+
}
|
|
1138
|
+
return CA_ITER_OK;
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
memset(st, 0, sizeof(*st));
|
|
1142
|
+
st->src = src;
|
|
1143
|
+
st->src_kind = CA_ITER_SRC_DESCRIPTOR; /* Phase B alias path */
|
|
1144
|
+
st->level = 2;
|
|
1145
|
+
st->policy = policy;
|
|
1146
|
+
st->ndim = src->ndim;
|
|
1147
|
+
st->flags = flags;
|
|
1148
|
+
st->bytes = src->bytes;
|
|
1149
|
+
st->axes = axes;
|
|
1150
|
+
st->naxes = naxes;
|
|
1151
|
+
|
|
1152
|
+
/* Parent row-major byte strides for offset computation. */
|
|
1153
|
+
ca_size_t pstrides[CA_RANK_MAX];
|
|
1154
|
+
{
|
|
1155
|
+
ca_size_t s = src->bytes;
|
|
1156
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1157
|
+
pstrides[k] = s;
|
|
1158
|
+
s *= raw_pdims[k];
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
/* View row-major element strides (= mask scratch layout). */
|
|
1162
|
+
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1163
|
+
{
|
|
1164
|
+
ca_size_t s = 1;
|
|
1165
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1166
|
+
row_elem_strides[k] = s;
|
|
1167
|
+
s *= src->dim[k];
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/* Partition + populate slab / outer metadata. Slab axes contribute
|
|
1172
|
+
to a constant slab_base (since they're all STRIDE: start +
|
|
1173
|
+
step*idx, where the base is start*pstride summed). Outer axes
|
|
1174
|
+
drive per-slab offset via direct (start + step*idx)*pstride or
|
|
1175
|
+
indices[idx]*pstride (= no classify_prefix engine, since outer
|
|
1176
|
+
axes here are not a prefix of raw_descs[] — they're a
|
|
1177
|
+
complement of slab_axes). */
|
|
1178
|
+
int8_t sp = 0, op = 0;
|
|
1179
|
+
ca_size_t slab_base = 0;
|
|
1180
|
+
st->slab_elements = 1;
|
|
1181
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1182
|
+
if ( in_slab[k] ) {
|
|
1183
|
+
/* STRIDE-kind slab axis: start*pstride goes to slab_base,
|
|
1184
|
+
step*pstride is the per-cell byte stride. */
|
|
1185
|
+
slab_base += raw_descs[k].start * pstrides[k];
|
|
1186
|
+
st->slab_axes_buf[sp] = k;
|
|
1187
|
+
st->slab_dims[sp] = raw_descs[k].count;
|
|
1188
|
+
st->slab_strides[sp] = raw_descs[k].step * pstrides[k];
|
|
1189
|
+
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1190
|
+
st->slab_elements *= raw_descs[k].count;
|
|
1191
|
+
sp++;
|
|
1192
|
+
} else {
|
|
1193
|
+
/* Outer axis: STRIDE or INDEX (SHIFT was rejected above).
|
|
1194
|
+
Store axis index + view's outer dim/stride for descriptor
|
|
1195
|
+
walk in next_slab_axes. We reuse outer_strides/outer_mask
|
|
1196
|
+
_strides as the OUTER walk metadata; outer-axis kind is
|
|
1197
|
+
carried implicitly via raw_descs[outer_axes[m]] in the
|
|
1198
|
+
state struct's descs[] field (populated below). */
|
|
1199
|
+
st->outer_axes[op] = k;
|
|
1200
|
+
st->outer_dims[op] = raw_descs[k].count;
|
|
1201
|
+
/* For STRIDE: per-axis stride = step * pstride; INDEX axes
|
|
1202
|
+
use indices[] from raw_descs and need a 0 stride here
|
|
1203
|
+
(next_slab_axes branches on descs[].kind to compute the
|
|
1204
|
+
real offset). We persist the full raw_descs in st->descs
|
|
1205
|
+
so the next_slab_axes implementation can dispatch. */
|
|
1206
|
+
if ( raw_descs[k].kind == CA_AXIS_KIND_STRIDE ) {
|
|
1207
|
+
st->outer_strides[op] = raw_descs[k].step * pstrides[k];
|
|
1208
|
+
/* Bake in start*pstride into slab_base for STRIDE outer too
|
|
1209
|
+
so per-cell offset is purely step*idx. */
|
|
1210
|
+
slab_base += raw_descs[k].start * pstrides[k];
|
|
1211
|
+
} else {
|
|
1212
|
+
/* INDEX kind: outer_strides[m] = pstrides[k] (element-unit
|
|
1213
|
+
from indices[]). We DON'T pre-bake into slab_base for
|
|
1214
|
+
INDEX axes; per-iter offset = indices[idx[m]] * pstrides[k]
|
|
1215
|
+
is computed in next_slab_axes. */
|
|
1216
|
+
st->outer_strides[op] = pstrides[k];
|
|
1217
|
+
}
|
|
1218
|
+
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1219
|
+
op++;
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
st->slab_ndim = sp;
|
|
1223
|
+
st->outer_ndim = op;
|
|
1224
|
+
st->composed_base = slab_base;
|
|
1225
|
+
|
|
1226
|
+
/* Persist raw_descs / pstrides in state so next_slab_axes can
|
|
1227
|
+
distinguish STRIDE vs INDEX outer per axis. */
|
|
1228
|
+
memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1229
|
+
memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
|
|
1230
|
+
memcpy(st->pstrides, pstrides, src->ndim * sizeof(ca_size_t));
|
|
1231
|
+
st->desc_ndim = src->ndim;
|
|
1232
|
+
|
|
1233
|
+
ca_size_t total = 1;
|
|
1234
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1235
|
+
st->total_slabs = total;
|
|
1236
|
+
st->slab_n = st->slab_elements;
|
|
1237
|
+
st->slabs_emitted = 0;
|
|
1238
|
+
st->chunk_size = st->slab_n;
|
|
1239
|
+
|
|
1240
|
+
if ( st->outer_ndim > 0 ) {
|
|
1241
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1242
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1243
|
+
} else {
|
|
1244
|
+
st->outer_idx = NULL;
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
/* Attach parent for alias path. */
|
|
1248
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1249
|
+
ca_attach(parent);
|
|
1250
|
+
st->root = parent;
|
|
1251
|
+
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
1252
|
+
st->alias_ptr = (char *) parent->ptr;
|
|
1253
|
+
|
|
1254
|
+
/* Mask gather (= view row-major boolean8_t). */
|
|
1255
|
+
if ( ca_has_mask(src) ) {
|
|
1256
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1257
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1258
|
+
if ( src->elements > 0 ) {
|
|
1259
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1260
|
+
}
|
|
1261
|
+
st->alias_mask = st->scratch_mask;
|
|
1262
|
+
}
|
|
1263
|
+
return CA_ITER_OK;
|
|
1264
|
+
}
|
|
1265
|
+
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
1266
|
+
/* Phase B.5: CA_SLAB_AXES + SRC_ATTACH (= CAFake / CAByteSwap /
|
|
1267
|
+
CABitfield / CABitarray / CAReduce). These overlay views
|
|
1268
|
+
materialise via their own ca_attach (= no descriptor framework
|
|
1269
|
+
path), so we let ca_attach(src) populate src->ptr in view
|
|
1270
|
+
row-major layout and walk it with Phase A-style K-D strides.
|
|
1271
|
+
|
|
1272
|
+
The most common entry path here is wrap_readonly(int_src,
|
|
1273
|
+
CA_FLOAT64) → CAFake, enabling sum_ki etc. to accept any
|
|
1274
|
+
numeric data_type source via auto-cast (= D2.2 confirmed in
|
|
1275
|
+
rev2 sparring). */
|
|
1276
|
+
if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
|
|
1277
|
+
return CA_ITER_ERR_POLICY;
|
|
1278
|
+
}
|
|
1279
|
+
int8_t in_slab[CA_RANK_MAX];
|
|
1280
|
+
int8_t k;
|
|
1281
|
+
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
1282
|
+
for ( k = 0; k < naxes; k++ ) {
|
|
1283
|
+
int8_t ax = axes[k];
|
|
1284
|
+
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1285
|
+
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
1286
|
+
in_slab[ax] = 1;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
memset(st, 0, sizeof(*st));
|
|
1290
|
+
st->src = src;
|
|
1291
|
+
st->src_kind = CA_ITER_SRC_ATTACH;
|
|
1292
|
+
st->level = 2;
|
|
1293
|
+
st->policy = policy;
|
|
1294
|
+
st->ndim = src->ndim;
|
|
1295
|
+
st->flags = flags;
|
|
1296
|
+
st->bytes = src->bytes;
|
|
1297
|
+
st->axes = axes;
|
|
1298
|
+
st->naxes = naxes;
|
|
1299
|
+
|
|
1300
|
+
/* Row-major byte / element strides on the attached view buffer.
|
|
1301
|
+
Computed first so the F.6.1 predicate can inspect fiber-axis
|
|
1302
|
+
effective stride before deciding whether to materialise. */
|
|
1303
|
+
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
1304
|
+
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1305
|
+
{
|
|
1306
|
+
ca_size_t b = src->bytes, e = 1;
|
|
1307
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1308
|
+
row_byte_strides[k] = b;
|
|
1309
|
+
row_elem_strides[k] = e;
|
|
1310
|
+
b *= src->dim[k];
|
|
1311
|
+
e *= src->dim[k];
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
/* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev4 (direct per-
|
|
1316
|
+
parent ptr access path + tile cache). Engages when:
|
|
1317
|
+
- source.attach is ca_stack_func.attach (= CAStack identity)
|
|
1318
|
+
- no mask (= horizontal mask propagation per-slab gather is
|
|
1319
|
+
out of scope; falls back to SRC_ATTACH whole-view path)
|
|
1320
|
+
- naxes == 1 && axes[0] == k_axis (= K-axis-only slab, the
|
|
1321
|
+
demand-driving case = reduce along the stacked axis like
|
|
1322
|
+
view.mean(axis: k_axis))
|
|
1323
|
+
init attaches K parents up front (O(1) per entity parent),
|
|
1324
|
+
caches parent->ptr aliases + uniform parent-native byte strides,
|
|
1325
|
+
allocates a slab-sized scratch (= K * bytes) plus a tile cache
|
|
1326
|
+
sized to fit the L1d budget. next_slab_axes refills TILE fibers
|
|
1327
|
+
per K contig parent reads (rev4 2026-06-19) and serves the next
|
|
1328
|
+
TILE-1 calls from the L1d-resident cache, beating SRC_ATTACH
|
|
1329
|
+
across all measured sizes. No size-threshold gate: rev3's gate
|
|
1330
|
+
was a perf trade-off justification that the tile cache erased.
|
|
1331
|
+
|
|
1332
|
+
K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): engage predicate
|
|
1333
|
+
generalised from `axes[0] == 0` to `axes[0] == k_axis`. The
|
|
1334
|
+
tile cache mechanism + outer_idx -> parent axis mapping are
|
|
1335
|
+
already k_axis-agnostic: outer_idx[m] maps to parent axis m
|
|
1336
|
+
regardless of k_axis position (= for k_axis = 0 outer is stack
|
|
1337
|
+
axes 1..N-1 = parent axes 0..N-2; for k_axis > 0 outer is stack
|
|
1338
|
+
axes [0..k_axis-1, k_axis+1..N-1] = parent axes [0..k_axis-1,
|
|
1339
|
+
k_axis..N-2]; either way outer_idx[m] = parent axis m). */
|
|
1340
|
+
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1341
|
+
&& !ca_has_mask(src)
|
|
1342
|
+
&& naxes == 1 && axes[0] == ((CAStack *) src)->k_axis ) {
|
|
1343
|
+
CAStack *stack = (CAStack *) src;
|
|
1344
|
+
int8_t parent_ndim = src->ndim - 1;
|
|
1345
|
+
int8_t sp = 0, op = 0;
|
|
1346
|
+
|
|
1347
|
+
st->slab_elements = 1;
|
|
1348
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1349
|
+
if ( in_slab[k] ) {
|
|
1350
|
+
st->slab_axes_buf[sp] = k;
|
|
1351
|
+
st->slab_dims[sp] = src->dim[k];
|
|
1352
|
+
st->slab_elements *= src->dim[k];
|
|
1353
|
+
sp++;
|
|
1354
|
+
} else {
|
|
1355
|
+
st->outer_axes[op] = k;
|
|
1356
|
+
st->outer_dims[op] = src->dim[k];
|
|
1357
|
+
st->outer_strides[op] = row_byte_strides[k]; /* unused on STACK path */
|
|
1358
|
+
st->outer_mask_strides[op] = row_elem_strides[k]; /* unused on STACK path */
|
|
1359
|
+
op++;
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
st->slab_ndim = sp;
|
|
1363
|
+
st->outer_ndim = op;
|
|
1364
|
+
st->composed_base = 0;
|
|
1365
|
+
st->desc_ndim = 0;
|
|
1366
|
+
/* slab_strides on STACK path: slab_axes == [0], scratch is
|
|
1367
|
+
packed contig over K elements (= K * bytes), so single-axis
|
|
1368
|
+
slab walk uses bytes stride. */
|
|
1369
|
+
st->slab_strides[0] = src->bytes;
|
|
1370
|
+
st->slab_mask_strides[0] = 1; /* harmless on no-mask path */
|
|
1371
|
+
|
|
1372
|
+
ca_size_t total = 1;
|
|
1373
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1374
|
+
st->total_slabs = total;
|
|
1375
|
+
st->slab_n = st->slab_elements;
|
|
1376
|
+
st->slabs_emitted = 0;
|
|
1377
|
+
st->chunk_size = st->slab_n;
|
|
1378
|
+
|
|
1379
|
+
if ( st->outer_ndim > 0 ) {
|
|
1380
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1381
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1382
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1383
|
+
} else {
|
|
1384
|
+
st->outer_idx = NULL;
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
/* Attach K parents and cache their ptr aliases. CAStack
|
|
1388
|
+
guarantees uniform shape (MEMO §3.2) so one set of parent
|
|
1389
|
+
native byte strides covers all K (= parents[0]->dim is
|
|
1390
|
+
canonical). */
|
|
1391
|
+
st->stack_n_parents = stack->n_parents;
|
|
1392
|
+
st->stack_parent_ptrs =
|
|
1393
|
+
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
1394
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1395
|
+
ca_attach(stack->parents[kk]);
|
|
1396
|
+
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
1397
|
+
}
|
|
1398
|
+
{
|
|
1399
|
+
ca_size_t s = src->bytes;
|
|
1400
|
+
for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
|
|
1401
|
+
st->stack_parent_strides[kk] = s;
|
|
1402
|
+
s *= stack->parents[0]->dim[kk];
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
/* Slab-sized scratch (= K * bytes for slab_axes == [0]). */
|
|
1407
|
+
st->scratch_cap = (ca_size_t) st->slab_elements * src->bytes;
|
|
1408
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
1409
|
+
|
|
1410
|
+
/* Tile cache (pilot/castack-axis0-loop-interchange): refill TILE
|
|
1411
|
+
fibers per K parent reads, serve next TILE-1 fibers from L1d.
|
|
1412
|
+
TILE budget = ~32 KB (half of M2 L1d). Empirically tested
|
|
1413
|
+
values; clamped to [8, 64] to keep the refill loop tight. */
|
|
1414
|
+
{
|
|
1415
|
+
ca_size_t K = (ca_size_t) stack->n_parents;
|
|
1416
|
+
ca_size_t bytes = (ca_size_t) src->bytes;
|
|
1417
|
+
ca_size_t tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
|
|
1418
|
+
if ( tile < 8 ) tile = 8;
|
|
1419
|
+
if ( tile > 64 ) tile = 64;
|
|
1420
|
+
st->stack_tile_cap = tile;
|
|
1421
|
+
st->stack_tile_pos = 0;
|
|
1422
|
+
st->stack_tile_have = 0; /* force refill on first call */
|
|
1423
|
+
st->stack_tile_cache = (char *) xmalloc(K * tile * bytes);
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
st->alias_mode = CA_ITER_ALIAS_STACK;
|
|
1427
|
+
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1428
|
+
st->alias_mask = NULL;
|
|
1429
|
+
return CA_ITER_OK;
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
/* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18):
|
|
1433
|
+
CAStack source + slab_axes excludes axis 0 (= K-axis stays in
|
|
1434
|
+
outer iter, e.g. view.mean(axis: 1), view.mean(axis: 2),
|
|
1435
|
+
view.mean(axis: 1, 2)). Each slab corresponds to a region
|
|
1436
|
+
inside ONE parent selected by outer_idx[K_outer_pos]. Engine
|
|
1437
|
+
pre-attaches K parents (+ K parent masks if mask present),
|
|
1438
|
+
caches their ptrs, and next_slab_axes aliases parents[k]->ptr +
|
|
1439
|
+
parent_off directly -- no scratch, no materialise, parent
|
|
1440
|
+
entity case = eager-equivalent memory bandwidth.
|
|
1441
|
+
|
|
1442
|
+
Layering: engine detects CAStack identity by function-pointer
|
|
1443
|
+
comparison and accesses parents[] / mask via ca_func[STACK]
|
|
1444
|
+
operation table where possible. CAStack downcast occurs only
|
|
1445
|
+
for the n_parents / parents[] read in init_l2 (= same scope as
|
|
1446
|
+
rev3 STACK; AC3 forbids parents[] walk in next_slab_axes only).
|
|
1447
|
+
Q1 disposition: option (i) minimal-diff inline in SLAB_AXES
|
|
1448
|
+
branch -- evaluated as smallest diff with acceptable layering
|
|
1449
|
+
(proposal §3.1). */
|
|
1450
|
+
/* STACK_OUTER_K: K-axis stays in outer iter (slab carved out of
|
|
1451
|
+
one parent at a time). K.3 (2026-06-20): engage predicate +
|
|
1452
|
+
stack-axis -> parent-axis mapping generalised for arbitrary
|
|
1453
|
+
k_axis. stack axis s maps to parent axis (s if s < k_axis else
|
|
1454
|
+
s - 1); only the K stack axis (s == k_axis) has no parent
|
|
1455
|
+
counterpart. */
|
|
1456
|
+
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1457
|
+
&& naxes >= 1 && !in_slab[((CAStack *) src)->k_axis] ) {
|
|
1458
|
+
CAStack *stack = (CAStack *) src;
|
|
1459
|
+
int8_t k_axis = stack->k_axis;
|
|
1460
|
+
int8_t parent_ndim = src->ndim - 1;
|
|
1461
|
+
int8_t sp = 0, op = 0;
|
|
1462
|
+
|
|
1463
|
+
st->slab_elements = 1;
|
|
1464
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1465
|
+
if ( in_slab[k] ) {
|
|
1466
|
+
st->slab_axes_buf[sp] = k;
|
|
1467
|
+
st->slab_dims[sp] = src->dim[k];
|
|
1468
|
+
st->slab_elements *= src->dim[k];
|
|
1469
|
+
/* slab_strides[sp] = parent native byte stride at parent
|
|
1470
|
+
axis (k if k < k_axis else k - 1). k == k_axis cannot
|
|
1471
|
+
appear here -- the engage predicate above excludes it. */
|
|
1472
|
+
sp++;
|
|
1473
|
+
} else {
|
|
1474
|
+
st->outer_axes[op] = k;
|
|
1475
|
+
st->outer_dims[op] = src->dim[k];
|
|
1476
|
+
st->outer_strides[op] = row_byte_strides[k];
|
|
1477
|
+
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1478
|
+
op++;
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
st->slab_ndim = sp;
|
|
1482
|
+
st->outer_ndim = op;
|
|
1483
|
+
st->composed_base = 0;
|
|
1484
|
+
st->desc_ndim = 0;
|
|
1485
|
+
|
|
1486
|
+
/* Locate the K-axis (= stack axis k_axis) within outer_axes. */
|
|
1487
|
+
st->stack_k_outer_pos = -1;
|
|
1488
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
1489
|
+
if ( st->outer_axes[m] == k_axis ) {
|
|
1490
|
+
st->stack_k_outer_pos = m;
|
|
1491
|
+
break;
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
/* Uniform parent native byte strides + parent element strides
|
|
1496
|
+
for mask (= MEMO §3.2 uniform shape across parents,
|
|
1497
|
+
parents[0]->dim is canonical). Mask strides cached so
|
|
1498
|
+
next_slab_axes can compute mask_off without downcasting
|
|
1499
|
+
(= AC3 layering goal). */
|
|
1500
|
+
{
|
|
1501
|
+
ca_size_t sb = src->bytes;
|
|
1502
|
+
ca_size_t se = 1;
|
|
1503
|
+
for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
|
|
1504
|
+
st->stack_parent_strides[kk] = sb;
|
|
1505
|
+
st->stack_parent_mask_strides[kk] = se;
|
|
1506
|
+
sb *= stack->parents[0]->dim[kk];
|
|
1507
|
+
se *= stack->parents[0]->dim[kk];
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
/* Fill slab_strides + slab_mask_strides for slab axes (all of
|
|
1512
|
+
which are parent inner axes since K is in outer). stack ax
|
|
1513
|
+
-> parent ax: s if s < k_axis else s - 1. */
|
|
1514
|
+
for ( int8_t s_i = 0; s_i < st->slab_ndim; s_i++ ) {
|
|
1515
|
+
int8_t stack_ax = st->slab_axes_buf[s_i];
|
|
1516
|
+
int8_t parent_ax = (stack_ax < k_axis) ? stack_ax : (stack_ax - 1);
|
|
1517
|
+
st->slab_strides[s_i] = st->stack_parent_strides[parent_ax];
|
|
1518
|
+
st->slab_mask_strides[s_i] = st->stack_parent_mask_strides[parent_ax];
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
ca_size_t total = 1;
|
|
1522
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1523
|
+
st->total_slabs = total;
|
|
1524
|
+
st->slab_n = st->slab_elements;
|
|
1525
|
+
st->slabs_emitted = 0;
|
|
1526
|
+
st->chunk_size = st->slab_n;
|
|
1527
|
+
|
|
1528
|
+
if ( st->outer_ndim > 0 ) {
|
|
1529
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1530
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1531
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1532
|
+
} else {
|
|
1533
|
+
st->outer_idx = NULL;
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
/* Attach K parents + cache ptr aliases. Symmetric with rev3
|
|
1537
|
+
Case B; for entity parents attach is O(1). */
|
|
1538
|
+
st->stack_n_parents = stack->n_parents;
|
|
1539
|
+
st->stack_parent_ptrs =
|
|
1540
|
+
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
1541
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1542
|
+
ca_attach(stack->parents[kk]);
|
|
1543
|
+
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/* If CAStack carries mask (= horizontal propagation already
|
|
1547
|
+
applied at create_mask), attach K parent masks + cache ptr
|
|
1548
|
+
aliases. next_slab_axes aliases parent->mask[k]->ptr +
|
|
1549
|
+
mask_off for the same slab. */
|
|
1550
|
+
if ( ca_has_mask(src) ) {
|
|
1551
|
+
st->stack_parent_mask_ptrs =
|
|
1552
|
+
(boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
|
|
1553
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1554
|
+
ca_attach(stack->parents[kk]->mask);
|
|
1555
|
+
st->stack_parent_mask_ptrs[kk] =
|
|
1556
|
+
(boolean8_t *) stack->parents[kk]->mask->ptr;
|
|
1557
|
+
}
|
|
1558
|
+
} else {
|
|
1559
|
+
st->stack_parent_mask_ptrs = NULL;
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
st->scratch_ptr = NULL; /* no scratch: aliasing parent->ptr */
|
|
1563
|
+
st->scratch_cap = 0;
|
|
1564
|
+
st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
|
|
1565
|
+
st->alias_ptr = NULL;
|
|
1566
|
+
st->alias_mask = NULL;
|
|
1567
|
+
return CA_ITER_OK;
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 hook: try per-fiber fused
|
|
1571
|
+
dispatch before whole-view materialise. When predicate fires,
|
|
1572
|
+
skip scratch alloc + xfer_all GET (= per-fiber ca_xfer_stride
|
|
1573
|
+
is called on demand in next_slab_axes). Predicate is stubbed
|
|
1574
|
+
in F.6.1 (returns 0); F.6.2+ enables specific source kinds. */
|
|
1575
|
+
int per_fiber_fused = 0;
|
|
1576
|
+
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1 ) {
|
|
1577
|
+
int8_t fiber_ax = axes[0];
|
|
1578
|
+
ca_size_t fiber_ax_stride = row_byte_strides[fiber_ax];
|
|
1579
|
+
if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
|
|
1580
|
+
fiber_ax_stride, flags) ) {
|
|
1581
|
+
per_fiber_fused = 1;
|
|
1582
|
+
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
1583
|
+
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1584
|
+
st->fiber_axis = fiber_ax;
|
|
1585
|
+
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
1586
|
+
st->fiber_native_strides[kk] = row_byte_strides[kk];
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
if ( !per_fiber_fused ) {
|
|
1592
|
+
/* 2026-05-31 refactor: iter-owns scratch via ca_xfer_all (was
|
|
1593
|
+
ca_attach(src) + alias src->ptr). */
|
|
1594
|
+
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
1595
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
1596
|
+
if ( src->elements > 0 ) {
|
|
1597
|
+
ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
|
|
1598
|
+
}
|
|
1599
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1600
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
int8_t sp = 0, op = 0;
|
|
1604
|
+
st->slab_elements = 1;
|
|
1605
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1606
|
+
if ( in_slab[k] ) {
|
|
1607
|
+
st->slab_axes_buf[sp] = k;
|
|
1608
|
+
st->slab_dims[sp] = src->dim[k];
|
|
1609
|
+
st->slab_strides[sp] = row_byte_strides[k];
|
|
1610
|
+
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1611
|
+
st->slab_elements *= src->dim[k];
|
|
1612
|
+
sp++;
|
|
1613
|
+
} else {
|
|
1614
|
+
st->outer_axes[op] = k;
|
|
1615
|
+
st->outer_dims[op] = src->dim[k];
|
|
1616
|
+
st->outer_strides[op] = row_byte_strides[k];
|
|
1617
|
+
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1618
|
+
op++;
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
st->slab_ndim = sp;
|
|
1622
|
+
st->outer_ndim = op;
|
|
1623
|
+
st->composed_base = 0;
|
|
1624
|
+
st->desc_ndim = 0; /* row-major STRIDE everywhere */
|
|
1625
|
+
|
|
1626
|
+
ca_size_t total = 1;
|
|
1627
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1628
|
+
st->total_slabs = total;
|
|
1629
|
+
st->slab_n = st->slab_elements;
|
|
1630
|
+
st->slabs_emitted = 0;
|
|
1631
|
+
st->chunk_size = st->slab_n;
|
|
1632
|
+
|
|
1633
|
+
if ( st->outer_ndim > 0 ) {
|
|
1634
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1635
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1636
|
+
} else {
|
|
1637
|
+
st->outer_idx = NULL;
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
if ( ca_has_mask(src) ) {
|
|
1641
|
+
if ( per_fiber_fused ) {
|
|
1642
|
+
/* F.6.1 rev2 §3.3: mask travels with data per-fiber. Mask
|
|
1643
|
+
buffer is delivered through fiber_mask_scratch by
|
|
1644
|
+
next_slab_axes; no whole-view materialise. */
|
|
1645
|
+
st->alias_mask = NULL;
|
|
1646
|
+
} else {
|
|
1647
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1648
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1649
|
+
if ( src->elements > 0 ) {
|
|
1650
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1651
|
+
}
|
|
1652
|
+
st->alias_mask = st->scratch_mask;
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
return CA_ITER_OK;
|
|
1656
|
+
}
|
|
1657
|
+
if ( src_kind != CA_ITER_SRC_CASTRIDE ) {
|
|
1658
|
+
/* Any other unclassified kind. */
|
|
1659
|
+
return CA_ITER_ERR_POLICY;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
/* F-2 minimal scope: SHIFT outer axes need OOB fill-slab support which
|
|
1664
|
+
is not yet implemented; downgrade to materialise (= existing
|
|
1665
|
+
SRC_DESCRIPTOR path) when present. Future work: allocate a 1-slab
|
|
1666
|
+
fill scratch and yield it on OOB iterations (see rev6 §3.5 deferred). */
|
|
1667
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE
|
|
1668
|
+
&& ca_axis_dispatch_outer_has_shift(raw_descs, raw_ndim) ) {
|
|
1669
|
+
src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/* Step 9 + 2026-05-31 refactor: L2 dispatch over SRC_ATTACH sources
|
|
1673
|
+
(CAFake / CAByteSwap / CABitfield / CABitarray / CAReduce / CAObject
|
|
1674
|
+
/ CATile / CARoll). Same shape as L1 SRC_ATTACH: iterator-owns
|
|
1675
|
+
scratch + xfer_all GET/PUT. Yield as one 1-D strided slab. */
|
|
1676
|
+
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
1677
|
+
memset(st, 0, sizeof(*st));
|
|
1678
|
+
st->src = src;
|
|
1679
|
+
st->src_kind = CA_ITER_SRC_ATTACH;
|
|
1680
|
+
st->level = 2;
|
|
1681
|
+
st->policy = policy;
|
|
1682
|
+
st->ndim = 1; /* logical 1-D L2 layout */
|
|
1683
|
+
st->flags = flags;
|
|
1684
|
+
st->bytes = src->bytes;
|
|
1685
|
+
st->axes = axes;
|
|
1686
|
+
st->naxes = naxes;
|
|
1687
|
+
|
|
1688
|
+
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
1689
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
1690
|
+
if ( src->elements > 0 ) {
|
|
1691
|
+
ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
|
|
1692
|
+
}
|
|
1693
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1694
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1695
|
+
st->composed_strides[0] = src->bytes;
|
|
1696
|
+
st->composed_base = 0;
|
|
1697
|
+
st->slab_n = src->elements;
|
|
1698
|
+
st->total_slabs = 1;
|
|
1699
|
+
st->slabs_emitted = 0;
|
|
1700
|
+
st->chunk_size = st->slab_n;
|
|
1701
|
+
st->outer_idx = NULL;
|
|
1702
|
+
|
|
1703
|
+
if ( ca_has_mask(src) ) {
|
|
1704
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1705
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1706
|
+
if ( src->elements > 0 ) {
|
|
1707
|
+
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
1708
|
+
}
|
|
1709
|
+
st->alias_mask = st->scratch_mask;
|
|
1710
|
+
}
|
|
1711
|
+
return CA_ITER_OK;
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
/* F-2 (rev6, PROPOSAL_F2_KERNEL_ITERATOR_ALIAS): descriptor L2 alias
|
|
1715
|
+
path. Eligibility (route_source verdict): innermost descriptor axis
|
|
1716
|
+
is STRIDE kind, no outer SHIFT axis (downgraded above when present).
|
|
1717
|
+
Setup: ca_attach(parent), alias_ptr = parent->ptr, no scratch alloc.
|
|
1718
|
+
Each outer-prefix iteration yields a strided slab
|
|
1719
|
+
slab_ptr = parent->ptr + inner_byte_start + outer_prefix_offset
|
|
1720
|
+
slab_n = descs[ndim-1].count
|
|
1721
|
+
slab_stride = descs[ndim-1].step * pstrides[ndim-1]
|
|
1722
|
+
where outer_prefix_offset is computed via ca_axis_dispatch_prefix_offset
|
|
1723
|
+
on the pre-classified prefix[]. next_slab_strided branches on
|
|
1724
|
+
src_kind to dispatch to this offset formula. */
|
|
1725
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
1726
|
+
int8_t nd = raw_ndim;
|
|
1727
|
+
int8_t k;
|
|
1728
|
+
|
|
1729
|
+
memset(st, 0, sizeof(*st));
|
|
1730
|
+
st->src = src;
|
|
1731
|
+
st->src_kind = CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE;
|
|
1732
|
+
st->level = 2;
|
|
1733
|
+
st->policy = policy;
|
|
1734
|
+
st->ndim = nd;
|
|
1735
|
+
st->flags = flags;
|
|
1736
|
+
st->bytes = src->bytes;
|
|
1737
|
+
st->axes = axes;
|
|
1738
|
+
st->naxes = naxes;
|
|
1739
|
+
|
|
1740
|
+
/* Persist descs / parent_axis_dims in state; build parent row-major
|
|
1741
|
+
pstrides so prefix_offset / inner offset share the same byte space. */
|
|
1742
|
+
memcpy(st->descs, raw_descs, nd * sizeof(ca_axis_desc_t));
|
|
1743
|
+
memcpy(st->parent_axis_dims, raw_pdims, nd * sizeof(ca_size_t));
|
|
1744
|
+
st->desc_ndim = nd;
|
|
1745
|
+
{
|
|
1746
|
+
ca_size_t s = src->bytes;
|
|
1747
|
+
for ( k = nd - 1; k >= 0; k-- ) {
|
|
1748
|
+
st->pstrides[k] = s;
|
|
1749
|
+
s *= raw_pdims[k];
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
/* Inner slab parameters from innermost STRIDE descriptor axis. */
|
|
1754
|
+
ca_size_t inner_start = raw_descs[nd - 1].start * st->pstrides[nd - 1];
|
|
1755
|
+
ca_size_t inner_stride = raw_descs[nd - 1].step * st->pstrides[nd - 1];
|
|
1756
|
+
ca_size_t inner_n = raw_descs[nd - 1].count;
|
|
1757
|
+
|
|
1758
|
+
/* Outer prefix classify (skipped for 1-D source since no outer). */
|
|
1759
|
+
if ( nd > 1 ) {
|
|
1760
|
+
ca_axis_dispatch_classify_prefix(st->descs, st->pstrides, nd - 1, st->prefix);
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
/* Total slabs = Π descs[0..nd-2].count (= src->elements / inner_n). */
|
|
1764
|
+
ca_size_t outer_total = 1;
|
|
1765
|
+
for ( k = 0; k < nd - 1; k++ ) outer_total *= raw_descs[k].count;
|
|
1766
|
+
st->total_slabs = outer_total;
|
|
1767
|
+
st->slab_n = inner_n;
|
|
1768
|
+
st->slabs_emitted = 0;
|
|
1769
|
+
st->chunk_size = inner_n;
|
|
1770
|
+
st->total_elements = src->elements;
|
|
1771
|
+
|
|
1772
|
+
/* composed_base holds inner_byte_start; composed_strides[nd-1] holds
|
|
1773
|
+
inner stride so next_slab_strided's existing inner_st extraction
|
|
1774
|
+
(= composed_strides[nd-1]) works without per-iter recomputation.
|
|
1775
|
+
Outer slots of composed_strides are unused by the descriptor
|
|
1776
|
+
branch in next_slab_strided (= prefix[] drives the offset). */
|
|
1777
|
+
st->composed_base = inner_start;
|
|
1778
|
+
st->composed_strides[nd - 1] = inner_stride;
|
|
1779
|
+
|
|
1780
|
+
if ( nd > 1 ) {
|
|
1781
|
+
CA_ASSUME(nd <= CA_RANK_MAX); /* with nd > 1: nd-1 in [1, CA_RANK_MAX-1] */
|
|
1782
|
+
st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
|
|
1783
|
+
for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
|
|
1784
|
+
} else {
|
|
1785
|
+
st->outer_idx = NULL;
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1789
|
+
ca_attach(parent);
|
|
1790
|
+
st->root = parent;
|
|
1791
|
+
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
1792
|
+
st->alias_ptr = (char *) parent->ptr;
|
|
1793
|
+
|
|
1794
|
+
/* Mask: gather into scratch_mask (same as SRC_DESCRIPTOR path).
|
|
1795
|
+
Data alias + mask materialise is per rev6 §4.4 deferred to F-4.c. */
|
|
1796
|
+
if ( ca_has_mask(src) ) {
|
|
1797
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1798
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1799
|
+
if ( src->elements > 0 ) {
|
|
1800
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1801
|
+
}
|
|
1802
|
+
st->alias_mask = st->scratch_mask;
|
|
1803
|
+
}
|
|
1804
|
+
return CA_ITER_OK;
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
/* Sub-step 5.3: L2 dispatch over descriptor sources. Per the
|
|
1808
|
+
delivery principle (CLAUDE.md "view-family surface prioritises delivery",
|
|
1809
|
+
PROPOSAL_T1_STEP5_DESCRIPTOR.md §0): always materialise into a
|
|
1810
|
+
scratch buffer via ca_axis_dispatch_attach and yield a single
|
|
1811
|
+
strided slab (stride = bytes). CASelect/CAMapping always reach
|
|
1812
|
+
here, CSA/CAGrid/CAWindow/CAShift when INDEX/SHIFT axes are
|
|
1813
|
+
present. F-2 (rev6) routes innermost-STRIDE descriptor cases to
|
|
1814
|
+
the L2 alias path above; this block now handles remaining mixed
|
|
1815
|
+
cases (innermost INDEX, outer SHIFT temporarily, etc.). */
|
|
1816
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
1817
|
+
/* raw_descs / raw_pdims / raw_ndim were populated by route_source
|
|
1818
|
+
above so we skip the local describe_axes call. Kept locals
|
|
1819
|
+
named raw_* to match the original code. */
|
|
1820
|
+
memset(st, 0, sizeof(*st));
|
|
1821
|
+
st->src = src;
|
|
1822
|
+
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1823
|
+
st->level = 2;
|
|
1824
|
+
st->policy = policy;
|
|
1825
|
+
st->ndim = src->ndim;
|
|
1826
|
+
st->flags = flags;
|
|
1827
|
+
st->bytes = src->bytes;
|
|
1828
|
+
st->axes = axes;
|
|
1829
|
+
st->naxes = naxes;
|
|
1830
|
+
|
|
1831
|
+
memcpy(st->parent_axis_dims, raw_pdims, raw_ndim * sizeof(ca_size_t));
|
|
1832
|
+
ca_axis_dispatch_prepare(st->parent_axis_dims, raw_descs, raw_ndim,
|
|
1833
|
+
st->bytes, st->descs, st->pstrides,
|
|
1834
|
+
st->mdim, &st->desc_ndim);
|
|
1835
|
+
ca_axis_dispatch_layout(st->descs, st->pstrides, st->mdim,
|
|
1836
|
+
st->desc_ndim, st->bytes,
|
|
1837
|
+
&st->slab_start, &st->slab_bytes_desc,
|
|
1838
|
+
&st->slab_base);
|
|
1839
|
+
if ( st->slab_start > 0 ) {
|
|
1840
|
+
ca_axis_dispatch_classify_prefix(st->descs, st->pstrides,
|
|
1841
|
+
st->slab_start, st->prefix);
|
|
1842
|
+
}
|
|
1843
|
+
st->total_elements = src->elements;
|
|
1844
|
+
|
|
1845
|
+
/* Materialise via the engine into a scratch buffer (contig
|
|
1846
|
+
layout = single strided run, stride = bytes). */
|
|
1847
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1848
|
+
ca_attach(parent);
|
|
1849
|
+
st->root = parent;
|
|
1850
|
+
st->scratch_cap = src->elements * src->bytes;
|
|
1851
|
+
const void *bound_fill = NULL;
|
|
1852
|
+
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
1853
|
+
bound_fill = ((CAWindow *) src)->fill;
|
|
1854
|
+
}
|
|
1855
|
+
st->scratch_ptr = ca_axis_dispatch_attach(parent,
|
|
1856
|
+
st->parent_axis_dims,
|
|
1857
|
+
raw_descs, raw_ndim,
|
|
1858
|
+
src->bytes,
|
|
1859
|
+
st->total_elements,
|
|
1860
|
+
bound_fill);
|
|
1861
|
+
|
|
1862
|
+
/* Lay out as a single 1-D L2 strided slab: ptr = scratch,
|
|
1863
|
+
n = total_elements, stride = bytes. next_slab_strided reads
|
|
1864
|
+
inner_stride from composed_strides[ndim - 1] and skips the
|
|
1865
|
+
outer loop when outer_idx == NULL, so a 1-D logical layout
|
|
1866
|
+
(st->ndim = 1, composed_strides[0] = bytes) yields exactly one
|
|
1867
|
+
contig run. Note st->ndim diverges from src->ndim here — the
|
|
1868
|
+
iterator's logical ndim is 1, but the view itself can be N-D
|
|
1869
|
+
(the materialised buffer is flat). */
|
|
1870
|
+
st->ndim = 1;
|
|
1871
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1872
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1873
|
+
st->composed_strides[0] = src->bytes;
|
|
1874
|
+
st->composed_base = 0;
|
|
1875
|
+
st->slab_n = src->elements;
|
|
1876
|
+
st->total_slabs = 1;
|
|
1877
|
+
st->slabs_emitted = 0;
|
|
1878
|
+
st->chunk_size = st->slab_n;
|
|
1879
|
+
st->outer_idx = NULL;
|
|
1880
|
+
|
|
1881
|
+
/* Step 6: mask gather, same as L1 descriptor branch. */
|
|
1882
|
+
if ( ca_has_mask(src) ) {
|
|
1883
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1884
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1885
|
+
if ( src->elements > 0 ) {
|
|
1886
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1887
|
+
}
|
|
1888
|
+
st->alias_mask = st->scratch_mask;
|
|
1889
|
+
}
|
|
1890
|
+
return CA_ITER_OK;
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
/* === CAStride / entity L2 path === */
|
|
1894
|
+
memset(st, 0, sizeof(*st));
|
|
1895
|
+
st->src = src;
|
|
1896
|
+
st->src_kind = CA_ITER_SRC_CASTRIDE;
|
|
1897
|
+
st->level = 2;
|
|
1898
|
+
st->policy = policy;
|
|
1899
|
+
st->ndim = src->ndim;
|
|
1900
|
+
st->flags = flags;
|
|
1901
|
+
st->bytes = src->bytes;
|
|
1902
|
+
st->axes = axes;
|
|
1903
|
+
st->naxes = naxes;
|
|
1904
|
+
|
|
1905
|
+
/* Compute composed strides + base. For entity / CAStride contig we
|
|
1906
|
+
synthesise row-major byte strides so next_slab_strided's offset
|
|
1907
|
+
math is uniform across alias modes; for CAStride non-contig we
|
|
1908
|
+
compose leaf strides up to the root entity via the substrate. */
|
|
1909
|
+
int8_t nd = src->ndim;
|
|
1910
|
+
int use_strided = 0;
|
|
1911
|
+
int use_view_scratch = 0;
|
|
1912
|
+
CArray *root = NULL;
|
|
1913
|
+
|
|
1914
|
+
if ( ca_iter_can_alias(src, 1) ) {
|
|
1915
|
+
/* Entity or CAStride contig: stride = row-major bytes, base = 0. */
|
|
1916
|
+
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
1917
|
+
src->dim, nd, src->bytes);
|
|
1918
|
+
st->composed_base = 0;
|
|
1919
|
+
} else {
|
|
1920
|
+
/* CAStride non-contig: leaf->root compose. */
|
|
1921
|
+
ca_size_t cs[CA_RANK_MAX];
|
|
1922
|
+
ca_size_t base;
|
|
1923
|
+
ca_stride_compose_to_root((CAStride *) src, &root, cs, &base);
|
|
1924
|
+
if ( !ca_root_lends_no_memory(root) ) {
|
|
1925
|
+
memcpy(st->composed_strides, cs, nd * sizeof(ca_size_t));
|
|
1926
|
+
st->composed_base = base;
|
|
1927
|
+
use_strided = 1;
|
|
1928
|
+
} else {
|
|
1929
|
+
/* Root holds nothing to read through: reaching root->ptr costs one
|
|
1930
|
+
whole-root materialise no matter how few cells the view touches —
|
|
1931
|
+
the same cliff the xfer path avoids by asking for a region. Take
|
|
1932
|
+
the view's own region protocol instead: ca_copy_data walks the
|
|
1933
|
+
leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
|
|
1934
|
+
CAObject asks for exactly that block. The gathered buffer is view
|
|
1935
|
+
row-major, so the slab arithmetic below is the contig-alias case
|
|
1936
|
+
unchanged. */
|
|
1937
|
+
root = NULL;
|
|
1938
|
+
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
1939
|
+
src->dim, nd, src->bytes);
|
|
1940
|
+
st->composed_base = 0;
|
|
1941
|
+
use_view_scratch = 1;
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
/* Phase A: CA_SLAB_AXES branch. Partition axes into slab vs outer,
|
|
1946
|
+
populate per-axis dims / strides (both data byte strides from
|
|
1947
|
+
composed_strides[] and mask element strides from view row-major
|
|
1948
|
+
dim products), allocate outer_idx if outer_ndim > 0, attach data
|
|
1949
|
+
buffer (alias or strided), gather mask if present.
|
|
1950
|
+
|
|
1951
|
+
Layout invariant: slab_axes_buf and outer_axes are both stored
|
|
1952
|
+
sort-ascending (in source-axis order), so multi-axis CA_SLAB_AXES
|
|
1953
|
+
is canonical regardless of the user's input order. */
|
|
1954
|
+
if ( policy == CA_SLAB_AXES ) {
|
|
1955
|
+
/* Validate axes input. */
|
|
1956
|
+
if ( axes == NULL || naxes <= 0 || naxes > nd ) {
|
|
1957
|
+
return CA_ITER_ERR_POLICY;
|
|
1958
|
+
}
|
|
1959
|
+
int8_t in_slab[CA_RANK_MAX];
|
|
1960
|
+
int8_t k;
|
|
1961
|
+
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
1962
|
+
for ( k = 0; k < naxes; k++ ) {
|
|
1963
|
+
int8_t ax = axes[k];
|
|
1964
|
+
if ( ax < 0 || ax >= nd ) return CA_ITER_ERR_POLICY;
|
|
1965
|
+
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY; /* duplicate */
|
|
1966
|
+
in_slab[ax] = 1;
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
/* View row-major element strides (= mask scratch layout strides). */
|
|
1970
|
+
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1971
|
+
{
|
|
1972
|
+
ca_size_t s = 1;
|
|
1973
|
+
for ( k = nd - 1; k >= 0; k-- ) {
|
|
1974
|
+
row_elem_strides[k] = s;
|
|
1975
|
+
s *= src->dim[k];
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
/* Partition axes (ascending order). */
|
|
1980
|
+
int8_t sp = 0, op = 0;
|
|
1981
|
+
st->slab_elements = 1;
|
|
1982
|
+
for ( k = 0; k < nd; k++ ) {
|
|
1983
|
+
if ( in_slab[k] ) {
|
|
1984
|
+
st->slab_axes_buf[sp] = k;
|
|
1985
|
+
st->slab_dims[sp] = src->dim[k];
|
|
1986
|
+
st->slab_strides[sp] = st->composed_strides[k];
|
|
1987
|
+
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1988
|
+
st->slab_elements *= src->dim[k];
|
|
1989
|
+
sp++;
|
|
1990
|
+
} else {
|
|
1991
|
+
st->outer_axes[op] = k;
|
|
1992
|
+
st->outer_dims[op] = src->dim[k];
|
|
1993
|
+
st->outer_strides[op] = st->composed_strides[k];
|
|
1994
|
+
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1995
|
+
op++;
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
st->slab_ndim = sp;
|
|
1999
|
+
st->outer_ndim = op;
|
|
2000
|
+
|
|
2001
|
+
/* total_slabs = Π outer_dims. Empty product (all-axes case) = 1
|
|
2002
|
+
→ single slab = whole array (D1.4 WHOLE-equivalent). */
|
|
2003
|
+
ca_size_t total = 1;
|
|
2004
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
2005
|
+
st->total_slabs = total;
|
|
2006
|
+
st->slab_n = st->slab_elements; /* mirror to legacy field */
|
|
2007
|
+
st->slabs_emitted = 0;
|
|
2008
|
+
st->chunk_size = st->slab_n;
|
|
2009
|
+
|
|
2010
|
+
if ( st->outer_ndim > 0 ) {
|
|
2011
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
2012
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
2013
|
+
} else {
|
|
2014
|
+
st->outer_idx = NULL;
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
if ( use_strided ) {
|
|
2018
|
+
st->root = root;
|
|
2019
|
+
ca_attach(root);
|
|
2020
|
+
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
2021
|
+
st->alias_ptr = (char *) root->ptr;
|
|
2022
|
+
} else if ( use_view_scratch ) {
|
|
2023
|
+
st->root = NULL;
|
|
2024
|
+
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
2025
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0
|
|
2026
|
+
? st->scratch_cap : 1);
|
|
2027
|
+
if ( src->elements > 0 ) {
|
|
2028
|
+
ca_copy_data(src, st->scratch_ptr);
|
|
2029
|
+
}
|
|
2030
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
2031
|
+
st->alias_ptr = st->scratch_ptr;
|
|
2032
|
+
} else {
|
|
2033
|
+
st->root = NULL;
|
|
2034
|
+
ca_attach(src);
|
|
2035
|
+
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
2036
|
+
st->alias_ptr = (char *) src->ptr;
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
/* Mask: gather to scratch_mask in view row-major order (= same as
|
|
2040
|
+
L1 path). The mask layout matches view->dim row-major, which is
|
|
2041
|
+
what slab_mask_strides / outer_mask_strides walk. */
|
|
2042
|
+
if ( ca_has_mask(src) ) {
|
|
2043
|
+
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
2044
|
+
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
2045
|
+
if ( src->elements > 0 ) {
|
|
2046
|
+
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
2047
|
+
}
|
|
2048
|
+
st->alias_mask = st->scratch_mask;
|
|
2049
|
+
}
|
|
2050
|
+
return CA_ITER_OK;
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
/* Outer prefix axes [0..ndim-2] drive total_slabs; innermost axis is
|
|
2054
|
+
the slab. 0-d / 1-d sources collapse to a single yield. */
|
|
2055
|
+
if ( nd <= 1 ) {
|
|
2056
|
+
st->total_slabs = 1;
|
|
2057
|
+
st->slab_n = src->elements;
|
|
2058
|
+
st->outer_idx = NULL;
|
|
2059
|
+
} else {
|
|
2060
|
+
ca_size_t total = 1;
|
|
2061
|
+
int8_t k;
|
|
2062
|
+
for ( k = 0; k < nd - 1; k++ ) total *= src->dim[k];
|
|
2063
|
+
st->total_slabs = total;
|
|
2064
|
+
st->slab_n = src->dim[nd - 1];
|
|
2065
|
+
st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
|
|
2066
|
+
for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
|
|
2067
|
+
}
|
|
2068
|
+
st->slabs_emitted = 0;
|
|
2069
|
+
st->chunk_size = st->slab_n;
|
|
2070
|
+
|
|
2071
|
+
if ( use_strided ) {
|
|
2072
|
+
st->root = root;
|
|
2073
|
+
ca_attach(root);
|
|
2074
|
+
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
2075
|
+
st->alias_ptr = (char *) root->ptr;
|
|
2076
|
+
} else if ( use_view_scratch ) {
|
|
2077
|
+
st->root = NULL;
|
|
2078
|
+
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
2079
|
+
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0
|
|
2080
|
+
? st->scratch_cap : 1);
|
|
2081
|
+
if ( src->elements > 0 ) {
|
|
2082
|
+
ca_copy_data(src, st->scratch_ptr);
|
|
2083
|
+
}
|
|
2084
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
2085
|
+
st->alias_ptr = st->scratch_ptr;
|
|
2086
|
+
} else {
|
|
2087
|
+
st->root = NULL;
|
|
2088
|
+
ca_attach(src);
|
|
2089
|
+
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
2090
|
+
st->alias_ptr = (char *) src->ptr;
|
|
2091
|
+
}
|
|
2092
|
+
return CA_ITER_OK;
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
int
|
|
2096
|
+
ca_iter_state_next_slab (ca_iter_state *st,
|
|
2097
|
+
char **out_ptr,
|
|
2098
|
+
boolean8_t **out_mask,
|
|
2099
|
+
ca_size_t *out_n)
|
|
2100
|
+
{
|
|
2101
|
+
if ( st == NULL || st->level != 1
|
|
2102
|
+
|| st->slabs_emitted >= st->total_slabs ) {
|
|
2103
|
+
if ( out_ptr ) *out_ptr = NULL;
|
|
2104
|
+
if ( out_mask ) *out_mask = NULL;
|
|
2105
|
+
if ( out_n ) *out_n = 0;
|
|
2106
|
+
return 0;
|
|
2107
|
+
}
|
|
2108
|
+
/* L1 WHOLE policy: a single slab whose ptr is either parent->ptr
|
|
2109
|
+
(alias) or the scratch buffer. alias_mask is populated in init
|
|
2110
|
+
when the source carries a mask (NULL otherwise). */
|
|
2111
|
+
if ( out_ptr ) *out_ptr = st->alias_ptr;
|
|
2112
|
+
if ( out_mask ) *out_mask = st->alias_mask;
|
|
2113
|
+
if ( out_n ) *out_n = st->slab_n;
|
|
2114
|
+
st->slabs_emitted += 1;
|
|
2115
|
+
return 1;
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
int
|
|
2119
|
+
ca_iter_state_next_slab_strided (ca_iter_state *st,
|
|
2120
|
+
char **out_ptr,
|
|
2121
|
+
boolean8_t **out_mask,
|
|
2122
|
+
ca_size_t *out_n,
|
|
2123
|
+
ca_size_t *out_stride_bytes)
|
|
2124
|
+
{
|
|
2125
|
+
if ( st == NULL || st->level != 2
|
|
2126
|
+
|| st->slabs_emitted >= st->total_slabs ) {
|
|
2127
|
+
if ( out_ptr ) *out_ptr = NULL;
|
|
2128
|
+
if ( out_mask ) *out_mask = NULL;
|
|
2129
|
+
if ( out_n ) *out_n = 0;
|
|
2130
|
+
if ( out_stride_bytes ) *out_stride_bytes = 0;
|
|
2131
|
+
return 0;
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
int8_t nd = st->ndim;
|
|
2135
|
+
ca_size_t inner_st = (nd > 0)
|
|
2136
|
+
? st->composed_strides[nd - 1]
|
|
2137
|
+
: st->bytes;
|
|
2138
|
+
ca_size_t off = st->composed_base;
|
|
2139
|
+
int8_t k;
|
|
2140
|
+
|
|
2141
|
+
/* Sum outer prefix offset. For F-2 descriptor L2 alias the prefix can
|
|
2142
|
+
contain INDEX axes (and, once OOB fill_slab lands, SHIFT axes), so
|
|
2143
|
+
we delegate to the pre-classified prefix[] engine. Otherwise (=
|
|
2144
|
+
CAStride / entity outer = STRIDE-only by construction) use the
|
|
2145
|
+
direct sum that has been the L2 inner loop since step 3. For
|
|
2146
|
+
0-d / 1-d sources, the loop / call is a no-op and off stays at
|
|
2147
|
+
composed_base. */
|
|
2148
|
+
if ( st->outer_idx != NULL ) {
|
|
2149
|
+
if ( st->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
2150
|
+
int oob = 0;
|
|
2151
|
+
off += ca_axis_dispatch_prefix_offset(st->prefix, st->outer_idx,
|
|
2152
|
+
nd - 1, &oob);
|
|
2153
|
+
/* oob unreachable: outer SHIFT was downgraded in init_l2. */
|
|
2154
|
+
} else {
|
|
2155
|
+
for ( k = 0; k < nd - 1; k++ ) {
|
|
2156
|
+
off += st->outer_idx[k] * st->composed_strides[k];
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
if ( out_ptr ) *out_ptr = st->alias_ptr + off;
|
|
2162
|
+
/* mask layout mirrors value layout, but mask is contig boolean8_t
|
|
2163
|
+
when alias_mask is a scratch buffer (step 6 baseline = always
|
|
2164
|
+
gather mask into scratch_mask for uniformity). Per-slab mask
|
|
2165
|
+
offset = i in the outer cursor (= slabs_emitted at this point). */
|
|
2166
|
+
if ( out_mask ) {
|
|
2167
|
+
*out_mask = st->alias_mask
|
|
2168
|
+
? st->alias_mask + st->slabs_emitted * st->slab_n
|
|
2169
|
+
: NULL;
|
|
2170
|
+
}
|
|
2171
|
+
if ( out_n ) *out_n = st->slab_n;
|
|
2172
|
+
if ( out_stride_bytes ) *out_stride_bytes = inner_st;
|
|
2173
|
+
|
|
2174
|
+
st->slabs_emitted += 1;
|
|
2175
|
+
|
|
2176
|
+
/* Advance outer_idx row-major (least-significant axis innermost,
|
|
2177
|
+
so we tick outer_idx[ndim-2] first). */
|
|
2178
|
+
if ( st->outer_idx != NULL ) {
|
|
2179
|
+
for ( k = nd - 2; k >= 0; k-- ) {
|
|
2180
|
+
if ( ++st->outer_idx[k] < st->src->dim[k] ) break;
|
|
2181
|
+
st->outer_idx[k] = 0;
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
return 1;
|
|
2185
|
+
}
|
|
2186
|
+
|
|
2187
|
+
int
|
|
2188
|
+
ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
2189
|
+
char **out_ptr,
|
|
2190
|
+
boolean8_t **out_mask)
|
|
2191
|
+
{
|
|
2192
|
+
if ( st == NULL || st->policy != CA_SLAB_AXES || st->level != 2
|
|
2193
|
+
|| st->slabs_emitted >= st->total_slabs ) {
|
|
2194
|
+
if ( out_ptr ) *out_ptr = NULL;
|
|
2195
|
+
if ( out_mask ) *out_mask = NULL;
|
|
2196
|
+
return 0;
|
|
2197
|
+
}
|
|
2198
|
+
|
|
2199
|
+
/* Phase C T3 specialised path (B-1b, C.1b, 2026-05-27): innermost
|
|
2200
|
+
slab axis is STRIDE, no SHIFT axes anywhere. Hoist outer + non-
|
|
2201
|
+
innermost-slab axes (= per-cell switch evaluated once per slab row),
|
|
2202
|
+
inner = pure STRIDE linear memcpy (= SIMD-friendly contig run, no
|
|
2203
|
+
engine per-cell dispatch). Target: 1.5-1.8x win for INDEX slab
|
|
2204
|
+
with innermost STRIDE (= grid / select sparse projection use cases). */
|
|
2205
|
+
if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB_HOIST ) {
|
|
2206
|
+
int8_t inner_view_ax = st->slab_axes_buf[st->slab_ndim - 1];
|
|
2207
|
+
ca_size_t inner_count = st->descs[inner_view_ax].count;
|
|
2208
|
+
ca_size_t inner_pstride = st->pstrides[inner_view_ax];
|
|
2209
|
+
ca_size_t inner_byte_step = st->descs[inner_view_ax].step * inner_pstride;
|
|
2210
|
+
ca_size_t inner_byte_base = st->descs[inner_view_ax].start * inner_pstride;
|
|
2211
|
+
ca_size_t bytes = st->bytes;
|
|
2212
|
+
|
|
2213
|
+
/* Outer contribution (hoisted, computed once per next_slab_axes
|
|
2214
|
+
call): walks outer_axes with their kind-specific offset. No
|
|
2215
|
+
SHIFT here (init ruled out SHIFT-anywhere). */
|
|
2216
|
+
ca_size_t outer_off = 0;
|
|
2217
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2218
|
+
int8_t ax = st->outer_axes[m];
|
|
2219
|
+
ca_size_t pos = st->outer_idx[m];
|
|
2220
|
+
if ( st->descs[ax].kind == CA_AXIS_KIND_STRIDE ) {
|
|
2221
|
+
outer_off += (st->descs[ax].start + pos * st->descs[ax].step)
|
|
2222
|
+
* st->pstrides[ax];
|
|
2223
|
+
} else { /* INDEX */
|
|
2224
|
+
outer_off += st->descs[ax].indices[pos] * st->pstrides[ax];
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2228
|
+
/* Non-innermost slab axes: walk row-major via single linear cursor.
|
|
2229
|
+
For sp = 1 (single slab axis = innermost STRIDE), the outer
|
|
2230
|
+
row-major loop runs once with nonin_off = 0. */
|
|
2231
|
+
int8_t nonin_n = st->slab_ndim - 1;
|
|
2232
|
+
int8_t nonin_view_ax[CA_RANK_MAX];
|
|
2233
|
+
ca_size_t nonin_count[CA_RANK_MAX];
|
|
2234
|
+
ca_size_t nonin_idx[CA_RANK_MAX];
|
|
2235
|
+
ca_size_t nonin_total = 1;
|
|
2236
|
+
for ( int8_t s = 0; s < nonin_n; s++ ) {
|
|
2237
|
+
nonin_view_ax[s] = st->slab_axes_buf[s];
|
|
2238
|
+
nonin_count[s] = st->descs[nonin_view_ax[s]].count;
|
|
2239
|
+
nonin_idx[s] = 0;
|
|
2240
|
+
nonin_total *= nonin_count[s];
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
char *parent_base = (char *) st->root->ptr + outer_off + inner_byte_base;
|
|
2244
|
+
for ( ca_size_t nlin = 0; nlin < nonin_total; nlin++ ) {
|
|
2245
|
+
/* Compute non-innermost-slab contribution at current nonin_idx. */
|
|
2246
|
+
ca_size_t nonin_off = 0;
|
|
2247
|
+
for ( int8_t s = 0; s < nonin_n; s++ ) {
|
|
2248
|
+
int8_t ax = nonin_view_ax[s];
|
|
2249
|
+
ca_size_t pos = nonin_idx[s];
|
|
2250
|
+
if ( st->descs[ax].kind == CA_AXIS_KIND_STRIDE ) {
|
|
2251
|
+
nonin_off += (st->descs[ax].start + pos * st->descs[ax].step)
|
|
2252
|
+
* st->pstrides[ax];
|
|
2253
|
+
} else { /* INDEX */
|
|
2254
|
+
nonin_off += st->descs[ax].indices[pos] * st->pstrides[ax];
|
|
2255
|
+
}
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
/* Inner loop: pure STRIDE linear copy of inner_count cells.
|
|
2259
|
+
For unit-bytes step == bytes (= contig run) the compiler can
|
|
2260
|
+
hoist this into a single memcpy. Otherwise per-cell memcpy
|
|
2261
|
+
with linear stride (= SIMD-friendly). */
|
|
2262
|
+
char *dst = st->scratch_ptr + nlin * inner_count * bytes;
|
|
2263
|
+
char *src_base = parent_base + nonin_off;
|
|
2264
|
+
if ( inner_byte_step == (ca_size_t) bytes ) {
|
|
2265
|
+
memcpy(dst, src_base, inner_count * bytes);
|
|
2266
|
+
} else {
|
|
2267
|
+
for ( ca_size_t i = 0; i < inner_count; i++ ) {
|
|
2268
|
+
memcpy(dst + i * bytes, src_base + i * inner_byte_step, bytes);
|
|
2269
|
+
}
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
/* Advance nonin_idx row-major (last axis ticks fastest). */
|
|
2273
|
+
for ( int8_t s = nonin_n - 1; s >= 0; s-- ) {
|
|
2274
|
+
if ( ++nonin_idx[s] < nonin_count[s] ) break;
|
|
2275
|
+
nonin_idx[s] = 0;
|
|
2276
|
+
}
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
/* Mask offset (= same as fallback path, whole-view mask scratch). */
|
|
2280
|
+
ca_size_t mask_off = 0;
|
|
2281
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2282
|
+
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
if ( out_ptr ) *out_ptr = st->scratch_ptr;
|
|
2286
|
+
if ( out_mask ) *out_mask = st->scratch_mask ? st->scratch_mask + mask_off : NULL;
|
|
2287
|
+
|
|
2288
|
+
st->slabs_emitted += 1;
|
|
2289
|
+
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
2290
|
+
if ( ++st->outer_idx[m] < st->outer_dims[m] ) break;
|
|
2291
|
+
st->outer_idx[m] = 0;
|
|
2292
|
+
}
|
|
2293
|
+
return 1;
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
/* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18): CAStack
|
|
2297
|
+
source + K-axis (k_axis) in outer iter. Each slab aliases a region
|
|
2298
|
+
inside parents[k]->ptr where k = outer_idx[K_outer_pos]. Parent
|
|
2299
|
+
inner byte offset = Σ outer_idx[m] * stack_parent_strides[parent_ax]
|
|
2300
|
+
over all outer axes except the K-axis itself. Mask: parallel alias
|
|
2301
|
+
into parents[k]->mask->ptr + mask_off. Zero copy / zero scratch /
|
|
2302
|
+
parent entity bandwidth.
|
|
2303
|
+
|
|
2304
|
+
K.3 (2026-06-20): parent_ax derivation generalised for arbitrary
|
|
2305
|
+
k_axis -- stack axis s != k_axis maps to parent axis s if
|
|
2306
|
+
s < k_axis else s - 1. */
|
|
2307
|
+
if ( st->alias_mode == CA_ITER_ALIAS_STACK_OUTER_K ) {
|
|
2308
|
+
int8_t kpos = st->stack_k_outer_pos;
|
|
2309
|
+
int32_t k = (int32_t) st->outer_idx[kpos];
|
|
2310
|
+
int8_t k_axis = ((CAStack *) st->src)->k_axis;
|
|
2311
|
+
|
|
2312
|
+
ca_size_t parent_off = 0; /* byte offset within parent data */
|
|
2313
|
+
ca_size_t mask_off = 0; /* element offset within parent mask */
|
|
2314
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2315
|
+
if ( m == kpos ) continue;
|
|
2316
|
+
int8_t stack_ax = st->outer_axes[m];
|
|
2317
|
+
int8_t parent_ax = (stack_ax < k_axis) ? stack_ax : (stack_ax - 1);
|
|
2318
|
+
parent_off += st->outer_idx[m] * st->stack_parent_strides[parent_ax];
|
|
2319
|
+
mask_off += st->outer_idx[m] * st->stack_parent_mask_strides[parent_ax];
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
if ( out_ptr ) *out_ptr = st->stack_parent_ptrs[k] + parent_off;
|
|
2323
|
+
if ( out_mask ) {
|
|
2324
|
+
*out_mask = st->stack_parent_mask_ptrs
|
|
2325
|
+
? st->stack_parent_mask_ptrs[k] + mask_off
|
|
2326
|
+
: NULL;
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
st->slabs_emitted += 1;
|
|
2330
|
+
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
2331
|
+
if ( ++st->outer_idx[m] < st->outer_dims[m] ) break;
|
|
2332
|
+
st->outer_idx[m] = 0;
|
|
2333
|
+
}
|
|
2334
|
+
return 1;
|
|
2335
|
+
}
|
|
2336
|
+
|
|
2337
|
+
/* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2: CAStack direct
|
|
2338
|
+
per-parent ptr access path. For each outer iter, compute the
|
|
2339
|
+
parent inner byte offset (= same across all K parents, uniform
|
|
2340
|
+
shape), then K-fold direct memcpy gather from cached parent ptrs
|
|
2341
|
+
into scratch. No ca_xfer_stride dispatch; per-cell cost = pointer
|
|
2342
|
+
arith + memcpy(bytes). For f64 the memcpy(8) compiles to a single
|
|
2343
|
+
mov, so the inner loop is tight.
|
|
2344
|
+
|
|
2345
|
+
Note: scope-narrowed to slab_axes == [0] at init, so outer_axes are
|
|
2346
|
+
stack axes 1..N-1 = parent axes 0..parent_ndim-1. outer_idx[m]
|
|
2347
|
+
maps directly to parent axis (m) (since the m-th outer axis is
|
|
2348
|
+
stack axis m+1 = parent axis m). */
|
|
2349
|
+
if ( st->alias_mode == CA_ITER_ALIAS_STACK ) {
|
|
2350
|
+
/* pilot/castack-axis0-loop-interchange: tile cache. Outer iter
|
|
2351
|
+
walks parent storage in row-major order so parent_off increments
|
|
2352
|
+
by `bytes` per call monotonically. Refill TILE fibers at once
|
|
2353
|
+
via K contig parent streams (= TILE consecutive cells from each
|
|
2354
|
+
parent), transposed into cache[t][k] layout. Subsequent (TILE-1)
|
|
2355
|
+
next_slab calls alias into the cache. */
|
|
2356
|
+
if ( st->stack_tile_pos >= st->stack_tile_have ) {
|
|
2357
|
+
ca_size_t bytes = st->bytes;
|
|
2358
|
+
ca_size_t parent_off = 0;
|
|
2359
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2360
|
+
parent_off += st->outer_idx[m] * st->stack_parent_strides[m];
|
|
2361
|
+
}
|
|
2362
|
+
ca_size_t want = st->stack_tile_cap;
|
|
2363
|
+
ca_size_t remaining = st->total_slabs - st->slabs_emitted;
|
|
2364
|
+
if ( want > remaining ) want = remaining;
|
|
2365
|
+
st->stack_tile_have = want;
|
|
2366
|
+
st->stack_tile_pos = 0;
|
|
2367
|
+
|
|
2368
|
+
char **pptrs = st->stack_parent_ptrs;
|
|
2369
|
+
int32_t K = st->stack_n_parents;
|
|
2370
|
+
char *cache = st->stack_tile_cache;
|
|
2371
|
+
ca_size_t stride_t = (ca_size_t) K * bytes; /* cache[t][*] stride */
|
|
2372
|
+
|
|
2373
|
+
if ( bytes == 8 ) {
|
|
2374
|
+
/* f64 / i64 hot path: store one cell per inner iter, compiler
|
|
2375
|
+
can keep `src` in a vector reg and stream cleanly. */
|
|
2376
|
+
for ( int32_t kk = 0; kk < K; kk++ ) {
|
|
2377
|
+
const uint64_t *src = (const uint64_t *)
|
|
2378
|
+
(pptrs[kk] + parent_off);
|
|
2379
|
+
uint64_t *dst = (uint64_t *) (cache + (ca_size_t) kk * bytes);
|
|
2380
|
+
for ( ca_size_t t = 0; t < want; t++ ) {
|
|
2381
|
+
*(uint64_t *)((char *) dst + t * stride_t) = src[t];
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
} else if ( bytes == 4 ) {
|
|
2385
|
+
for ( int32_t kk = 0; kk < K; kk++ ) {
|
|
2386
|
+
const uint32_t *src = (const uint32_t *)
|
|
2387
|
+
(pptrs[kk] + parent_off);
|
|
2388
|
+
uint32_t *dst = (uint32_t *) (cache + (ca_size_t) kk * bytes);
|
|
2389
|
+
for ( ca_size_t t = 0; t < want; t++ ) {
|
|
2390
|
+
*(uint32_t *)((char *) dst + t * stride_t) = src[t];
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
} else {
|
|
2394
|
+
for ( int32_t kk = 0; kk < K; kk++ ) {
|
|
2395
|
+
const char *src = pptrs[kk] + parent_off;
|
|
2396
|
+
char *dst = cache + (ca_size_t) kk * bytes;
|
|
2397
|
+
for ( ca_size_t t = 0; t < want; t++ ) {
|
|
2398
|
+
memcpy(dst + t * stride_t, src + t * bytes, bytes);
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
|
|
2404
|
+
if ( out_ptr ) {
|
|
2405
|
+
*out_ptr = st->stack_tile_cache
|
|
2406
|
+
+ st->stack_tile_pos
|
|
2407
|
+
* (ca_size_t) st->stack_n_parents * st->bytes;
|
|
2408
|
+
}
|
|
2409
|
+
if ( out_mask ) *out_mask = NULL;
|
|
2410
|
+
|
|
2411
|
+
st->stack_tile_pos += 1;
|
|
2412
|
+
st->slabs_emitted += 1;
|
|
2413
|
+
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
2414
|
+
if ( ++st->outer_idx[m] < st->outer_dims[m] ) break;
|
|
2415
|
+
st->outer_idx[m] = 0;
|
|
2416
|
+
}
|
|
2417
|
+
return 1;
|
|
2418
|
+
}
|
|
2419
|
+
|
|
2420
|
+
/* Phase C T3 fallback (C.1, 2026-05-27): per-slab materialise via
|
|
2421
|
+
ca_axis_dispatch_gather with a subset descriptor built by pinning
|
|
2422
|
+
outer axes at the current outer_idx position (D1.2 (A): caller-side
|
|
2423
|
+
subset construction, engine API unchanged). scratch_ptr is sized
|
|
2424
|
+
for one max slab and reused across iters (D1.1 (B)). */
|
|
2425
|
+
if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB ) {
|
|
2426
|
+
ca_axis_desc_t subset_descs[CA_RANK_MAX];
|
|
2427
|
+
memcpy(subset_descs, st->descs, st->ndim * sizeof(ca_axis_desc_t));
|
|
2428
|
+
|
|
2429
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2430
|
+
int8_t ax = st->outer_axes[m];
|
|
2431
|
+
ca_size_t pos = st->outer_idx[m];
|
|
2432
|
+
switch ( st->descs[ax].kind ) {
|
|
2433
|
+
case CA_AXIS_KIND_STRIDE:
|
|
2434
|
+
subset_descs[ax].start = st->descs[ax].start
|
|
2435
|
+
+ pos * st->descs[ax].step;
|
|
2436
|
+
subset_descs[ax].step = 0;
|
|
2437
|
+
subset_descs[ax].count = 1;
|
|
2438
|
+
break;
|
|
2439
|
+
case CA_AXIS_KIND_INDEX:
|
|
2440
|
+
/* Borrow into the indices[] array at offset pos; count=1
|
|
2441
|
+
means engine reads indices[0] which is original indices[pos].
|
|
2442
|
+
No allocation, no mutation of the producer's array. */
|
|
2443
|
+
subset_descs[ax].indices = &st->descs[ax].indices[pos];
|
|
2444
|
+
subset_descs[ax].count = 1;
|
|
2445
|
+
break;
|
|
2446
|
+
case CA_AXIS_KIND_SHIFT:
|
|
2447
|
+
/* SHIFT outer pinned at pos: collapse to a count=1 axis at the
|
|
2448
|
+
projected start. size0 / policy unchanged so engine's
|
|
2449
|
+
bound check + bound_fill writeback still applies if the
|
|
2450
|
+
projected position is OOB. */
|
|
2451
|
+
subset_descs[ax].start = st->descs[ax].start
|
|
2452
|
+
+ pos * st->descs[ax].step;
|
|
2453
|
+
subset_descs[ax].step = 0;
|
|
2454
|
+
subset_descs[ax].count = 1;
|
|
2455
|
+
break;
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
/* CAWindow fill value capture (= same lookup as B.1.5 init). */
|
|
2460
|
+
const void *bound_fill = NULL;
|
|
2461
|
+
if ( ca_func[st->src->obj_type].attach == ca_window_func.attach ) {
|
|
2462
|
+
bound_fill = ((CAWindow *) st->src)->fill;
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
ca_axis_dispatch_gather(st->root, st->parent_axis_dims, subset_descs,
|
|
2466
|
+
st->ndim, st->bytes, st->slab_elements,
|
|
2467
|
+
bound_fill, st->scratch_ptr);
|
|
2468
|
+
|
|
2469
|
+
/* Mask offset into whole-view scratch_mask (T3 path keeps mask
|
|
2470
|
+
layout view-row-major to match slab_mask_strides). */
|
|
2471
|
+
ca_size_t mask_off = 0;
|
|
2472
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2473
|
+
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
2474
|
+
}
|
|
2475
|
+
|
|
2476
|
+
if ( out_ptr ) *out_ptr = st->scratch_ptr;
|
|
2477
|
+
if ( out_mask ) *out_mask = st->scratch_mask ? st->scratch_mask + mask_off : NULL;
|
|
2478
|
+
|
|
2479
|
+
st->slabs_emitted += 1;
|
|
2480
|
+
/* Advance outer_idx row-major (innermost outer axis ticks first). */
|
|
2481
|
+
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
2482
|
+
if ( ++st->outer_idx[m] < st->outer_dims[m] ) break;
|
|
2483
|
+
st->outer_idx[m] = 0;
|
|
2484
|
+
}
|
|
2485
|
+
return 1;
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
/* Per-slab base offsets via outer_idx walk. Data offset uses
|
|
2489
|
+
outer_strides (byte units); mask offset uses outer_mask_strides
|
|
2490
|
+
(element units, view row-major). Both are zero when outer_ndim
|
|
2491
|
+
== 0 (= all-axes WHOLE-equivalent case).
|
|
2492
|
+
|
|
2493
|
+
STRIDE outer axes (Phase A SRC_CASTRIDE, and Phase B
|
|
2494
|
+
SRC_DESCRIPTOR STRIDE-kind axes) use outer_idx[m] directly as the
|
|
2495
|
+
multiplier. INDEX outer axes (Phase B SRC_DESCRIPTOR INDEX-kind)
|
|
2496
|
+
need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
|
|
2497
|
+
Mask offset uses outer_idx[m] directly in both cases (= mask
|
|
2498
|
+
scratch is in view row-major order, outer_idx walks view-axis
|
|
2499
|
+
positions). */
|
|
2500
|
+
ca_size_t data_off = st->composed_base;
|
|
2501
|
+
ca_size_t mask_off = 0;
|
|
2502
|
+
int has_descs = (st->src_kind == CA_ITER_SRC_DESCRIPTOR
|
|
2503
|
+
&& st->desc_ndim > 0);
|
|
2504
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2505
|
+
ca_size_t multiplier = st->outer_idx[m];
|
|
2506
|
+
if ( has_descs ) {
|
|
2507
|
+
int8_t ax = st->outer_axes[m];
|
|
2508
|
+
if ( st->descs[ax].kind == CA_AXIS_KIND_INDEX ) {
|
|
2509
|
+
multiplier = st->descs[ax].indices[st->outer_idx[m]];
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
data_off += multiplier * st->outer_strides[m];
|
|
2513
|
+
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
2514
|
+
}
|
|
2515
|
+
|
|
2516
|
+
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1: per-fiber fused dispatch.
|
|
2517
|
+
When init_l2 selected CA_ITER_ALIAS_PER_FIBER_FUSED, there is no
|
|
2518
|
+
whole-view buffer to alias from. Build the fiber region from
|
|
2519
|
+
outer_idx + fiber_axis and call ca_xfer_stride(src, ..., GET)
|
|
2520
|
+
directly into fiber_data_scratch. For fused-aware views (X.1
|
|
2521
|
+
OOB-fused / X.4 transform-fused) this routes to a 1-pass per-
|
|
2522
|
+
region path.
|
|
2523
|
+
|
|
2524
|
+
last_data_off carries the fiber region's outer footprint so
|
|
2525
|
+
sync_slab can reconstruct the same region for WRITE PUT. Encode
|
|
2526
|
+
it as the linear outer_idx position scaled by fiber_axis stride
|
|
2527
|
+
equivalence; for PER_FIBER_FUSED sync_slab reads outer_idx state
|
|
2528
|
+
directly rather than data_off so the value is informational. */
|
|
2529
|
+
if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
|
|
2530
|
+
ca_size_t fiber_n = st->slab_dims[0];
|
|
2531
|
+
ca_size_t need = fiber_n * st->bytes;
|
|
2532
|
+
int8_t fiber_ax = st->fiber_axis;
|
|
2533
|
+
int8_t nd = st->ndim;
|
|
2534
|
+
ca_size_t starts[CA_RANK_MAX];
|
|
2535
|
+
ca_size_t counts[CA_RANK_MAX];
|
|
2536
|
+
|
|
2537
|
+
if ( st->fiber_data_scratch_cap < need ) {
|
|
2538
|
+
if ( st->fiber_data_scratch ) xfree(st->fiber_data_scratch);
|
|
2539
|
+
st->fiber_data_scratch = (char *) xmalloc(need > 0 ? need : 1);
|
|
2540
|
+
st->fiber_data_scratch_cap = need;
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
/* Build fiber region: fiber_axis spans the full fiber, all other
|
|
2544
|
+
axes pinned to outer_idx position (count=1). Cache starts[] in
|
|
2545
|
+
state so sync_slab can rebuild the same region for WRITE PUT
|
|
2546
|
+
(= captured BEFORE outer_idx advance below, same hazard pattern
|
|
2547
|
+
as F.1a last_data_off). */
|
|
2548
|
+
{
|
|
2549
|
+
int8_t op = 0;
|
|
2550
|
+
for ( int8_t k = 0; k < nd; k++ ) {
|
|
2551
|
+
if ( k == fiber_ax ) {
|
|
2552
|
+
starts[k] = 0;
|
|
2553
|
+
counts[k] = fiber_n;
|
|
2554
|
+
} else {
|
|
2555
|
+
starts[k] = st->outer_idx ? st->outer_idx[op] : 0;
|
|
2556
|
+
counts[k] = 1;
|
|
2557
|
+
op++;
|
|
2558
|
+
}
|
|
2559
|
+
st->fiber_region_starts[k] = starts[k];
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
st->last_data_off = 0; /* not used for PER_FIBER_FUSED */
|
|
2564
|
+
|
|
2565
|
+
ca_xfer_stride(st->src, starts, counts, st->fiber_native_strides,
|
|
2566
|
+
st->fiber_data_scratch, CA_XFER_GET);
|
|
2567
|
+
|
|
2568
|
+
char *yield_ptr = st->fiber_data_scratch;
|
|
2569
|
+
boolean8_t *yield_mask = NULL;
|
|
2570
|
+
|
|
2571
|
+
if ( ca_has_mask(st->src) ) {
|
|
2572
|
+
if ( st->fiber_mask_scratch_cap < fiber_n ) {
|
|
2573
|
+
if ( st->fiber_mask_scratch ) xfree(st->fiber_mask_scratch);
|
|
2574
|
+
st->fiber_mask_scratch = (boolean8_t *) xmalloc(fiber_n > 0 ? fiber_n : 1);
|
|
2575
|
+
st->fiber_mask_scratch_cap = fiber_n;
|
|
2576
|
+
}
|
|
2577
|
+
/* Mask uses element strides (= bytes 1 per cell, identity).
|
|
2578
|
+
Reuse fiber_native_strides scaled down by bytes for mask;
|
|
2579
|
+
actually mask is boolean8_t (1 byte per cell), so native
|
|
2580
|
+
strides over src->mask are simply Π dims (= element index
|
|
2581
|
+
strides). Compute on the fly. */
|
|
2582
|
+
ca_size_t mask_strides[CA_RANK_MAX];
|
|
2583
|
+
{
|
|
2584
|
+
ca_size_t s = 1;
|
|
2585
|
+
for ( int8_t k = nd - 1; k >= 0; k-- ) {
|
|
2586
|
+
mask_strides[k] = s;
|
|
2587
|
+
s *= st->src->dim[k];
|
|
2588
|
+
}
|
|
2589
|
+
}
|
|
2590
|
+
ca_xfer_stride(st->src->mask, starts, counts, mask_strides,
|
|
2591
|
+
(char *) st->fiber_mask_scratch, CA_XFER_GET);
|
|
2592
|
+
yield_mask = st->fiber_mask_scratch;
|
|
2593
|
+
}
|
|
2594
|
+
|
|
2595
|
+
if ( out_ptr ) *out_ptr = yield_ptr;
|
|
2596
|
+
if ( out_mask ) *out_mask = yield_mask;
|
|
2597
|
+
|
|
2598
|
+
/* Advance outer_idx (= same logic as default fall-through). */
|
|
2599
|
+
st->slabs_emitted += 1;
|
|
2600
|
+
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
2601
|
+
if ( ++st->outer_idx[m] < st->outer_dims[m] ) break;
|
|
2602
|
+
st->outer_idx[m] = 0;
|
|
2603
|
+
}
|
|
2604
|
+
return 1;
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
/* PROPOSAL_FIBER_DELIVERY F.1a: per-axis fiber contig delivery.
|
|
2608
|
+
For naxes==1 (= single slab axis = fiber) the catalog contract
|
|
2609
|
+
CA_FOR_EACH_FIBER promises contig data delivery. When the fiber
|
|
2610
|
+
is not innermost-contig (= slab_strides[0] != bytes), the engine
|
|
2611
|
+
gathers the fiber into fiber_data_scratch before yielding so the
|
|
2612
|
+
author can write p[i] without stride math.
|
|
2613
|
+
|
|
2614
|
+
Capture last_data_off BEFORE the outer_idx advance below; sync_slab
|
|
2615
|
+
consumes it for WRITE scatter. See header field doc + PROPOSAL
|
|
2616
|
+
§4.3.2 hazard comment in sync_slab. */
|
|
2617
|
+
char *yield_ptr;
|
|
2618
|
+
if ( (st->flags & CA_KERNEL_FIBER_CONTIG)
|
|
2619
|
+
&& st->naxes == 1 && st->slab_ndim == 1 ) {
|
|
2620
|
+
ca_size_t n = st->slab_dims[0];
|
|
2621
|
+
ca_size_t data_step = st->slab_strides[0];
|
|
2622
|
+
ca_size_t bytes = st->bytes;
|
|
2623
|
+
char *src_data = st->alias_ptr + data_off;
|
|
2624
|
+
|
|
2625
|
+
st->last_data_off = data_off;
|
|
2626
|
+
|
|
2627
|
+
if ( data_step == (ca_size_t) bytes ) {
|
|
2628
|
+
/* Fast path: fiber is already contig (= innermost-axis or stride
|
|
2629
|
+
coincidentally == bytes). No gather needed. */
|
|
2630
|
+
yield_ptr = src_data;
|
|
2631
|
+
} else {
|
|
2632
|
+
/* Per-fiber gather via the typed-store inline helper
|
|
2633
|
+
(ca_iter_substrate.h). For bytes in {1,2,4,8} this uses a
|
|
2634
|
+
compiler-vectorize-friendly `*dp++ = v; sp += step` loop;
|
|
2635
|
+
other sizes fall back to per-element memcpy. Lazy-alloc
|
|
2636
|
+
scratch sized to max fiber bytes (slab_dims[0] constant per
|
|
2637
|
+
walk → single alloc in practice). */
|
|
2638
|
+
ca_size_t need = n * bytes;
|
|
2639
|
+
if ( st->fiber_data_scratch_cap < need ) {
|
|
2640
|
+
if ( st->fiber_data_scratch ) xfree(st->fiber_data_scratch);
|
|
2641
|
+
st->fiber_data_scratch = (char *) xmalloc(need);
|
|
2642
|
+
st->fiber_data_scratch_cap = need;
|
|
2643
|
+
}
|
|
2644
|
+
ca_stride_gather_run(st->fiber_data_scratch, src_data,
|
|
2645
|
+
bytes, n, data_step);
|
|
2646
|
+
yield_ptr = st->fiber_data_scratch;
|
|
2647
|
+
}
|
|
2648
|
+
} else {
|
|
2649
|
+
yield_ptr = st->alias_ptr + data_off;
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
/* PROPOSAL_FIBER_DELIVERY F.1b: per-fiber contig mask delivery.
|
|
2653
|
+
Symmetric to F.1a data path above. When the source has a mask
|
|
2654
|
+
(= alias_mask != NULL) and the fiber's mask is not innermost-contig
|
|
2655
|
+
(= slab_mask_strides[0] != 1), gather it into fiber_mask_scratch
|
|
2656
|
+
so the author can write m[i] without stride math. Mask is
|
|
2657
|
+
read-only here (= L2 WRITE never propagates to mask state), so no
|
|
2658
|
+
scatter is needed in sync_slab. */
|
|
2659
|
+
boolean8_t *yield_mask;
|
|
2660
|
+
if ( (st->flags & CA_KERNEL_FIBER_CONTIG)
|
|
2661
|
+
&& st->naxes == 1 && st->slab_ndim == 1
|
|
2662
|
+
&& st->alias_mask != NULL ) {
|
|
2663
|
+
ca_size_t n = st->slab_dims[0];
|
|
2664
|
+
ca_size_t mask_step = st->slab_mask_strides[0];
|
|
2665
|
+
boolean8_t *src_mask = st->alias_mask + mask_off;
|
|
2666
|
+
|
|
2667
|
+
if ( mask_step == 1 ) {
|
|
2668
|
+
yield_mask = src_mask;
|
|
2669
|
+
} else {
|
|
2670
|
+
if ( st->fiber_mask_scratch_cap < (ca_size_t) n ) {
|
|
2671
|
+
if ( st->fiber_mask_scratch ) xfree(st->fiber_mask_scratch);
|
|
2672
|
+
st->fiber_mask_scratch = (boolean8_t *) xmalloc(n);
|
|
2673
|
+
st->fiber_mask_scratch_cap = n;
|
|
2674
|
+
}
|
|
2675
|
+
for ( ca_size_t i = 0; i < n; i++ ) {
|
|
2676
|
+
st->fiber_mask_scratch[i] = src_mask[i * mask_step];
|
|
2677
|
+
}
|
|
2678
|
+
yield_mask = st->fiber_mask_scratch;
|
|
2679
|
+
}
|
|
2680
|
+
} else {
|
|
2681
|
+
yield_mask = st->alias_mask ? st->alias_mask + mask_off : NULL;
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2684
|
+
if ( out_ptr ) *out_ptr = yield_ptr;
|
|
2685
|
+
if ( out_mask ) *out_mask = yield_mask;
|
|
2686
|
+
|
|
2687
|
+
st->slabs_emitted += 1;
|
|
2688
|
+
|
|
2689
|
+
/* Advance outer_idx row-major (innermost outer axis ticks first).
|
|
2690
|
+
No-op when outer_ndim == 0 (= single slab walk). */
|
|
2691
|
+
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
2692
|
+
if ( ++st->outer_idx[m] < st->outer_dims[m] ) break;
|
|
2693
|
+
st->outer_idx[m] = 0;
|
|
2694
|
+
}
|
|
2695
|
+
return 1;
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
void
|
|
2699
|
+
ca_iter_state_sync_slab (ca_iter_state *st)
|
|
2700
|
+
{
|
|
2701
|
+
/* READ walk: nothing to sync. */
|
|
2702
|
+
if ( st == NULL || !(st->flags & CA_KERNEL_WRITE) ) return;
|
|
2703
|
+
|
|
2704
|
+
/* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2: STACK path is
|
|
2705
|
+
READ-only in initial scope. Kernel writes into scratch would not
|
|
2706
|
+
be valid to scatter back via xfer_all PUT (= scratch is slab-sized
|
|
2707
|
+
not whole-view), so the SRC_ATTACH PUT below would be a semantic
|
|
2708
|
+
mismatch. Per-slab scatter via direct per-parent memcpy is a
|
|
2709
|
+
future extension once a WRITE-using kernel materialises. */
|
|
2710
|
+
if ( st->alias_mode == CA_ITER_ALIAS_STACK
|
|
2711
|
+
|| st->alias_mode == CA_ITER_ALIAS_STACK_OUTER_K ) {
|
|
2712
|
+
/* STACK_OUTER_K (P.2 Case A) is READ-only scope: slabs are direct
|
|
2713
|
+
aliases into parents[k]->ptr, kernel WRITE would scatter into
|
|
2714
|
+
parent memory which is out of scope (see proposal R1). Skip
|
|
2715
|
+
sync. */
|
|
2716
|
+
st->write_dirty = 0;
|
|
2717
|
+
return;
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
/* SRC_ATTACH path (step 9 + 2026-05-31 refactor): kernel wrote into
|
|
2721
|
+
iterator-owned scratch (= scratch_ptr). Push back via xfer_all PUT
|
|
2722
|
+
which routes through the view's xfer_all slot -- handles CAFake
|
|
2723
|
+
(cast back), CAByteSwap (swap back), CABitfield/CABitarray (bit
|
|
2724
|
+
pack back), CAReduce (broadcast across reduce window). Inherits
|
|
2725
|
+
transform-fused / partial materialise / etc. automatically. */
|
|
2726
|
+
if ( st->src_kind == CA_ITER_SRC_ATTACH ) {
|
|
2727
|
+
if ( st->src->elements > 0 ) {
|
|
2728
|
+
ca_xfer_all(st->src, st->scratch_ptr, CA_XFER_PUT);
|
|
2729
|
+
}
|
|
2730
|
+
st->write_dirty = 0;
|
|
2731
|
+
return;
|
|
2732
|
+
}
|
|
2733
|
+
|
|
2734
|
+
/* ======================================================================
|
|
2735
|
+
* !!! CORRECTNESS HAZARD - DO NOT MOVE !!!
|
|
2736
|
+
*
|
|
2737
|
+
* (PROPOSAL_FIBER_DELIVERY F.1a)
|
|
2738
|
+
*
|
|
2739
|
+
* Per-fiber scratch reuse + scatter correctness depends on the strict
|
|
2740
|
+
* evaluation order of the CA_FOR_EACH_FIBER_* macro sandwich:
|
|
2741
|
+
*
|
|
2742
|
+
* for ( init ; next_slab_axes(k) ; sync_slab(k) ) { body(k) }
|
|
2743
|
+
*
|
|
2744
|
+
* Concretely: sync_slab(fiber k) MUST run BEFORE next_slab_axes(k+1).
|
|
2745
|
+
* The invariant at sync_slab(k) time:
|
|
2746
|
+
*
|
|
2747
|
+
* - st->fiber_data_scratch holds author-written data for fiber k
|
|
2748
|
+
* (= body(k) just modified it, no other call has touched it since)
|
|
2749
|
+
* - st->last_data_off holds the source byte offset for fiber k
|
|
2750
|
+
* (= captured by next_slab_axes(k) before outer_idx advance)
|
|
2751
|
+
*
|
|
2752
|
+
* next_slab_axes(k+1) will overwrite BOTH (= refill scratch + advance
|
|
2753
|
+
* last_data_off) BEFORE body(k+1) starts. Per-fiber scratch reuse
|
|
2754
|
+
* (= only one buffer for all fibers) is correct ONLY because this
|
|
2755
|
+
* sequence holds.
|
|
2756
|
+
*
|
|
2757
|
+
* DO NOT introduce: prefetch of next_slab_axes(k+1), async sync_slab,
|
|
2758
|
+
* sandwich reordering, batched sync, or any pattern that breaks the
|
|
2759
|
+
* (next -> body -> sync -> next -> body -> sync ...) sequence. Per-
|
|
2760
|
+
* fiber scratch reuse becomes UB the moment this invariant is violated.
|
|
2761
|
+
* If lookahead / batching is needed, allocate one scratch per fiber
|
|
2762
|
+
* instead of reusing -- separate phase, separate design.
|
|
2763
|
+
* ====================================================================== */
|
|
2764
|
+
|
|
2765
|
+
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1: per-fiber fused WRITE PUT.
|
|
2766
|
+
When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, the author
|
|
2767
|
+
wrote into fiber_data_scratch and there is no whole-view buffer
|
|
2768
|
+
to xfer_all PUT. Rebuild the same fiber region from cached
|
|
2769
|
+
fiber_region_starts[] (= captured pre-advance in next_slab_axes,
|
|
2770
|
+
same hazard pattern as F.1a last_data_off) and call
|
|
2771
|
+
ca_xfer_stride(src, ..., PUT) which routes through the view's
|
|
2772
|
+
fused PUT path (X.1 / X.4) for 1-pass scatter back. */
|
|
2773
|
+
if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
|
|
2774
|
+
ca_size_t fiber_n = st->slab_dims[0];
|
|
2775
|
+
int8_t nd = st->ndim;
|
|
2776
|
+
ca_size_t counts[CA_RANK_MAX];
|
|
2777
|
+
for ( int8_t k = 0; k < nd; k++ ) {
|
|
2778
|
+
counts[k] = (k == st->fiber_axis) ? fiber_n : 1;
|
|
2779
|
+
}
|
|
2780
|
+
ca_xfer_stride(st->src, st->fiber_region_starts, counts,
|
|
2781
|
+
st->fiber_native_strides,
|
|
2782
|
+
st->fiber_data_scratch, CA_XFER_PUT);
|
|
2783
|
+
st->write_dirty = 0;
|
|
2784
|
+
return;
|
|
2785
|
+
}
|
|
2786
|
+
|
|
2787
|
+
/* Per-fiber gather path scatter (PROPOSAL_FIBER_DELIVERY F.1a).
|
|
2788
|
+
When next_slab_axes gathered the fiber into fiber_data_scratch
|
|
2789
|
+
(= naxes==1 + slab_strides[0] != bytes), scatter it back to the
|
|
2790
|
+
source layout via the strided write. When the fiber was the
|
|
2791
|
+
contig fast path (= data_step == bytes), the author wrote directly
|
|
2792
|
+
into the source via alias_ptr; no scatter needed. */
|
|
2793
|
+
if ( (st->flags & CA_KERNEL_FIBER_CONTIG)
|
|
2794
|
+
&& st->naxes == 1 && st->slab_ndim == 1
|
|
2795
|
+
&& st->fiber_data_scratch != NULL ) {
|
|
2796
|
+
ca_size_t n = st->slab_dims[0];
|
|
2797
|
+
ca_size_t data_step = st->slab_strides[0];
|
|
2798
|
+
ca_size_t bytes = st->bytes;
|
|
2799
|
+
if ( data_step != (ca_size_t) bytes ) {
|
|
2800
|
+
char *dst = st->alias_ptr + st->last_data_off;
|
|
2801
|
+
/* ^ captured by next_slab_axes(k) BEFORE
|
|
2802
|
+
* outer_idx advance; see hazard above. */
|
|
2803
|
+
/* Typed scatter helper (ca_iter_substrate.h): same SIMD-friendly
|
|
2804
|
+
loop structure as ca_stride_gather_run, in reverse direction. */
|
|
2805
|
+
ca_stride_scatter_run(dst, st->fiber_data_scratch,
|
|
2806
|
+
bytes, n, data_step);
|
|
2807
|
+
}
|
|
2808
|
+
/* Fall through to any subsequent src_kind scatter (= harmless: for
|
|
2809
|
+
the alias paths reached here, scratch_ptr is NULL and the switch
|
|
2810
|
+
below early-returns). But for SRC_DESCRIPTOR / SRC_ATTACH that
|
|
2811
|
+
use the per-slab materialise path, fiber_data_scratch stays NULL
|
|
2812
|
+
(those paths use scratch_ptr and the PER_SLAB(_HOIST) yield), so
|
|
2813
|
+
this block does not fire. */
|
|
2814
|
+
}
|
|
2815
|
+
|
|
2816
|
+
/* alias path: kernel wrote through alias_ptr into parent directly
|
|
2817
|
+
(case A semantics, PROPOSAL_T1_WRITE_SEMANTICS.md §(a)). No
|
|
2818
|
+
scatter needed. Applies to all CAStride alias and to
|
|
2819
|
+
STRIDE-only descriptor alias (= descriptor L2 alias future
|
|
2820
|
+
optimisation, not yet enabled — but if it lands, scratch_ptr
|
|
2821
|
+
stays NULL and we no-op correctly). */
|
|
2822
|
+
if ( st->scratch_ptr == NULL ) return;
|
|
2823
|
+
|
|
2824
|
+
/* scratch path: scatter back the materialised buffer into the
|
|
2825
|
+
source view. src_kind chooses the engine: */
|
|
2826
|
+
switch ( st->src_kind ) {
|
|
2827
|
+
case CA_ITER_SRC_CASTRIDE:
|
|
2828
|
+
/* Two producers of CAStride + scratch: L1 non-contig, and the L2
|
|
2829
|
+
non-entity-root path (init_l2's use_view_scratch). Both gathered
|
|
2830
|
+
with ca_copy_data into a view row-major buffer, so both scatter
|
|
2831
|
+
back the same way — ca_sync_data routes through the view's
|
|
2832
|
+
xfer_all(PUT), which asks the root for the region it owns rather
|
|
2833
|
+
than writing a whole-root materialise back. */
|
|
2834
|
+
ca_sync_data(st->src, st->scratch_ptr);
|
|
2835
|
+
break;
|
|
2836
|
+
|
|
2837
|
+
case CA_ITER_SRC_DESCRIPTOR: {
|
|
2838
|
+
/* Sub-step 5.4: descriptor framework scatter back.
|
|
2839
|
+
ca_axis_dispatch_scatter is the P3-landed engine entry that
|
|
2840
|
+
handles per-axis kind (STRIDE / INDEX / SHIFT) gather direction
|
|
2841
|
+
in reverse — INDEX duplicates yield last-write-wins (R5 spec),
|
|
2842
|
+
SHIFT OOB cells in CAWindow FILL policy are skipped (no parent
|
|
2843
|
+
destination), CAShift WRAP/REFLECT bounds map back to interior
|
|
2844
|
+
and are written normally. Iteration order is engine-defined;
|
|
2845
|
+
user kernels must not rely on it.
|
|
2846
|
+
|
|
2847
|
+
scatter wants the pre-merge raw descriptors (the engine re-runs
|
|
2848
|
+
_prepare internally with whatever we give it). init cached
|
|
2849
|
+
the post-merge axes in st->descs for next_slab_strided's
|
|
2850
|
+
offset math; we re-call describe_axes here for the scatter
|
|
2851
|
+
call rather than caching a second copy in the state struct. */
|
|
2852
|
+
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
2853
|
+
ca_size_t raw_pdims[CA_RANK_MAX];
|
|
2854
|
+
int8_t raw_ndim = 0;
|
|
2855
|
+
ca_iter_describe_axes(st->src, raw_descs, raw_pdims, &raw_ndim);
|
|
2856
|
+
ca_axis_dispatch_scatter(st->root /* parent */,
|
|
2857
|
+
raw_pdims,
|
|
2858
|
+
raw_descs, raw_ndim,
|
|
2859
|
+
st->bytes, st->total_elements,
|
|
2860
|
+
st->scratch_ptr);
|
|
2861
|
+
break;
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
st->write_dirty = 0;
|
|
2865
|
+
}
|
|
2866
|
+
|
|
2867
|
+
void
|
|
2868
|
+
ca_iter_state_finish (ca_iter_state *st)
|
|
2869
|
+
{
|
|
2870
|
+
if ( st == NULL || st->src == NULL ) {
|
|
2871
|
+
return;
|
|
2872
|
+
}
|
|
2873
|
+
/* composed_strides is inline — no free needed. outer_idx is heap
|
|
2874
|
+
for L2 multi-d sources (NULL on L1 paths and on L2 0/1-d). */
|
|
2875
|
+
if ( st->outer_idx ) {
|
|
2876
|
+
xfree(st->outer_idx);
|
|
2877
|
+
st->outer_idx = NULL;
|
|
2878
|
+
}
|
|
2879
|
+
/* Lifecycle cleanup — orders matter slightly (free scratch before
|
|
2880
|
+
detaching parent so the kernel iterator's resources are released
|
|
2881
|
+
symmetrically with init):
|
|
2882
|
+
- scratch_ptr: owned by iter (CAStride non-contig L1, or
|
|
2883
|
+
descriptor materialise via ca_axis_dispatch_attach). xfree.
|
|
2884
|
+
- root: descriptor parent attached at init (sub-step 5.1+) or
|
|
2885
|
+
CAStride L2 compose-fold root attached at init (step 3).
|
|
2886
|
+
ca_detach.
|
|
2887
|
+
- else (alias paths): src was attached at init, detach. */
|
|
2888
|
+
if ( st->scratch_ptr ) {
|
|
2889
|
+
xfree(st->scratch_ptr);
|
|
2890
|
+
st->scratch_ptr = NULL;
|
|
2891
|
+
st->scratch_cap = 0;
|
|
2892
|
+
}
|
|
2893
|
+
if ( st->scratch_mask ) {
|
|
2894
|
+
xfree(st->scratch_mask);
|
|
2895
|
+
st->scratch_mask = NULL;
|
|
2896
|
+
}
|
|
2897
|
+
/* PROPOSAL_FIBER_DELIVERY F.1a/F.1b: per-fiber scratch lifecycle. */
|
|
2898
|
+
if ( st->fiber_data_scratch ) {
|
|
2899
|
+
xfree(st->fiber_data_scratch);
|
|
2900
|
+
st->fiber_data_scratch = NULL;
|
|
2901
|
+
st->fiber_data_scratch_cap = 0;
|
|
2902
|
+
}
|
|
2903
|
+
if ( st->fiber_mask_scratch ) {
|
|
2904
|
+
xfree(st->fiber_mask_scratch);
|
|
2905
|
+
st->fiber_mask_scratch = NULL;
|
|
2906
|
+
st->fiber_mask_scratch_cap = 0;
|
|
2907
|
+
}
|
|
2908
|
+
/* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev2 + P.2 Case A: detach
|
|
2909
|
+
K parents (+ K parent masks if cached) and free the cached ptr
|
|
2910
|
+
arrays. Symmetric with init_l2 per-parent ca_attach loop. */
|
|
2911
|
+
if ( st->stack_parent_ptrs ) {
|
|
2912
|
+
CAStack *stack = (CAStack *) st->src;
|
|
2913
|
+
for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
|
|
2914
|
+
ca_detach(stack->parents[kk]);
|
|
2915
|
+
}
|
|
2916
|
+
xfree(st->stack_parent_ptrs);
|
|
2917
|
+
st->stack_parent_ptrs = NULL;
|
|
2918
|
+
}
|
|
2919
|
+
/* pilot/castack-axis0-loop-interchange: free tile cache. */
|
|
2920
|
+
if ( st->stack_tile_cache ) {
|
|
2921
|
+
xfree(st->stack_tile_cache);
|
|
2922
|
+
st->stack_tile_cache = NULL;
|
|
2923
|
+
st->stack_tile_cap = 0;
|
|
2924
|
+
st->stack_tile_pos = 0;
|
|
2925
|
+
st->stack_tile_have = 0;
|
|
2926
|
+
}
|
|
2927
|
+
if ( st->stack_parent_mask_ptrs ) {
|
|
2928
|
+
CAStack *stack = (CAStack *) st->src;
|
|
2929
|
+
for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
|
|
2930
|
+
ca_detach(stack->parents[kk]->mask);
|
|
2931
|
+
}
|
|
2932
|
+
xfree(st->stack_parent_mask_ptrs);
|
|
2933
|
+
st->stack_parent_mask_ptrs = NULL;
|
|
2934
|
+
}
|
|
2935
|
+
st->stack_n_parents = 0;
|
|
2936
|
+
if ( st->root ) {
|
|
2937
|
+
ca_detach(st->root);
|
|
2938
|
+
st->root = NULL;
|
|
2939
|
+
} else if ( st->alias_mode == CA_ITER_ALIAS_CONTIG
|
|
2940
|
+
|| st->alias_mode == CA_ITER_ALIAS_STRIDED ) {
|
|
2941
|
+
/* alias path (CONTIG/STRIDED for CAStride family): we attached
|
|
2942
|
+
src directly at init, so detach it here.
|
|
2943
|
+
Note: SRC_ATTACH used to be in this list (ALIAS_ATTACH); after
|
|
2944
|
+
the 2026-05-31 refactor it owns its own scratch (alias_mode =
|
|
2945
|
+
NONE, scratch_ptr xfree'd above), so no src.detach needed. */
|
|
2946
|
+
ca_detach(st->src);
|
|
2947
|
+
}
|
|
2948
|
+
st->src = NULL;
|
|
2949
|
+
st->alias_ptr = NULL;
|
|
2950
|
+
}
|
|
2951
|
+
|
|
2952
|
+
#ifdef CARRAY_DEV_BUILD
|
|
2953
|
+
/* ============================================================
|
|
2954
|
+
* smoke surface (dev-only, stripped in release)
|
|
2955
|
+
*
|
|
2956
|
+
* Gated by CARRAY_DEV_BUILD (enabled via `extconf.rb --enable-dev-build`
|
|
2957
|
+
* or `CARRAY_DEV=1 rake build_ext`). These helpers expose internal
|
|
2958
|
+
* engine state to Ruby for spec_ai regression pins. Do not consume
|
|
2959
|
+
* from user code.
|
|
2960
|
+
*
|
|
2961
|
+
* See devel/PROPOSAL_SMOKE_DEV_BUILD_GATE.md
|
|
2962
|
+
* ============================================================ */
|
|
2963
|
+
|
|
2964
|
+
/* ---- Ruby smoke surface --------------------------------------------- */
|
|
2965
|
+
|
|
2966
|
+
/* L1 smoke (steps 1+2): CArray.t1_smoke(ca) -> Hash with
|
|
2967
|
+
rc / slabs / total_elems / ptr_nonnull / alias_mode / data. */
|
|
2968
|
+
static VALUE
|
|
2969
|
+
rb_t1_smoke (VALUE klass, VALUE vsrc)
|
|
2970
|
+
{
|
|
2971
|
+
CArray *src;
|
|
2972
|
+
ca_iter_state st;
|
|
2973
|
+
char *p;
|
|
2974
|
+
ca_size_t n;
|
|
2975
|
+
int rc;
|
|
2976
|
+
int slabs = 0;
|
|
2977
|
+
ca_size_t total_elems = 0;
|
|
2978
|
+
int ptr_nonnull = 0;
|
|
2979
|
+
int alias_mode = CA_ITER_ALIAS_NONE;
|
|
2980
|
+
VALUE result;
|
|
2981
|
+
VALUE data;
|
|
2982
|
+
|
|
2983
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
2984
|
+
data = rb_str_new(0, 0);
|
|
2985
|
+
|
|
2986
|
+
rc = ca_iter_state_init_l1(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
2987
|
+
if ( rc == CA_ITER_OK ) {
|
|
2988
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
2989
|
+
if ( slabs == 0 && p != NULL ) ptr_nonnull = 1;
|
|
2990
|
+
if ( p != NULL && n > 0 ) rb_str_cat(data, p, n * st.bytes);
|
|
2991
|
+
total_elems += n;
|
|
2992
|
+
slabs++;
|
|
2993
|
+
}
|
|
2994
|
+
alias_mode = st.alias_mode;
|
|
2995
|
+
ca_iter_state_finish(&st);
|
|
2996
|
+
}
|
|
2997
|
+
|
|
2998
|
+
result = rb_hash_new();
|
|
2999
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3000
|
+
rb_hash_aset(result, ID2SYM(rb_intern("slabs")), INT2NUM(slabs));
|
|
3001
|
+
rb_hash_aset(result, ID2SYM(rb_intern("total_elems")), SIZE2NUM(total_elems));
|
|
3002
|
+
rb_hash_aset(result, ID2SYM(rb_intern("ptr_nonnull")), ptr_nonnull ? Qtrue : Qfalse);
|
|
3003
|
+
rb_hash_aset(result, ID2SYM(rb_intern("alias_mode")), INT2NUM(alias_mode));
|
|
3004
|
+
rb_hash_aset(result, ID2SYM(rb_intern("data")), data);
|
|
3005
|
+
return result;
|
|
3006
|
+
}
|
|
3007
|
+
|
|
3008
|
+
/* L2 smoke (step 3): CArray.t1_smoke_strided(ca) -> Hash with
|
|
3009
|
+
rc => Integer
|
|
3010
|
+
slabs => Integer (= total_slabs)
|
|
3011
|
+
total_elems => Integer
|
|
3012
|
+
alias_mode => Integer
|
|
3013
|
+
data => String — slab bytes reconstructed via the
|
|
3014
|
+
reported (ptr, n, stride_bytes) tuples, in
|
|
3015
|
+
iteration order; should equal view.to_ca.dump_binary
|
|
3016
|
+
strides => Array<Integer> — stride_bytes per yielded slab
|
|
3017
|
+
(constant across yields under WHOLE policy in
|
|
3018
|
+
step 3; surfaced for inspection)
|
|
3019
|
+
On error rc != OK, the walk fields are 0 / empty. */
|
|
3020
|
+
static VALUE
|
|
3021
|
+
rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
|
|
3022
|
+
{
|
|
3023
|
+
CArray *src;
|
|
3024
|
+
ca_iter_state st;
|
|
3025
|
+
char *p;
|
|
3026
|
+
ca_size_t n, stride_bytes;
|
|
3027
|
+
int rc;
|
|
3028
|
+
int slabs = 0;
|
|
3029
|
+
ca_size_t total_elems = 0;
|
|
3030
|
+
int alias_mode = CA_ITER_ALIAS_NONE;
|
|
3031
|
+
VALUE result, data, strides_arr;
|
|
3032
|
+
|
|
3033
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3034
|
+
data = rb_str_new(0, 0);
|
|
3035
|
+
strides_arr = rb_ary_new();
|
|
3036
|
+
|
|
3037
|
+
rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3038
|
+
if ( rc == CA_ITER_OK ) {
|
|
3039
|
+
while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
|
|
3040
|
+
if ( p != NULL && n > 0 ) {
|
|
3041
|
+
ca_size_t i;
|
|
3042
|
+
for ( i = 0; i < n; i++ ) {
|
|
3043
|
+
rb_str_cat(data, p + i * stride_bytes, st.bytes);
|
|
3044
|
+
}
|
|
3045
|
+
}
|
|
3046
|
+
total_elems += n;
|
|
3047
|
+
slabs++;
|
|
3048
|
+
rb_ary_push(strides_arr, SIZE2NUM(stride_bytes));
|
|
3049
|
+
}
|
|
3050
|
+
alias_mode = st.alias_mode;
|
|
3051
|
+
ca_iter_state_finish(&st);
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
result = rb_hash_new();
|
|
3055
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3056
|
+
rb_hash_aset(result, ID2SYM(rb_intern("slabs")), INT2NUM(slabs));
|
|
3057
|
+
rb_hash_aset(result, ID2SYM(rb_intern("total_elems")), SIZE2NUM(total_elems));
|
|
3058
|
+
rb_hash_aset(result, ID2SYM(rb_intern("alias_mode")), INT2NUM(alias_mode));
|
|
3059
|
+
rb_hash_aset(result, ID2SYM(rb_intern("data")), data);
|
|
3060
|
+
rb_hash_aset(result, ID2SYM(rb_intern("strides")), strides_arr);
|
|
3061
|
+
return result;
|
|
3062
|
+
}
|
|
3063
|
+
|
|
3064
|
+
/* Bench-grade L2 sum kernel: total reduction via L2 iteration with no
|
|
3065
|
+
Ruby String materialisation in the hot loop. Use this rather than
|
|
3066
|
+
t1_smoke_strided when measuring the actual L2 dispatch overhead
|
|
3067
|
+
(the smoke variant's rb_str_cat dominates timing for moderate
|
|
3068
|
+
slab counts). Only supports float64 sources for now — the smoke
|
|
3069
|
+
API isn't a public surface and this is bench scaffolding. */
|
|
3070
|
+
static VALUE
|
|
3071
|
+
rb_t1_smoke_sum_strided_f64 (VALUE klass, VALUE vsrc)
|
|
3072
|
+
{
|
|
3073
|
+
CArray *src;
|
|
3074
|
+
ca_iter_state st;
|
|
3075
|
+
char *p;
|
|
3076
|
+
ca_size_t n, stride_bytes, i;
|
|
3077
|
+
int rc;
|
|
3078
|
+
double acc = 0.0;
|
|
3079
|
+
|
|
3080
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3081
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3082
|
+
rb_raise(rb_eTypeError,
|
|
3083
|
+
"t1_smoke_sum_strided_f64 expects a float64 source");
|
|
3084
|
+
}
|
|
3085
|
+
|
|
3086
|
+
rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3087
|
+
if ( rc != CA_ITER_OK ) {
|
|
3088
|
+
rb_raise(rb_eRuntimeError, "ca_iter_state_init L2 failed (rc=%d)", rc);
|
|
3089
|
+
}
|
|
3090
|
+
|
|
3091
|
+
while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
|
|
3092
|
+
/* Step 8.1: macro picks contig fast path when stride == sizeof(double),
|
|
3093
|
+
falls back to strided loop otherwise. Removes the SIMD inhibition
|
|
3094
|
+
observed in step 5.5 §10.4.5 on descriptor materialise paths. */
|
|
3095
|
+
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3096
|
+
acc += *dp;
|
|
3097
|
+
});
|
|
3098
|
+
}
|
|
3099
|
+
(void) i;
|
|
3100
|
+
ca_iter_state_finish(&st);
|
|
3101
|
+
return DBL2NUM(acc);
|
|
3102
|
+
}
|
|
3103
|
+
|
|
3104
|
+
/* WRITE smoke: in-place fill via L1. Fills every element with `val`
|
|
3105
|
+
using next_slab (alias direct write if cheap, scratch+sync if not).
|
|
3106
|
+
Returns iter rc; on rc != OK src is not modified. */
|
|
3107
|
+
static VALUE
|
|
3108
|
+
rb_t1_smoke_write_fill_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
3109
|
+
{
|
|
3110
|
+
CArray *src;
|
|
3111
|
+
ca_iter_state st;
|
|
3112
|
+
char *p;
|
|
3113
|
+
ca_size_t n, i;
|
|
3114
|
+
double v = NUM2DBL(vval);
|
|
3115
|
+
int rc;
|
|
3116
|
+
|
|
3117
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3118
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3119
|
+
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3120
|
+
}
|
|
3121
|
+
|
|
3122
|
+
rc = ca_iter_state_init_l1(&st, src, CA_SLAB_WHOLE, NULL, 0,
|
|
3123
|
+
CA_KERNEL_WRITE);
|
|
3124
|
+
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3125
|
+
|
|
3126
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3127
|
+
double *d = (double *) p;
|
|
3128
|
+
for ( i = 0; i < n; i++ ) d[i] = v;
|
|
3129
|
+
ca_iter_state_sync_slab(&st);
|
|
3130
|
+
}
|
|
3131
|
+
ca_iter_state_finish(&st);
|
|
3132
|
+
return INT2NUM(CA_ITER_OK);
|
|
3133
|
+
}
|
|
3134
|
+
|
|
3135
|
+
/* WRITE smoke: in-place fill via L2 strided dispatch. Used by the
|
|
3136
|
+
step 5.5 aggregate bench to round out the matrix (L1 WRITE was
|
|
3137
|
+
already in t1_smoke_write_fill_f64). Kernel walks the strided
|
|
3138
|
+
slab and writes val at every position; sync_slab scatters back. */
|
|
3139
|
+
static VALUE
|
|
3140
|
+
rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
3141
|
+
{
|
|
3142
|
+
CArray *src;
|
|
3143
|
+
ca_iter_state st;
|
|
3144
|
+
char *p;
|
|
3145
|
+
ca_size_t n, stride, i;
|
|
3146
|
+
double v = NUM2DBL(vval);
|
|
3147
|
+
int rc;
|
|
3148
|
+
|
|
3149
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3150
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3151
|
+
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3152
|
+
}
|
|
3153
|
+
|
|
3154
|
+
rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0,
|
|
3155
|
+
CA_KERNEL_WRITE);
|
|
3156
|
+
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3157
|
+
|
|
3158
|
+
while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
|
|
3159
|
+
/* Step 8.1: contig fast path via macro (stride == sizeof(double)
|
|
3160
|
+
on materialise scratch, which is the descriptor L2 path that
|
|
3161
|
+
hit +18-22% in step 5.5 — macro removes that overhead). */
|
|
3162
|
+
CA_L2_FOR_EACH(double, p, n, stride, dp, {
|
|
3163
|
+
*dp = v;
|
|
3164
|
+
});
|
|
3165
|
+
ca_iter_state_sync_slab(&st);
|
|
3166
|
+
}
|
|
3167
|
+
(void) i;
|
|
3168
|
+
ca_iter_state_finish(&st);
|
|
3169
|
+
return INT2NUM(CA_ITER_OK);
|
|
3170
|
+
}
|
|
3171
|
+
|
|
3172
|
+
/* WRITE smoke: partial write then ruby raise — used by exception
|
|
3173
|
+
safety tests. Writes the first `raise_at` elements, then raises.
|
|
3174
|
+
Parent is left in a partially-written state (alias path) or
|
|
3175
|
+
unchanged (scratch path, pre-sync). */
|
|
3176
|
+
static VALUE
|
|
3177
|
+
rb_t1_smoke_write_partial_raise_f64 (VALUE klass, VALUE vsrc,
|
|
3178
|
+
VALUE vval, VALUE vraise_at)
|
|
3179
|
+
{
|
|
3180
|
+
CArray *src;
|
|
3181
|
+
ca_iter_state st;
|
|
3182
|
+
char *p;
|
|
3183
|
+
ca_size_t n, i;
|
|
3184
|
+
double v = NUM2DBL(vval);
|
|
3185
|
+
ca_size_t raise_at = NUM2SIZET(vraise_at);
|
|
3186
|
+
int rc;
|
|
3187
|
+
|
|
3188
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3189
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3190
|
+
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3191
|
+
}
|
|
3192
|
+
|
|
3193
|
+
rc = ca_iter_state_init_l1(&st, src, CA_SLAB_WHOLE, NULL, 0,
|
|
3194
|
+
CA_KERNEL_WRITE);
|
|
3195
|
+
if ( rc != CA_ITER_OK ) {
|
|
3196
|
+
rb_raise(rb_eRuntimeError, "init_l1 failed (rc=%d)", rc);
|
|
3197
|
+
}
|
|
3198
|
+
|
|
3199
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3200
|
+
double *d = (double *) p;
|
|
3201
|
+
for ( i = 0; i < n; i++ ) {
|
|
3202
|
+
if ( i == raise_at ) {
|
|
3203
|
+
/* Raise without finishing the walk. alias path: writes 0..raise_at-1
|
|
3204
|
+
are now visible to parent. scratch path: scratch has the partial
|
|
3205
|
+
writes but sync_slab was not called, so parent is unchanged. */
|
|
3206
|
+
ca_iter_state_finish(&st); /* release lifecycle */
|
|
3207
|
+
rb_raise(rb_eRuntimeError, "kernel raise at %ld", (long) raise_at);
|
|
3208
|
+
}
|
|
3209
|
+
d[i] = v;
|
|
3210
|
+
}
|
|
3211
|
+
ca_iter_state_sync_slab(&st);
|
|
3212
|
+
}
|
|
3213
|
+
ca_iter_state_finish(&st);
|
|
3214
|
+
return INT2NUM(CA_ITER_OK);
|
|
3215
|
+
}
|
|
3216
|
+
|
|
3217
|
+
/* qsort comparator for double */
|
|
3218
|
+
static int
|
|
3219
|
+
cmp_double (const void *a, const void *b)
|
|
3220
|
+
{
|
|
3221
|
+
double da = *(const double *)a, db = *(const double *)b;
|
|
3222
|
+
if ( da < db ) return -1;
|
|
3223
|
+
if ( da > db ) return 1;
|
|
3224
|
+
return 0;
|
|
3225
|
+
}
|
|
3226
|
+
|
|
3227
|
+
/* WRITE smoke: per-row sort via L2 strided dispatch. For a 2D
|
|
3228
|
+
float64 src of shape [m, n], sort each of the m rows in ascending
|
|
3229
|
+
order. Uses next_slab_strided so each row is a strided slab; the
|
|
3230
|
+
kernel materialises into a tight contig scratch, qsorts, then
|
|
3231
|
+
writes back via the same stride. This exercises L2 WRITE
|
|
3232
|
+
mechanics (multi-slab walk, per-slab fill of strided cells)
|
|
3233
|
+
without over-engineering a strided qsort itself — that would be a
|
|
3234
|
+
Pattern H specialised op, out of step 4 scope (reviewer advice #3).
|
|
3235
|
+
*/
|
|
3236
|
+
static VALUE
|
|
3237
|
+
rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
3238
|
+
{
|
|
3239
|
+
CArray *src;
|
|
3240
|
+
ca_iter_state st;
|
|
3241
|
+
char *p;
|
|
3242
|
+
ca_size_t n, stride_bytes;
|
|
3243
|
+
int rc;
|
|
3244
|
+
double scratch[CA_DIM_MAX > 0 ? 4096 : 4096]; /* row-cap, see below */
|
|
3245
|
+
|
|
3246
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3247
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3248
|
+
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3251
|
+
rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0,
|
|
3252
|
+
CA_KERNEL_WRITE);
|
|
3253
|
+
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3254
|
+
|
|
3255
|
+
while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride_bytes) ) {
|
|
3256
|
+
if ( n > (ca_size_t) (sizeof(scratch) / sizeof(double)) ) {
|
|
3257
|
+
ca_iter_state_finish(&st);
|
|
3258
|
+
rb_raise(rb_eRuntimeError, "row too large for smoke scratch");
|
|
3259
|
+
}
|
|
3260
|
+
/* strided -> contig scratch. Step 8.1: macro handles the
|
|
3261
|
+
stride == bytes contig fast path automatically. */
|
|
3262
|
+
ca_size_t _sc_i = 0;
|
|
3263
|
+
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3264
|
+
scratch[_sc_i++] = *dp;
|
|
3265
|
+
});
|
|
3266
|
+
/* in-place qsort on the contig scratch */
|
|
3267
|
+
qsort(scratch, n, sizeof(double), cmp_double);
|
|
3268
|
+
/* contig scratch -> strided (writes back to parent via alias_ptr,
|
|
3269
|
+
case A direct write). Step 8.1: macro contig fast path. */
|
|
3270
|
+
ca_size_t _wb_i = 0;
|
|
3271
|
+
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3272
|
+
*dp = scratch[_wb_i++];
|
|
3273
|
+
});
|
|
3274
|
+
ca_iter_state_sync_slab(&st); /* no-op for L2 alias, by invariant */
|
|
3275
|
+
}
|
|
3276
|
+
ca_iter_state_finish(&st);
|
|
3277
|
+
return INT2NUM(CA_ITER_OK);
|
|
3278
|
+
}
|
|
3279
|
+
|
|
3280
|
+
/* Step 6 smoke: L1 walk that exposes BOTH the value slab and the
|
|
3281
|
+
mask slab. Used to pin "mask is delivered to kernel" semantics.
|
|
3282
|
+
Returns rc + mask_seen (Boolean: was out_mask non-NULL at first
|
|
3283
|
+
yield) + mask_bytes (String: concatenated mask bytes). */
|
|
3284
|
+
static VALUE
|
|
3285
|
+
rb_t1_smoke_with_mask (VALUE klass, VALUE vsrc)
|
|
3286
|
+
{
|
|
3287
|
+
CArray *src;
|
|
3288
|
+
ca_iter_state st;
|
|
3289
|
+
char *p;
|
|
3290
|
+
boolean8_t *m;
|
|
3291
|
+
ca_size_t n;
|
|
3292
|
+
int rc;
|
|
3293
|
+
int mask_seen = 0;
|
|
3294
|
+
VALUE mask_bytes, result;
|
|
3295
|
+
|
|
3296
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3297
|
+
mask_bytes = rb_str_new(0, 0);
|
|
3298
|
+
|
|
3299
|
+
rc = ca_iter_state_init_l1(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3300
|
+
if ( rc != CA_ITER_OK ) {
|
|
3301
|
+
result = rb_hash_new();
|
|
3302
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3303
|
+
rb_hash_aset(result, ID2SYM(rb_intern("mask_seen")), Qfalse);
|
|
3304
|
+
rb_hash_aset(result, ID2SYM(rb_intern("mask_bytes")), mask_bytes);
|
|
3305
|
+
return result;
|
|
3306
|
+
}
|
|
3307
|
+
|
|
3308
|
+
while ( ca_iter_state_next_slab(&st, &p, &m, &n) ) {
|
|
3309
|
+
if ( m != NULL ) {
|
|
3310
|
+
mask_seen = 1;
|
|
3311
|
+
if ( n > 0 ) rb_str_cat(mask_bytes, (char *) m, n);
|
|
3312
|
+
}
|
|
3313
|
+
}
|
|
3314
|
+
ca_iter_state_finish(&st);
|
|
3315
|
+
|
|
3316
|
+
result = rb_hash_new();
|
|
3317
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3318
|
+
rb_hash_aset(result, ID2SYM(rb_intern("mask_seen")), mask_seen ? Qtrue : Qfalse);
|
|
3319
|
+
rb_hash_aset(result, ID2SYM(rb_intern("mask_bytes")), mask_bytes);
|
|
3320
|
+
return result;
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
/* Bench-grade smoke for the L1 walk (CAStride + descriptor sources).
|
|
3324
|
+
Runs the full init / next_slab / finish cycle with no Ruby String
|
|
3325
|
+
materialisation in the hot loop — the kernel "consumes" the slab
|
|
3326
|
+
by xoring its first byte into a volatile sink (defeats dead-code
|
|
3327
|
+
elimination, costs nothing measurable on top of the materialise
|
|
3328
|
+
already done inside init). Returns total elements walked as an
|
|
3329
|
+
Integer. Use this when comparing against view.to_ca; the regular
|
|
3330
|
+
t1_smoke includes an rb_str_cat that doubles the materialise
|
|
3331
|
+
memcpy cost. */
|
|
3332
|
+
static VALUE
|
|
3333
|
+
rb_t1_smoke_attach (VALUE klass, VALUE vsrc)
|
|
3334
|
+
{
|
|
3335
|
+
CArray *src;
|
|
3336
|
+
ca_iter_state st;
|
|
3337
|
+
char *p;
|
|
3338
|
+
ca_size_t n;
|
|
3339
|
+
int rc;
|
|
3340
|
+
ca_size_t total = 0;
|
|
3341
|
+
volatile char sink = 0;
|
|
3342
|
+
|
|
3343
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3344
|
+
rc = ca_iter_state_init_l1(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3345
|
+
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3346
|
+
|
|
3347
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3348
|
+
if ( p != NULL && n > 0 ) sink ^= p[0];
|
|
3349
|
+
total += n;
|
|
3350
|
+
}
|
|
3351
|
+
(void) sink;
|
|
3352
|
+
ca_iter_state_finish(&st);
|
|
3353
|
+
return SIZE2NUM(total);
|
|
3354
|
+
}
|
|
3355
|
+
|
|
3356
|
+
/* L2 bench-grade smoke (sub-step 5.3+). Same shape as t1_smoke_attach
|
|
3357
|
+
but uses init_l2 / next_slab_strided so descriptor sources can be
|
|
3358
|
+
exercised through the L2 path. */
|
|
3359
|
+
static VALUE
|
|
3360
|
+
rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
3361
|
+
{
|
|
3362
|
+
CArray *src;
|
|
3363
|
+
ca_iter_state st;
|
|
3364
|
+
char *p;
|
|
3365
|
+
ca_size_t n, stride;
|
|
3366
|
+
int rc;
|
|
3367
|
+
ca_size_t total = 0;
|
|
3368
|
+
volatile char sink = 0;
|
|
3369
|
+
|
|
3370
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3371
|
+
rc = ca_iter_state_init_l2(&st, src, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3372
|
+
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3373
|
+
|
|
3374
|
+
while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
|
|
3375
|
+
if ( p != NULL && n > 0 ) sink ^= p[0];
|
|
3376
|
+
total += n;
|
|
3377
|
+
}
|
|
3378
|
+
(void) sink;
|
|
3379
|
+
ca_iter_state_finish(&st);
|
|
3380
|
+
return SIZE2NUM(total);
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
/* step 7: minimal smoke that exposes the `flags` argument to Ruby
|
|
3384
|
+
tests. Just runs init_l1 with the requested flags and returns
|
|
3385
|
+
the rc — used to verify NO_MASK enforcement (and any future
|
|
3386
|
+
flag-gated rejection paths) without needing a full kernel walk. */
|
|
3387
|
+
static VALUE
|
|
3388
|
+
rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
|
|
3389
|
+
{
|
|
3390
|
+
CArray *src;
|
|
3391
|
+
ca_iter_state st;
|
|
3392
|
+
int rc;
|
|
3393
|
+
uint32_t flags = (uint32_t) NUM2UINT(vflags);
|
|
3394
|
+
|
|
3395
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3396
|
+
rc = ca_iter_state_init_l1(&st, src, CA_SLAB_WHOLE, NULL, 0, flags);
|
|
3397
|
+
if ( rc == CA_ITER_OK ) ca_iter_state_finish(&st);
|
|
3398
|
+
return INT2NUM(rc);
|
|
3399
|
+
}
|
|
3400
|
+
|
|
3401
|
+
/* ---- Phase A capstone: CA_SLAB_AXES smoke -------------------------
|
|
3402
|
+
sum a float64 source over user-specified slab axes via init_l2 +
|
|
3403
|
+
next_slab_axes. K-D walk over slab dims using slab_strides for data
|
|
3404
|
+
and slab_mask_strides for mask (so non-contig CAStride sources work
|
|
3405
|
+
too). Returns total sum as a Float. Variadic int axes arg
|
|
3406
|
+
(= matches CArray#sum surface):
|
|
3407
|
+
CArray.t1_smoke_sum_axes_f64(ca, 0)
|
|
3408
|
+
CArray.t1_smoke_sum_axes_f64(ca, 0, 2)
|
|
3409
|
+
*/
|
|
3410
|
+
static VALUE
|
|
3411
|
+
rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
|
|
3412
|
+
{
|
|
3413
|
+
CArray *src;
|
|
3414
|
+
ca_iter_state st;
|
|
3415
|
+
char *p;
|
|
3416
|
+
boolean8_t *m;
|
|
3417
|
+
int rc;
|
|
3418
|
+
int8_t slab_axes[CA_RANK_MAX];
|
|
3419
|
+
int8_t naxes;
|
|
3420
|
+
double acc = 0.0;
|
|
3421
|
+
int i;
|
|
3422
|
+
|
|
3423
|
+
if ( argc < 2 ) {
|
|
3424
|
+
rb_raise(rb_eArgError, "t1_smoke_sum_axes_f64 expects (ca, axis...)");
|
|
3425
|
+
}
|
|
3426
|
+
TypedData_Get_Struct(argv[0], CArray, &carray_data_type, src);
|
|
3427
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3428
|
+
rb_raise(rb_eTypeError, "t1_smoke_sum_axes_f64 expects a float64 source");
|
|
3429
|
+
}
|
|
3430
|
+
naxes = (int8_t) (argc - 1);
|
|
3431
|
+
if ( naxes > CA_RANK_MAX ) {
|
|
3432
|
+
rb_raise(rb_eArgError, "too many axes");
|
|
3433
|
+
}
|
|
3434
|
+
for ( i = 0; i < naxes; i++ ) {
|
|
3435
|
+
slab_axes[i] = (int8_t) NUM2INT(argv[i + 1]);
|
|
3436
|
+
}
|
|
3437
|
+
|
|
3438
|
+
rc = ca_iter_state_init_l2(&st, src, CA_SLAB_AXES, slab_axes, naxes, 0);
|
|
3439
|
+
if ( rc != CA_ITER_OK ) {
|
|
3440
|
+
rb_raise(rb_eRuntimeError, "init_l2 CA_SLAB_AXES failed rc=%d", rc);
|
|
3441
|
+
}
|
|
3442
|
+
|
|
3443
|
+
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
3444
|
+
/* K-D walk over slab via outer_idx-style iteration over slab_dims.
|
|
3445
|
+
For simplicity / correctness, use a flat counter that decomposes
|
|
3446
|
+
into per-axis indices (= general-purpose walk, no SIMD pattern). */
|
|
3447
|
+
ca_size_t cell_count = st.slab_elements;
|
|
3448
|
+
ca_size_t idx[CA_RANK_MAX] = { 0 };
|
|
3449
|
+
for ( ca_size_t c = 0; c < cell_count; c++ ) {
|
|
3450
|
+
ca_size_t data_off = 0;
|
|
3451
|
+
ca_size_t mask_off = 0;
|
|
3452
|
+
for ( int8_t k = 0; k < st.slab_ndim; k++ ) {
|
|
3453
|
+
data_off += idx[k] * st.slab_strides[k];
|
|
3454
|
+
mask_off += idx[k] * st.slab_mask_strides[k];
|
|
3455
|
+
}
|
|
3456
|
+
if ( m == NULL || !m[mask_off] ) {
|
|
3457
|
+
acc += *(double *)(p + data_off);
|
|
3458
|
+
}
|
|
3459
|
+
/* advance idx row-major (innermost first) */
|
|
3460
|
+
for ( int8_t k = st.slab_ndim - 1; k >= 0; k-- ) {
|
|
3461
|
+
if ( ++idx[k] < st.slab_dims[k] ) break;
|
|
3462
|
+
idx[k] = 0;
|
|
3463
|
+
}
|
|
3464
|
+
}
|
|
3465
|
+
}
|
|
3466
|
+
ca_iter_state_finish(&st);
|
|
3467
|
+
return DBL2NUM(acc);
|
|
3468
|
+
}
|
|
3469
|
+
|
|
3470
|
+
/* ---- Phase B.3 helper smoke: rb_ca_parse_reduce_axes ---------------
|
|
3471
|
+
Calls the helper and returns the parsed axes as a Ruby Array of
|
|
3472
|
+
Integer. Lets tests inspect parsing behaviour directly.
|
|
3473
|
+
Signature: CArray.t1_test_parse_reduce_axes(ca, *args) */
|
|
3474
|
+
static VALUE
|
|
3475
|
+
rb_t1_test_parse_reduce_axes (int argc, VALUE *argv, VALUE klass)
|
|
3476
|
+
{
|
|
3477
|
+
CArray *ca;
|
|
3478
|
+
int8_t axes[CA_RANK_MAX];
|
|
3479
|
+
int8_t naxes;
|
|
3480
|
+
int i;
|
|
3481
|
+
|
|
3482
|
+
if ( argc < 1 ) {
|
|
3483
|
+
rb_raise(rb_eArgError,
|
|
3484
|
+
"t1_test_parse_reduce_axes expects (ca, *axis_args)");
|
|
3485
|
+
}
|
|
3486
|
+
TypedData_Get_Struct(argv[0], CArray, &carray_data_type, ca);
|
|
3487
|
+
naxes = rb_ca_parse_reduce_axes(argc - 1, argv + 1, ca, axes);
|
|
3488
|
+
|
|
3489
|
+
VALUE arr = rb_ary_new_capa(naxes);
|
|
3490
|
+
for ( i = 0; i < naxes; i++ ) {
|
|
3491
|
+
rb_ary_push(arr, INT2NUM((int) axes[i]));
|
|
3492
|
+
}
|
|
3493
|
+
return arr;
|
|
3494
|
+
}
|
|
3495
|
+
|
|
3496
|
+
/* ---- API harmonisation A.1 smoke: rb_ca_parse_reduce_axes_kw -------
|
|
3497
|
+
Calls the kwarg helper and returns the parsed axes as a Ruby Array.
|
|
3498
|
+
Lets tests inspect kwarg-form parsing (Qnil / Integer / Array) +
|
|
3499
|
+
validation parity with the variadic entry.
|
|
3500
|
+
Signature: CArray.test_parse_reduce_axes_kw(ca, axis: ...) */
|
|
3501
|
+
static VALUE
|
|
3502
|
+
rb_test_parse_reduce_axes_kw (int argc, VALUE *argv, VALUE klass)
|
|
3503
|
+
{
|
|
3504
|
+
CArray *ca;
|
|
3505
|
+
int8_t axes[CA_RANK_MAX];
|
|
3506
|
+
int8_t naxes;
|
|
3507
|
+
int i;
|
|
3508
|
+
VALUE ca_obj, kw_hash, axis_val = Qnil;
|
|
3509
|
+
|
|
3510
|
+
rb_scan_args(argc, argv, "1:", &ca_obj, &kw_hash);
|
|
3511
|
+
TypedData_Get_Struct(ca_obj, CArray, &carray_data_type, ca);
|
|
3512
|
+
|
|
3513
|
+
rb_scan_options(kw_hash, "axis", &axis_val);
|
|
3514
|
+
|
|
3515
|
+
naxes = rb_ca_parse_reduce_axes_kw(axis_val, ca, axes);
|
|
3516
|
+
|
|
3517
|
+
VALUE arr = rb_ary_new_capa(naxes);
|
|
3518
|
+
for ( i = 0; i < naxes; i++ ) {
|
|
3519
|
+
rb_ary_push(arr, INT2NUM((int) axes[i]));
|
|
3520
|
+
}
|
|
3521
|
+
return arr;
|
|
3522
|
+
}
|
|
3523
|
+
|
|
3524
|
+
/* ---- Phase A.3 helper smoke: rb_ca_new_reduced ---------------------
|
|
3525
|
+
Exposes rb_ca_new_reduced (carray_core.c) for unit testing. Returns
|
|
3526
|
+
the newly allocated output CArray (= tests verify shape + data_type).
|
|
3527
|
+
Signature: CArray.t1_test_new_reduced(ca, data_type, axis1, axis2, ...) */
|
|
3528
|
+
static VALUE
|
|
3529
|
+
rb_t1_test_new_reduced (int argc, VALUE *argv, VALUE klass)
|
|
3530
|
+
{
|
|
3531
|
+
int8_t axes[CA_RANK_MAX];
|
|
3532
|
+
int8_t naxes;
|
|
3533
|
+
int32_t data_type;
|
|
3534
|
+
|
|
3535
|
+
if ( argc < 3 ) {
|
|
3536
|
+
rb_raise(rb_eArgError,
|
|
3537
|
+
"t1_test_new_reduced expects (ca, data_type, axis...) with >= 1 axis");
|
|
3538
|
+
}
|
|
3539
|
+
/* PROPOSAL_DTYPE_SYMBOL_FLIP: accept Symbol / Integer / Class / String
|
|
3540
|
+
uniformly via rb_ca_guess_type so post-flip CA_* (Symbol) callers
|
|
3541
|
+
and legacy Integer-code callers both work. */
|
|
3542
|
+
data_type = (int32_t) rb_ca_guess_type(argv[1]);
|
|
3543
|
+
naxes = (int8_t) (argc - 2);
|
|
3544
|
+
if ( naxes > CA_RANK_MAX ) {
|
|
3545
|
+
rb_raise(rb_eArgError, "too many axes");
|
|
3546
|
+
}
|
|
3547
|
+
for ( int i = 0; i < naxes; i++ ) {
|
|
3548
|
+
axes[i] = (int8_t) NUM2INT(argv[2 + i]);
|
|
3549
|
+
}
|
|
3550
|
+
return rb_ca_new_reduced(argv[0], axes, naxes, data_type, 0);
|
|
3551
|
+
}
|
|
3552
|
+
|
|
3553
|
+
/* ---- step 9.3: CAReduce-specific smoke -----------------------------
|
|
3554
|
+
CAReduce has no public Ruby ctor (it's an internal class used by
|
|
3555
|
+
CARefer mask handling). Tests need to drive kernel_iterator over
|
|
3556
|
+
a CAReduce, so we expose a thin construction helper plus dedicated
|
|
3557
|
+
read / write smokes. All three accept a boolean parent — CAReduce
|
|
3558
|
+
is fixed to CA_BOOLEAN (ca_obj_reduce.c:71). */
|
|
3559
|
+
|
|
3560
|
+
static VALUE
|
|
3561
|
+
rb_t1_make_reduce (VALUE klass, VALUE vparent, VALUE vcount, VALUE voffset)
|
|
3562
|
+
{
|
|
3563
|
+
CArray *parent;
|
|
3564
|
+
TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
|
|
3565
|
+
ca_size_t count = NUM2SIZE(vcount);
|
|
3566
|
+
ca_size_t offset = NUM2SIZE(voffset);
|
|
3567
|
+
CArray *r = (CArray *) ca_reduce_new(parent, count, offset);
|
|
3568
|
+
return ca_wrap_struct(r);
|
|
3569
|
+
}
|
|
3570
|
+
|
|
3571
|
+
/* READ smoke for CAReduce: returns Hash mirroring t1_smoke (rc / slabs /
|
|
3572
|
+
total_elems / alias_mode / data) so binary parity is checkable. */
|
|
3573
|
+
static VALUE
|
|
3574
|
+
rb_t1_smoke_reduce_read (VALUE klass, VALUE vparent, VALUE vcount, VALUE voffset)
|
|
3575
|
+
{
|
|
3576
|
+
CArray *parent;
|
|
3577
|
+
ca_iter_state st;
|
|
3578
|
+
char *p;
|
|
3579
|
+
ca_size_t n;
|
|
3580
|
+
int rc;
|
|
3581
|
+
int slabs = 0;
|
|
3582
|
+
ca_size_t total_elems = 0;
|
|
3583
|
+
int alias_mode = CA_ITER_ALIAS_NONE;
|
|
3584
|
+
VALUE result, data;
|
|
3585
|
+
CArray *reduce;
|
|
3586
|
+
|
|
3587
|
+
TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
|
|
3588
|
+
ca_size_t count = NUM2SIZE(vcount);
|
|
3589
|
+
ca_size_t offset = NUM2SIZE(voffset);
|
|
3590
|
+
reduce = (CArray *) ca_reduce_new(parent, count, offset);
|
|
3591
|
+
data = rb_str_new(0, 0);
|
|
3592
|
+
|
|
3593
|
+
rc = ca_iter_state_init_l1(&st, reduce, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3594
|
+
if ( rc == CA_ITER_OK ) {
|
|
3595
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3596
|
+
if ( p != NULL && n > 0 ) rb_str_cat(data, p, n * st.bytes);
|
|
3597
|
+
total_elems += n;
|
|
3598
|
+
slabs++;
|
|
3599
|
+
}
|
|
3600
|
+
alias_mode = st.alias_mode;
|
|
3601
|
+
ca_iter_state_finish(&st);
|
|
3602
|
+
}
|
|
3603
|
+
/* reduce was allocated via ALLOC (ca_reduce_new) but not wrapped in
|
|
3604
|
+
a Ruby VALUE, so it would leak. ca_free dispatches to the view's
|
|
3605
|
+
free_object which already xfrees the struct (ca_obj_reduce.c:113). */
|
|
3606
|
+
ca_free(reduce);
|
|
3607
|
+
|
|
3608
|
+
result = rb_hash_new();
|
|
3609
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3610
|
+
rb_hash_aset(result, ID2SYM(rb_intern("slabs")), INT2NUM(slabs));
|
|
3611
|
+
rb_hash_aset(result, ID2SYM(rb_intern("total_elems")), SIZE2NUM(total_elems));
|
|
3612
|
+
rb_hash_aset(result, ID2SYM(rb_intern("alias_mode")), INT2NUM(alias_mode));
|
|
3613
|
+
rb_hash_aset(result, ID2SYM(rb_intern("data")), data);
|
|
3614
|
+
return result;
|
|
3615
|
+
}
|
|
3616
|
+
|
|
3617
|
+
/* WRITE broadcast smoke for CAReduce: fills the reduce view with
|
|
3618
|
+
`val` (boolean / 0 or 1) and lets sync_slab → ca_sync run the
|
|
3619
|
+
broadcast scatter back to parent. After the call parent's
|
|
3620
|
+
[offset..offset+elems*count) bytes should all equal val. Returns
|
|
3621
|
+
rc; on rc != OK parent is not modified. */
|
|
3622
|
+
static VALUE
|
|
3623
|
+
rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
|
|
3624
|
+
VALUE vcount, VALUE voffset, VALUE vval)
|
|
3625
|
+
{
|
|
3626
|
+
CArray *parent;
|
|
3627
|
+
ca_iter_state st;
|
|
3628
|
+
char *p;
|
|
3629
|
+
ca_size_t n, i;
|
|
3630
|
+
int rc;
|
|
3631
|
+
CArray *reduce;
|
|
3632
|
+
uint8_t v = (uint8_t) NUM2UINT(vval);
|
|
3633
|
+
|
|
3634
|
+
TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
|
|
3635
|
+
ca_size_t count = NUM2SIZE(vcount);
|
|
3636
|
+
ca_size_t offset = NUM2SIZE(voffset);
|
|
3637
|
+
reduce = (CArray *) ca_reduce_new(parent, count, offset);
|
|
3638
|
+
|
|
3639
|
+
rc = ca_iter_state_init_l1(&st, reduce, CA_SLAB_WHOLE, NULL, 0,
|
|
3640
|
+
CA_KERNEL_WRITE);
|
|
3641
|
+
if ( rc != CA_ITER_OK ) {
|
|
3642
|
+
ca_free(reduce);
|
|
3643
|
+
return INT2NUM(rc);
|
|
3644
|
+
}
|
|
3645
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3646
|
+
for ( i = 0; i < n; i++ ) p[i] = v;
|
|
3647
|
+
ca_iter_state_sync_slab(&st); /* triggers ca_sync → broadcast */
|
|
3648
|
+
}
|
|
3649
|
+
ca_iter_state_finish(&st);
|
|
3650
|
+
ca_free(reduce);
|
|
3651
|
+
return INT2NUM(CA_ITER_OK);
|
|
3652
|
+
}
|
|
3653
|
+
|
|
3654
|
+
/* ---- M.6: CARemap kernel_iterator smoke helpers --------------------
|
|
3655
|
+
CARemap has no public Ruby ctor (internal-only per §2.2), so we
|
|
3656
|
+
construct it from C using the (ref, idx) args. Pattern mirrors
|
|
3657
|
+
rb_t1_smoke_reduce_*: build the view, run the iterator state
|
|
3658
|
+
machine, return the standard hash (rc / slabs / total_elems /
|
|
3659
|
+
alias_mode / data). Expected alias_mode = ALIAS_NONE since
|
|
3660
|
+
SRC_ATTACH always materialises via xfer_all into iter-owned
|
|
3661
|
+
scratch. */
|
|
3662
|
+
|
|
3663
|
+
static VALUE
|
|
3664
|
+
rb_t1_smoke_remap_read (VALUE klass, VALUE vref, VALUE vidx)
|
|
3665
|
+
{
|
|
3666
|
+
CArray *ref, *idx, *view;
|
|
3667
|
+
ca_iter_state st;
|
|
3668
|
+
char *p;
|
|
3669
|
+
ca_size_t n;
|
|
3670
|
+
int rc;
|
|
3671
|
+
int slabs = 0;
|
|
3672
|
+
ca_size_t total_elems = 0;
|
|
3673
|
+
int alias_mode = CA_ITER_ALIAS_NONE;
|
|
3674
|
+
VALUE result, data;
|
|
3675
|
+
|
|
3676
|
+
(void) klass;
|
|
3677
|
+
TypedData_Get_Struct(vref, CArray, &carray_data_type, ref);
|
|
3678
|
+
TypedData_Get_Struct(vidx, CArray, &carray_data_type, idx);
|
|
3679
|
+
view = ca_remap_new(ref, idx);
|
|
3680
|
+
data = rb_str_new(0, 0);
|
|
3681
|
+
|
|
3682
|
+
rc = ca_iter_state_init_l1(&st, view, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3683
|
+
if ( rc == CA_ITER_OK ) {
|
|
3684
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3685
|
+
if ( p != NULL && n > 0 ) rb_str_cat(data, p, n * st.bytes);
|
|
3686
|
+
total_elems += n;
|
|
3687
|
+
slabs++;
|
|
3688
|
+
}
|
|
3689
|
+
alias_mode = st.alias_mode;
|
|
3690
|
+
ca_iter_state_finish(&st);
|
|
3691
|
+
}
|
|
3692
|
+
ca_free(view);
|
|
3693
|
+
|
|
3694
|
+
result = rb_hash_new();
|
|
3695
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3696
|
+
rb_hash_aset(result, ID2SYM(rb_intern("slabs")), INT2NUM(slabs));
|
|
3697
|
+
rb_hash_aset(result, ID2SYM(rb_intern("total_elems")), SIZE2NUM(total_elems));
|
|
3698
|
+
rb_hash_aset(result, ID2SYM(rb_intern("alias_mode")), INT2NUM(alias_mode));
|
|
3699
|
+
rb_hash_aset(result, ID2SYM(rb_intern("data")), data);
|
|
3700
|
+
return result;
|
|
3701
|
+
}
|
|
3702
|
+
|
|
3703
|
+
static VALUE
|
|
3704
|
+
rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
|
|
3705
|
+
{
|
|
3706
|
+
CArray *ref, *idx, *view;
|
|
3707
|
+
ca_iter_state st;
|
|
3708
|
+
char *p;
|
|
3709
|
+
ca_size_t n;
|
|
3710
|
+
int rc;
|
|
3711
|
+
int slabs = 0;
|
|
3712
|
+
ca_size_t total_elems = 0;
|
|
3713
|
+
int alias_mode = CA_ITER_ALIAS_NONE;
|
|
3714
|
+
VALUE result, data;
|
|
3715
|
+
|
|
3716
|
+
(void) klass;
|
|
3717
|
+
TypedData_Get_Struct(vref, CArray, &carray_data_type, ref);
|
|
3718
|
+
TypedData_Get_Struct(vidx, CArray, &carray_data_type, idx);
|
|
3719
|
+
view = ca_remap_new(ref, idx);
|
|
3720
|
+
data = rb_str_new(0, 0);
|
|
3721
|
+
|
|
3722
|
+
rc = ca_iter_state_init_l2(&st, view, CA_SLAB_WHOLE, NULL, 0, 0);
|
|
3723
|
+
if ( rc == CA_ITER_OK ) {
|
|
3724
|
+
ca_size_t stride;
|
|
3725
|
+
while ( ca_iter_state_next_slab_strided(&st, &p, NULL, &n, &stride) ) {
|
|
3726
|
+
ca_size_t i;
|
|
3727
|
+
for ( i = 0; i < n; i++ ) {
|
|
3728
|
+
rb_str_cat(data, p + i * stride, st.bytes);
|
|
3729
|
+
}
|
|
3730
|
+
total_elems += n;
|
|
3731
|
+
slabs++;
|
|
3732
|
+
}
|
|
3733
|
+
alias_mode = st.alias_mode;
|
|
3734
|
+
ca_iter_state_finish(&st);
|
|
3735
|
+
}
|
|
3736
|
+
ca_free(view);
|
|
3737
|
+
|
|
3738
|
+
result = rb_hash_new();
|
|
3739
|
+
rb_hash_aset(result, ID2SYM(rb_intern("rc")), INT2NUM(rc));
|
|
3740
|
+
rb_hash_aset(result, ID2SYM(rb_intern("slabs")), INT2NUM(slabs));
|
|
3741
|
+
rb_hash_aset(result, ID2SYM(rb_intern("total_elems")), SIZE2NUM(total_elems));
|
|
3742
|
+
rb_hash_aset(result, ID2SYM(rb_intern("alias_mode")), INT2NUM(alias_mode));
|
|
3743
|
+
rb_hash_aset(result, ID2SYM(rb_intern("data")), data);
|
|
3744
|
+
return result;
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3747
|
+
/* WRITE smoke: fill view's slab buffer with `val` (float64), let
|
|
3748
|
+
sync_slab → ca_sync run the scatter back to ref via xfer_all PUT. */
|
|
3749
|
+
static VALUE
|
|
3750
|
+
rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
|
|
3751
|
+
VALUE vval)
|
|
3752
|
+
{
|
|
3753
|
+
CArray *ref, *idx, *view;
|
|
3754
|
+
ca_iter_state st;
|
|
3755
|
+
char *p;
|
|
3756
|
+
ca_size_t n, i;
|
|
3757
|
+
int rc;
|
|
3758
|
+
double v = NUM2DBL(vval);
|
|
3759
|
+
|
|
3760
|
+
(void) klass;
|
|
3761
|
+
TypedData_Get_Struct(vref, CArray, &carray_data_type, ref);
|
|
3762
|
+
TypedData_Get_Struct(vidx, CArray, &carray_data_type, idx);
|
|
3763
|
+
view = ca_remap_new(ref, idx);
|
|
3764
|
+
|
|
3765
|
+
rc = ca_iter_state_init_l1(&st, view, CA_SLAB_WHOLE, NULL, 0,
|
|
3766
|
+
CA_KERNEL_WRITE);
|
|
3767
|
+
if ( rc != CA_ITER_OK ) {
|
|
3768
|
+
ca_free(view);
|
|
3769
|
+
return INT2NUM(rc);
|
|
3770
|
+
}
|
|
3771
|
+
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3772
|
+
double *q = (double *) p;
|
|
3773
|
+
for ( i = 0; i < n; i++ ) q[i] = v;
|
|
3774
|
+
ca_iter_state_sync_slab(&st);
|
|
3775
|
+
}
|
|
3776
|
+
ca_iter_state_finish(&st);
|
|
3777
|
+
ca_free(view);
|
|
3778
|
+
return INT2NUM(CA_ITER_OK);
|
|
3779
|
+
}
|
|
3780
|
+
|
|
3781
|
+
/* ---- Phase C C.3: CA_FOR_EACH_SLAB macro smoke surfaces -------------
|
|
3782
|
+
Smoke kernels that exercise the block macros end-to-end. Used by
|
|
3783
|
+
spec_ai/test_ca_for_each_slab_macros.rb to pin behavioural
|
|
3784
|
+
correctness of the macro expansion (= same byte-parity result as
|
|
3785
|
+
the raw API equivalent). */
|
|
3786
|
+
|
|
3787
|
+
/* sum reduction using CA_FOR_EACH_SLAB. innermost slab axis only,
|
|
3788
|
+
float64 source, accumulator = scalar (single full-reduction slab
|
|
3789
|
+
when naxes == src->ndim, otherwise per-outer-slab accumulator
|
|
3790
|
+
written to a row-major output buffer). Mirrors carray_kernel_sum
|
|
3791
|
+
structurally but uses the macro for lifecycle. */
|
|
3792
|
+
static VALUE
|
|
3793
|
+
rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
3794
|
+
{
|
|
3795
|
+
(void) klass;
|
|
3796
|
+
if ( argc < 2 ) {
|
|
3797
|
+
rb_raise(rb_eArgError, "expected (src, axis_int, ...)");
|
|
3798
|
+
}
|
|
3799
|
+
|
|
3800
|
+
VALUE vsrc = argv[0];
|
|
3801
|
+
CArray *ca;
|
|
3802
|
+
GetCArray(vsrc, ca);
|
|
3803
|
+
|
|
3804
|
+
int8_t slab_axes[CA_RANK_MAX];
|
|
3805
|
+
int8_t naxes = (int8_t) (argc - 1);
|
|
3806
|
+
if ( naxes < 1 || naxes > ca->ndim ) {
|
|
3807
|
+
rb_raise(rb_eArgError, "bad axes count");
|
|
3808
|
+
}
|
|
3809
|
+
for ( int8_t k = 0; k < naxes; k++ ) {
|
|
3810
|
+
slab_axes[k] = (int8_t) NUM2INT(argv[1 + k]);
|
|
3811
|
+
if ( slab_axes[k] < 0 ) slab_axes[k] += ca->ndim;
|
|
3812
|
+
}
|
|
3813
|
+
|
|
3814
|
+
/* Output: same layout policy as sum_ki (= reduced shape via
|
|
3815
|
+
rb_ca_new_reduced). For full reduction naxes == ndim, output
|
|
3816
|
+
is a 1-element CArray that we unwrap to Float below. */
|
|
3817
|
+
VALUE vout = rb_ca_new_reduced(vsrc, slab_axes, naxes, CA_FLOAT64, 0);
|
|
3818
|
+
CArray *co;
|
|
3819
|
+
GetCArray(vout, co);
|
|
3820
|
+
double *op = (double *) co->ptr;
|
|
3821
|
+
|
|
3822
|
+
ca_iter_state st;
|
|
3823
|
+
char *p;
|
|
3824
|
+
boolean8_t *m;
|
|
3825
|
+
ca_size_t out_i = 0;
|
|
3826
|
+
|
|
3827
|
+
CA_FOR_EACH_SLAB(st, ca, slab_axes, naxes, 0, p, m) {
|
|
3828
|
+
/* K-1 outer carry + innermost inner walk (= same shape as sum_ki).
|
|
3829
|
+
Required for slab_ndim > 1 because slab cells aren't a single
|
|
3830
|
+
contig run with one stride — outer slab axes have their own
|
|
3831
|
+
strides. */
|
|
3832
|
+
double acc = 0.0;
|
|
3833
|
+
int8_t K = st.slab_ndim;
|
|
3834
|
+
int8_t outer_K = K - 1;
|
|
3835
|
+
ca_size_t inner_n = st.slab_dims[K - 1];
|
|
3836
|
+
ca_size_t inner_s = st.slab_strides[K - 1];
|
|
3837
|
+
ca_size_t inner_ms = st.slab_mask_strides[K - 1];
|
|
3838
|
+
|
|
3839
|
+
ca_size_t outer_count = 1;
|
|
3840
|
+
for ( int8_t k = 0; k < outer_K; k++ ) outer_count *= st.slab_dims[k];
|
|
3841
|
+
|
|
3842
|
+
ca_size_t idx[CA_RANK_MAX] = { 0 };
|
|
3843
|
+
for ( ca_size_t o = 0; o < outer_count; o++ ) {
|
|
3844
|
+
ca_size_t data_off = 0;
|
|
3845
|
+
ca_size_t mask_off = 0;
|
|
3846
|
+
for ( int8_t k = 0; k < outer_K; k++ ) {
|
|
3847
|
+
data_off += idx[k] * st.slab_strides[k];
|
|
3848
|
+
mask_off += idx[k] * st.slab_mask_strides[k];
|
|
3849
|
+
}
|
|
3850
|
+
const char *q = p + data_off;
|
|
3851
|
+
if ( m == NULL ) {
|
|
3852
|
+
for ( ca_size_t j = 0; j < inner_n; j++ ) {
|
|
3853
|
+
acc += *(const double *) (q + j * inner_s);
|
|
3854
|
+
}
|
|
3855
|
+
} else {
|
|
3856
|
+
const boolean8_t *mm = m + mask_off;
|
|
3857
|
+
for ( ca_size_t j = 0; j < inner_n; j++ ) {
|
|
3858
|
+
if ( ! mm[j * inner_ms] ) {
|
|
3859
|
+
acc += *(const double *) (q + j * inner_s);
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
}
|
|
3863
|
+
for ( int8_t k = outer_K - 1; k >= 0; k-- ) {
|
|
3864
|
+
if ( ++idx[k] < st.slab_dims[k] ) break;
|
|
3865
|
+
idx[k] = 0;
|
|
3866
|
+
}
|
|
3867
|
+
}
|
|
3868
|
+
op[out_i++] = acc;
|
|
3869
|
+
}
|
|
3870
|
+
|
|
3871
|
+
if ( naxes == ca->ndim ) {
|
|
3872
|
+
return rb_float_new(op[0]);
|
|
3873
|
+
}
|
|
3874
|
+
return vout;
|
|
3875
|
+
}
|
|
3876
|
+
|
|
3877
|
+
/* map kernel using CA_FOR_EACH_SLAB_INOUT. Doubles each element of
|
|
3878
|
+
a float64 source into a fresh same-shape output. Slab axis is
|
|
3879
|
+
always the innermost (= argv[1] is unused for simplicity, axis
|
|
3880
|
+
fixed to ca->ndim - 1). Demonstrates the 2-iter pattern. */
|
|
3881
|
+
static VALUE
|
|
3882
|
+
rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
|
|
3883
|
+
{
|
|
3884
|
+
(void) klass;
|
|
3885
|
+
CArray *ca;
|
|
3886
|
+
GetCArray(vsrc, ca);
|
|
3887
|
+
if ( ca->data_type != CA_FLOAT64 ) {
|
|
3888
|
+
rb_raise(rb_eTypeError, "expected float64 source");
|
|
3889
|
+
}
|
|
3890
|
+
|
|
3891
|
+
VALUE vout = rb_ca_template_with_type(vsrc, INT2NUM(CA_FLOAT64), Qnil);
|
|
3892
|
+
CArray *co;
|
|
3893
|
+
GetCArray(vout, co);
|
|
3894
|
+
|
|
3895
|
+
int8_t slab_axes[CA_RANK_MAX];
|
|
3896
|
+
int8_t naxes = 1;
|
|
3897
|
+
slab_axes[0] = (int8_t) (ca->ndim - 1);
|
|
3898
|
+
|
|
3899
|
+
ca_iter_state st_in, st_out;
|
|
3900
|
+
char *p_in, *p_out;
|
|
3901
|
+
boolean8_t *m_in, *m_out;
|
|
3902
|
+
|
|
3903
|
+
CA_FOR_EACH_SLAB_INOUT(st_in, st_out, ca, co,
|
|
3904
|
+
slab_axes, naxes,
|
|
3905
|
+
p_in, p_out, m_in, m_out) {
|
|
3906
|
+
ca_size_t n = st_in.slab_n;
|
|
3907
|
+
ca_size_t in_s = st_in.slab_strides[st_in.slab_ndim - 1];
|
|
3908
|
+
ca_size_t out_s = st_out.slab_strides[st_out.slab_ndim - 1];
|
|
3909
|
+
/* Both sides float64; for the typical entity output, in_s = out_s
|
|
3910
|
+
= sizeof(double). Mask propagation: if input is masked at cell
|
|
3911
|
+
j, just leave output alone (= CArray's default mask propagation
|
|
3912
|
+
happens via co's own mask, not our concern here for the smoke). */
|
|
3913
|
+
(void) m_in;
|
|
3914
|
+
(void) m_out;
|
|
3915
|
+
for ( ca_size_t j = 0; j < n; j++ ) {
|
|
3916
|
+
double v = *(const double *) (p_in + j * in_s);
|
|
3917
|
+
*(double *) (p_out + j * out_s) = v * 2.0;
|
|
3918
|
+
}
|
|
3919
|
+
}
|
|
3920
|
+
|
|
3921
|
+
return vout;
|
|
3922
|
+
}
|
|
3923
|
+
|
|
3924
|
+
/* ---- PROPOSAL_FIBER_DELIVERY F.2: catalog macro smokes -------------
|
|
3925
|
+
Exercise each of the 4 catalog forms end-to-end with float64 fibers,
|
|
3926
|
+
verifying contig delivery for both innermost (= stride==bytes) and
|
|
3927
|
+
non-innermost (= stride>bytes, gather path) axis positions. Used by
|
|
3928
|
+
spec_ai/test_fiber_delivery.rb (= F.3). */
|
|
3929
|
+
|
|
3930
|
+
/* form 1 (NO_MASK single): per-axis sum, returns total sum.
|
|
3931
|
+
Catalog: CA_FOR_EACH_FIBER. */
|
|
3932
|
+
static VALUE
|
|
3933
|
+
rb_caf_fiber_smoke_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
3934
|
+
{
|
|
3935
|
+
CArray *src;
|
|
3936
|
+
ca_iter_state st;
|
|
3937
|
+
char *p;
|
|
3938
|
+
ca_size_t n;
|
|
3939
|
+
double total = 0.0;
|
|
3940
|
+
int axis = NUM2INT(vaxis);
|
|
3941
|
+
|
|
3942
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3943
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3944
|
+
rb_raise(rb_eArgError, "caf_fiber_smoke_sum_f64: requires float64");
|
|
3945
|
+
}
|
|
3946
|
+
CA_FOR_EACH_FIBER(st, src, axis, CA_KERNEL_NO_MASK, p, n) {
|
|
3947
|
+
const double *pd = (const double *) p;
|
|
3948
|
+
for ( ca_size_t i = 0; i < n; i++ ) total += pd[i];
|
|
3949
|
+
}
|
|
3950
|
+
return rb_float_new(total);
|
|
3951
|
+
}
|
|
3952
|
+
|
|
3953
|
+
/* form 2 (NO_MASK INOUT): per-axis fiber copy * scalar.
|
|
3954
|
+
Catalog: CA_FOR_EACH_FIBER_INOUT. Sorts each fiber for non-trivial
|
|
3955
|
+
gather/scatter behavior. */
|
|
3956
|
+
static VALUE
|
|
3957
|
+
rb_caf_fiber_smoke_double_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
3958
|
+
{
|
|
3959
|
+
CArray *src, *out;
|
|
3960
|
+
ca_iter_state st_in, st_out;
|
|
3961
|
+
char *pi, *po;
|
|
3962
|
+
ca_size_t n;
|
|
3963
|
+
int axis = NUM2INT(vaxis);
|
|
3964
|
+
VALUE vout;
|
|
3965
|
+
|
|
3966
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3967
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3968
|
+
rb_raise(rb_eArgError, "caf_fiber_smoke_double_f64: requires float64");
|
|
3969
|
+
}
|
|
3970
|
+
vout = rb_ca_template(vsrc);
|
|
3971
|
+
TypedData_Get_Struct(vout, CArray, &carray_data_type, out);
|
|
3972
|
+
|
|
3973
|
+
CA_FOR_EACH_FIBER_INOUT(st_in, st_out, src, out, axis,
|
|
3974
|
+
CA_KERNEL_NO_MASK, pi, po, n) {
|
|
3975
|
+
double *dpi = (double *) pi;
|
|
3976
|
+
double *dpo = (double *) po;
|
|
3977
|
+
for ( ca_size_t i = 0; i < n; i++ ) dpo[i] = dpi[i] * 2.0;
|
|
3978
|
+
}
|
|
3979
|
+
return vout;
|
|
3980
|
+
}
|
|
3981
|
+
|
|
3982
|
+
/* form 3 (mask-aware single): unmasked sum.
|
|
3983
|
+
Catalog: CA_FOR_EACH_FIBER_MASKED. */
|
|
3984
|
+
static VALUE
|
|
3985
|
+
rb_caf_fiber_smoke_unmasked_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
3986
|
+
{
|
|
3987
|
+
CArray *src;
|
|
3988
|
+
ca_iter_state st;
|
|
3989
|
+
char *p;
|
|
3990
|
+
boolean8_t *m;
|
|
3991
|
+
ca_size_t n;
|
|
3992
|
+
double total = 0.0;
|
|
3993
|
+
int axis = NUM2INT(vaxis);
|
|
3994
|
+
|
|
3995
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3996
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
3997
|
+
rb_raise(rb_eArgError, "caf_fiber_smoke_unmasked_sum_f64: requires float64");
|
|
3998
|
+
}
|
|
3999
|
+
CA_FOR_EACH_FIBER_MASKED(st, src, axis, 0, p, n, m) {
|
|
4000
|
+
double *dp = (double *) p;
|
|
4001
|
+
for ( ca_size_t i = 0; i < n; i++ ) {
|
|
4002
|
+
if ( !m || !m[i] ) total += dp[i];
|
|
4003
|
+
}
|
|
4004
|
+
}
|
|
4005
|
+
return rb_float_new(total);
|
|
4006
|
+
}
|
|
4007
|
+
|
|
4008
|
+
/* form 4 (mask-aware INOUT): copy input but zero out masked cells in
|
|
4009
|
+
output. Catalog: CA_FOR_EACH_FIBER_INOUT_MASKED. */
|
|
4010
|
+
static VALUE
|
|
4011
|
+
rb_caf_fiber_smoke_zero_masked_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
4012
|
+
{
|
|
4013
|
+
CArray *src, *out;
|
|
4014
|
+
ca_iter_state st_in, st_out;
|
|
4015
|
+
char *pi, *po;
|
|
4016
|
+
boolean8_t *m;
|
|
4017
|
+
ca_size_t n;
|
|
4018
|
+
int axis = NUM2INT(vaxis);
|
|
4019
|
+
VALUE vout;
|
|
4020
|
+
|
|
4021
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
4022
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
4023
|
+
rb_raise(rb_eArgError, "caf_fiber_smoke_zero_masked_f64: requires float64");
|
|
4024
|
+
}
|
|
4025
|
+
vout = rb_ca_template(vsrc);
|
|
4026
|
+
TypedData_Get_Struct(vout, CArray, &carray_data_type, out);
|
|
4027
|
+
|
|
4028
|
+
CA_FOR_EACH_FIBER_INOUT_MASKED(st_in, st_out, src, out, axis,
|
|
4029
|
+
0, pi, po, n, m) {
|
|
4030
|
+
double *dpi = (double *) pi;
|
|
4031
|
+
double *dpo = (double *) po;
|
|
4032
|
+
for ( ca_size_t i = 0; i < n; i++ ) {
|
|
4033
|
+
dpo[i] = (m && m[i]) ? 0.0 : dpi[i];
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
return vout;
|
|
4037
|
+
}
|
|
4038
|
+
|
|
4039
|
+
/* F.5 bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
|
|
4040
|
+
Functional equivalent of ext/carray_order.c::rb_ca_sort_copy_axis_*
|
|
4041
|
+
(= hand-rolled gather/scatter with explicit slab_strides[0] loops).
|
|
4042
|
+
The macro form lets us A/B the author-side overhead of stride math
|
|
4043
|
+
vs the catalog-macro-driven engine gather/scatter. */
|
|
4044
|
+
static int
|
|
4045
|
+
caf_fiber_bench_cmp_double (const void *a, const void *b)
|
|
4046
|
+
{
|
|
4047
|
+
double da = *(const double *) a;
|
|
4048
|
+
double db = *(const double *) b;
|
|
4049
|
+
return (da > db) - (da < db);
|
|
4050
|
+
}
|
|
4051
|
+
|
|
4052
|
+
static VALUE
|
|
4053
|
+
rb_caf_fiber_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
4054
|
+
{
|
|
4055
|
+
CArray *src, *out;
|
|
4056
|
+
ca_iter_state st_in, st_out;
|
|
4057
|
+
char *pi, *po;
|
|
4058
|
+
ca_size_t n;
|
|
4059
|
+
int axis = NUM2INT(vaxis);
|
|
4060
|
+
VALUE vout;
|
|
4061
|
+
|
|
4062
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
4063
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
4064
|
+
rb_raise(rb_eArgError, "caf_fiber_bench_sort_copy_f64: requires float64");
|
|
4065
|
+
}
|
|
4066
|
+
vout = rb_ca_template(vsrc);
|
|
4067
|
+
TypedData_Get_Struct(vout, CArray, &carray_data_type, out);
|
|
4068
|
+
|
|
4069
|
+
CA_FOR_EACH_FIBER_INOUT(st_in, st_out, src, out, axis,
|
|
4070
|
+
CA_KERNEL_NO_MASK, pi, po, n) {
|
|
4071
|
+
double *dpo = (double *) po;
|
|
4072
|
+
memcpy(po, pi, n * sizeof(double));
|
|
4073
|
+
#ifdef HAVE_MERGESORT
|
|
4074
|
+
if ( mergesort(dpo, n, sizeof(double), caf_fiber_bench_cmp_double) != 0 ) {
|
|
4075
|
+
qsort(dpo, n, sizeof(double), caf_fiber_bench_cmp_double);
|
|
4076
|
+
}
|
|
4077
|
+
#else
|
|
4078
|
+
qsort(dpo, n, sizeof(double), caf_fiber_bench_cmp_double);
|
|
4079
|
+
#endif
|
|
4080
|
+
}
|
|
4081
|
+
return vout;
|
|
4082
|
+
}
|
|
4083
|
+
|
|
4084
|
+
/* F.5 bench helper variant: sort_copy via SLAB macro + manual
|
|
4085
|
+
gather/scatter (= what hand-rolled sort_copy effectively does, but
|
|
4086
|
+
driven through the public CA_FOR_EACH_SLAB_INOUT macro instead of
|
|
4087
|
+
the raw next_slab_axes API). Lets us isolate macro overhead from
|
|
4088
|
+
gather/scatter overhead. */
|
|
4089
|
+
static VALUE
|
|
4090
|
+
rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
4091
|
+
{
|
|
4092
|
+
CArray *src, *out;
|
|
4093
|
+
ca_iter_state st_in, st_out;
|
|
4094
|
+
char *pi, *po;
|
|
4095
|
+
boolean8_t *mi, *mo;
|
|
4096
|
+
int axis = NUM2INT(vaxis);
|
|
4097
|
+
VALUE vout;
|
|
4098
|
+
int8_t slab_axes[1];
|
|
4099
|
+
|
|
4100
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
4101
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
4102
|
+
rb_raise(rb_eArgError, "caf_slab_bench_sort_copy_f64: requires float64");
|
|
4103
|
+
}
|
|
4104
|
+
vout = rb_ca_template(vsrc);
|
|
4105
|
+
TypedData_Get_Struct(vout, CArray, &carray_data_type, out);
|
|
4106
|
+
slab_axes[0] = (int8_t) axis;
|
|
4107
|
+
|
|
4108
|
+
ca_size_t fiber_n = src->dim[axis];
|
|
4109
|
+
ca_size_t bytes = sizeof(double);
|
|
4110
|
+
char *buf = (char *) xmalloc(fiber_n * bytes);
|
|
4111
|
+
|
|
4112
|
+
CA_FOR_EACH_SLAB_INOUT(st_in, st_out, src, out,
|
|
4113
|
+
slab_axes, 1,
|
|
4114
|
+
pi, po, mi, mo) {
|
|
4115
|
+
ca_size_t n = st_in.slab_dims[0];
|
|
4116
|
+
ca_size_t is = st_in.slab_strides[0];
|
|
4117
|
+
ca_size_t os = st_out.slab_strides[0];
|
|
4118
|
+
/* Author-side gather (= what hand-rolled sort_copy does). */
|
|
4119
|
+
if ( is == (ca_size_t) bytes ) {
|
|
4120
|
+
memcpy(buf, pi, n * bytes);
|
|
4121
|
+
} else {
|
|
4122
|
+
for ( ca_size_t k = 0; k < n; k++ ) {
|
|
4123
|
+
memcpy(buf + k * bytes, pi + k * is, bytes);
|
|
4124
|
+
}
|
|
4125
|
+
}
|
|
4126
|
+
#ifdef HAVE_MERGESORT
|
|
4127
|
+
if ( mergesort(buf, n, bytes, caf_fiber_bench_cmp_double) != 0 ) {
|
|
4128
|
+
qsort(buf, n, bytes, caf_fiber_bench_cmp_double);
|
|
4129
|
+
}
|
|
4130
|
+
#else
|
|
4131
|
+
qsort(buf, n, bytes, caf_fiber_bench_cmp_double);
|
|
4132
|
+
#endif
|
|
4133
|
+
/* Author-side scatter. */
|
|
4134
|
+
if ( os == (ca_size_t) bytes ) {
|
|
4135
|
+
memcpy(po, buf, n * bytes);
|
|
4136
|
+
} else {
|
|
4137
|
+
for ( ca_size_t k = 0; k < n; k++ ) {
|
|
4138
|
+
memcpy(po + k * os, buf + k * bytes, bytes);
|
|
4139
|
+
}
|
|
4140
|
+
}
|
|
4141
|
+
}
|
|
4142
|
+
xfree(buf);
|
|
4143
|
+
return vout;
|
|
4144
|
+
}
|
|
4145
|
+
|
|
4146
|
+
/* F.5 follow-up bench: per-fiber fused xfer_stride direct into a contig
|
|
4147
|
+
scratch, skipping the kernel_iterator SRC_ATTACH whole-view materialise.
|
|
4148
|
+
Tests user's hypothesis: for transform views (CAFake/CAByteSwap etc.),
|
|
4149
|
+
does bypassing the whole-view scratch_ptr and calling ca_xfer_stride
|
|
4150
|
+
per-fiber (= fused 1-pass per fiber) beat the current FIBER path?
|
|
4151
|
+
|
|
4152
|
+
Scope: float64 view, sum along a single axis. No kernel_iterator
|
|
4153
|
+
state is created -- this directly walks fiber regions via outer
|
|
4154
|
+
odometer + per-fiber ca_xfer_stride call. */
|
|
4155
|
+
static VALUE
|
|
4156
|
+
rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
4157
|
+
{
|
|
4158
|
+
CArray *src;
|
|
4159
|
+
int axis;
|
|
4160
|
+
double total = 0.0;
|
|
4161
|
+
ca_size_t starts[CA_RANK_MAX], counts[CA_RANK_MAX], strides[CA_RANK_MAX];
|
|
4162
|
+
ca_size_t outer_idx[CA_RANK_MAX];
|
|
4163
|
+
int8_t nd, k;
|
|
4164
|
+
|
|
4165
|
+
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
4166
|
+
if ( src->data_type != CA_FLOAT64 ) {
|
|
4167
|
+
rb_raise(rb_eArgError,
|
|
4168
|
+
"caf_bench_per_fiber_xfer_sum_f64: requires float64 view");
|
|
4169
|
+
}
|
|
4170
|
+
axis = NUM2INT(vaxis);
|
|
4171
|
+
nd = src->ndim;
|
|
4172
|
+
if ( axis < 0 || axis >= nd ) {
|
|
4173
|
+
rb_raise(rb_eArgError, "axis out of range");
|
|
4174
|
+
}
|
|
4175
|
+
|
|
4176
|
+
/* Fiber length = src->dim[axis], fiber count = product of other dims. */
|
|
4177
|
+
ca_size_t fiber_n = src->dim[axis];
|
|
4178
|
+
ca_size_t total_fibers = 1;
|
|
4179
|
+
for ( k = 0; k < nd; k++ ) {
|
|
4180
|
+
if ( k != axis ) total_fibers *= src->dim[k];
|
|
4181
|
+
outer_idx[k] = 0;
|
|
4182
|
+
}
|
|
4183
|
+
|
|
4184
|
+
/* Per-axis native byte strides (row-major) for use in ca_xfer_stride. */
|
|
4185
|
+
ca_size_t native[CA_RANK_MAX];
|
|
4186
|
+
{
|
|
4187
|
+
ca_size_t s = src->bytes;
|
|
4188
|
+
for ( k = nd - 1; k >= 0; k-- ) { native[k] = s; s *= src->dim[k]; }
|
|
4189
|
+
}
|
|
4190
|
+
|
|
4191
|
+
/* Fiber-sized contig scratch. */
|
|
4192
|
+
char *buf = (char *) xmalloc(fiber_n * src->bytes);
|
|
4193
|
+
|
|
4194
|
+
for ( ca_size_t f = 0; f < total_fibers; f++ ) {
|
|
4195
|
+
/* Build region: counts = 1 on all non-axis, fiber_n on axis;
|
|
4196
|
+
starts from outer_idx (0 on axis); strides = native bytes. */
|
|
4197
|
+
for ( k = 0; k < nd; k++ ) {
|
|
4198
|
+
starts[k] = (k == axis) ? 0 : outer_idx[k];
|
|
4199
|
+
counts[k] = (k == axis) ? fiber_n : 1;
|
|
4200
|
+
strides[k] = native[k];
|
|
4201
|
+
}
|
|
4202
|
+
|
|
4203
|
+
/* ONE fused xfer_stride per fiber: for CAFake/CAByteSwap this
|
|
4204
|
+
routes to ca_xfer_stride_transform_fused (= 1-pass cast direct
|
|
4205
|
+
from parent to buf, no intermediate whole-view scratch). */
|
|
4206
|
+
ca_xfer_stride(src, starts, counts, strides, buf, CA_XFER_GET);
|
|
4207
|
+
|
|
4208
|
+
/* Sum the fiber. */
|
|
4209
|
+
double *p = (double *) buf;
|
|
4210
|
+
for ( ca_size_t i = 0; i < fiber_n; i++ ) total += p[i];
|
|
4211
|
+
|
|
4212
|
+
/* Advance outer_idx row-major over non-axis dims. */
|
|
4213
|
+
for ( k = nd - 1; k >= 0; k-- ) {
|
|
4214
|
+
if ( k == axis ) continue;
|
|
4215
|
+
if ( ++outer_idx[k] < src->dim[k] ) break;
|
|
4216
|
+
outer_idx[k] = 0;
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
|
|
4220
|
+
xfree(buf);
|
|
4221
|
+
return rb_float_new(total);
|
|
4222
|
+
}
|
|
4223
|
+
|
|
4224
|
+
#endif /* CARRAY_DEV_BUILD — end smoke surface fence */
|
|
4225
|
+
|
|
4226
|
+
void
|
|
4227
|
+
Init_ca_kernel_iterator (void)
|
|
4228
|
+
{
|
|
4229
|
+
#ifdef CARRAY_DEV_BUILD
|
|
4230
|
+
/* ==== smoke surface registrations (dev-only, stripped in release) ====
|
|
4231
|
+
* See PROPOSAL_SMOKE_DEV_BUILD_GATE.md. All `t1_smoke_*`, `caf_smoke_*`,
|
|
4232
|
+
* `caf_fiber_smoke_*`, `caf_*_bench_*`, helper smokes, and `T1_*` test
|
|
4233
|
+
* constants are gated here together — none of them are consumed by
|
|
4234
|
+
* production code (lib/ / other ext/), only by spec_ai regression pins. */
|
|
4235
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke",
|
|
4236
|
+
rb_t1_smoke, 1);
|
|
4237
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_strided",
|
|
4238
|
+
rb_t1_smoke_strided, 1);
|
|
4239
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_sum_strided_f64",
|
|
4240
|
+
rb_t1_smoke_sum_strided_f64, 1);
|
|
4241
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_write_fill_f64",
|
|
4242
|
+
rb_t1_smoke_write_fill_f64, 2);
|
|
4243
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_write_fill_strided_f64",
|
|
4244
|
+
rb_t1_smoke_write_fill_strided_f64, 2);
|
|
4245
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_write_partial_raise_f64",
|
|
4246
|
+
rb_t1_smoke_write_partial_raise_f64, 3);
|
|
4247
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_sort_row_f64",
|
|
4248
|
+
rb_t1_smoke_sort_row_f64, 1);
|
|
4249
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_attach",
|
|
4250
|
+
rb_t1_smoke_attach, 1);
|
|
4251
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_attach_strided",
|
|
4252
|
+
rb_t1_smoke_attach_strided, 1);
|
|
4253
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_with_mask",
|
|
4254
|
+
rb_t1_smoke_with_mask, 1);
|
|
4255
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_init_rc",
|
|
4256
|
+
rb_t1_smoke_init_rc, 2);
|
|
4257
|
+
/* Phase A capstone: CA_SLAB_AXES smoke (variadic axes) */
|
|
4258
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_sum_axes_f64",
|
|
4259
|
+
rb_t1_smoke_sum_axes_f64, -1);
|
|
4260
|
+
/* Phase A.3: rb_ca_new_reduced helper smoke */
|
|
4261
|
+
rb_define_singleton_method(rb_cCArray, "t1_test_new_reduced",
|
|
4262
|
+
rb_t1_test_new_reduced, -1);
|
|
4263
|
+
/* Phase B.3: rb_ca_parse_reduce_axes helper smoke */
|
|
4264
|
+
rb_define_singleton_method(rb_cCArray, "t1_test_parse_reduce_axes",
|
|
4265
|
+
rb_t1_test_parse_reduce_axes, -1);
|
|
4266
|
+
|
|
4267
|
+
/* API harmonisation A.1: rb_ca_parse_reduce_axes_kw helper smoke */
|
|
4268
|
+
rb_define_singleton_method(rb_cCArray, "test_parse_reduce_axes_kw",
|
|
4269
|
+
rb_test_parse_reduce_axes_kw, -1);
|
|
4270
|
+
/* CAReduce-specific (step 9.3): no public Ruby API */
|
|
4271
|
+
rb_define_singleton_method(rb_cCArray, "t1_make_reduce",
|
|
4272
|
+
rb_t1_make_reduce, 3);
|
|
4273
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_read",
|
|
4274
|
+
rb_t1_smoke_reduce_read, 3);
|
|
4275
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_write_broadcast",
|
|
4276
|
+
rb_t1_smoke_reduce_write_broadcast, 4);
|
|
4277
|
+
|
|
4278
|
+
/* M.6 CARemap kernel_iterator smokes (test-only). */
|
|
4279
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read",
|
|
4280
|
+
rb_t1_smoke_remap_read, 2);
|
|
4281
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read_strided",
|
|
4282
|
+
rb_t1_smoke_remap_read_strided, 2);
|
|
4283
|
+
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_write_fill_f64",
|
|
4284
|
+
rb_t1_smoke_remap_write_fill_f64, 3);
|
|
4285
|
+
/* Phase C C.3 (2026-05-27): block macro smokes */
|
|
4286
|
+
rb_define_singleton_method(rb_cCArray, "caf_smoke_sum_f64",
|
|
4287
|
+
rb_caf_smoke_sum_f64, -1);
|
|
4288
|
+
rb_define_singleton_method(rb_cCArray, "caf_smoke_double_f64",
|
|
4289
|
+
rb_caf_smoke_double_f64, 1);
|
|
4290
|
+
/* PROPOSAL_FIBER_DELIVERY F.2: catalog macro smokes */
|
|
4291
|
+
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_sum_f64",
|
|
4292
|
+
rb_caf_fiber_smoke_sum_f64, 2);
|
|
4293
|
+
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_double_f64",
|
|
4294
|
+
rb_caf_fiber_smoke_double_f64, 2);
|
|
4295
|
+
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_unmasked_sum_f64",
|
|
4296
|
+
rb_caf_fiber_smoke_unmasked_sum_f64, 2);
|
|
4297
|
+
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_zero_masked_f64",
|
|
4298
|
+
rb_caf_fiber_smoke_zero_masked_f64, 2);
|
|
4299
|
+
/* F.5 bench helper */
|
|
4300
|
+
rb_define_singleton_method(rb_cCArray, "caf_fiber_bench_sort_copy_f64",
|
|
4301
|
+
rb_caf_fiber_bench_sort_copy_f64, 2);
|
|
4302
|
+
rb_define_singleton_method(rb_cCArray, "caf_slab_bench_sort_copy_f64",
|
|
4303
|
+
rb_caf_slab_bench_sort_copy_f64, 2);
|
|
4304
|
+
rb_define_singleton_method(rb_cCArray, "caf_bench_per_fiber_xfer_sum_f64",
|
|
4305
|
+
rb_caf_bench_per_fiber_xfer_sum_f64, 2);
|
|
4306
|
+
|
|
4307
|
+
rb_define_const(rb_cCArray, "T1_ITER_OK", INT2NUM(CA_ITER_OK));
|
|
4308
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_NOT_CHEAP", INT2NUM(CA_ITER_ERR_NOT_CHEAP));
|
|
4309
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_POLICY", INT2NUM(CA_ITER_ERR_POLICY));
|
|
4310
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_FLAGS", INT2NUM(CA_ITER_ERR_FLAGS));
|
|
4311
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_READONLY", INT2NUM(CA_ITER_ERR_READONLY));
|
|
4312
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_MASK", INT2NUM(CA_ITER_ERR_MASK));
|
|
4313
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_MASK_NOT_ALLOWED", INT2NUM(CA_ITER_ERR_MASK_NOT_ALLOWED));
|
|
4314
|
+
rb_define_const(rb_cCArray, "T1_KERNEL_NO_MASK", INT2NUM(CA_KERNEL_NO_MASK));
|
|
4315
|
+
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_NONE", INT2NUM(CA_ITER_ALIAS_NONE));
|
|
4316
|
+
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_CONTIG", INT2NUM(CA_ITER_ALIAS_CONTIG));
|
|
4317
|
+
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_STRIDED", INT2NUM(CA_ITER_ALIAS_STRIDED));
|
|
4318
|
+
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_ATTACH", INT2NUM(CA_ITER_ALIAS_ATTACH));
|
|
4319
|
+
rb_define_const(rb_cCArray, "T1_ITER_ERR_UNBOUND_SHAPE", INT2NUM(CA_ITER_ERR_UNBOUND_SHAPE));
|
|
4320
|
+
#endif /* CARRAY_DEV_BUILD */
|
|
4321
|
+
}
|