carray 3.0.2 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +568 -0
- data/README.md +6 -6
- data/carray.gemspec +1 -1
- data/ext/ca_axis_descriptor.h +27 -24
- data/ext/ca_axis_dispatch.c +220 -0
- data/ext/ca_axis_group.c +201 -123
- data/ext/ca_binop_dispatch.c +46 -52
- data/ext/ca_binop_dispatch.h +1 -1
- data/ext/ca_categorical_iterator.c +249 -183
- data/ext/ca_compare.c +7 -12
- data/ext/ca_for_buffer.h +21 -68
- data/ext/ca_group_iter.c +38 -23
- data/ext/ca_kernel_iterator.c +1765 -2076
- data/ext/ca_kernel_iterator.h +584 -566
- data/ext/ca_monop_dispatch.c +86 -63
- data/ext/ca_monop_dispatch.h +5 -0
- data/ext/ca_obj_array.c +223 -56
- data/ext/ca_obj_bincmp.c +172 -138
- data/ext/ca_obj_binop.c +116 -230
- data/ext/ca_obj_bitarray.c +40 -26
- data/ext/ca_obj_bitfield.c +100 -68
- data/ext/ca_obj_block.c +19 -9
- data/ext/ca_obj_byte_swap.c +45 -46
- data/ext/ca_obj_const_string.c +269 -44
- data/ext/ca_obj_face.c +612 -89
- data/ext/ca_obj_face.h +111 -47
- data/ext/ca_obj_fake.c +36 -79
- data/ext/ca_obj_farray.c +7 -3
- data/ext/ca_obj_field.c +33 -8
- data/ext/ca_obj_fixlen_string.c +23 -24
- data/ext/ca_obj_grid.c +541 -210
- data/ext/ca_obj_meld.c +331 -111
- data/ext/ca_obj_moncmp.c +80 -65
- data/ext/ca_obj_monop.c +177 -149
- data/ext/ca_obj_object.c +664 -248
- data/ext/ca_obj_record.c +63 -23
- data/ext/ca_obj_reduce.c +18 -16
- data/ext/ca_obj_refer.c +140 -58
- data/ext/ca_obj_remap.c +85 -24
- data/ext/ca_obj_repeat.c +24 -9
- data/ext/ca_obj_roll.c +121 -16
- data/ext/ca_obj_select.c +61 -58
- data/ext/ca_obj_select_axis.c +73 -107
- data/ext/ca_obj_shift.c +32 -102
- data/ext/ca_obj_stack.c +372 -122
- data/ext/ca_obj_stride.c +348 -140
- data/ext/ca_obj_string.c +88 -22
- data/ext/ca_obj_tile.c +38 -21
- data/ext/ca_obj_time.c +45 -31
- data/ext/ca_obj_timedelta.c +43 -29
- data/ext/ca_obj_transpose.c +17 -6
- data/ext/ca_obj_triop.c +125 -141
- data/ext/ca_obj_window.c +330 -44
- data/ext/ca_op_ipower.c +24 -11
- data/ext/ca_op_powi.h +57 -30
- data/ext/ca_pow5_table.h +656 -0
- data/ext/ca_sort_kernels.h +70 -0
- data/ext/ca_sweep_engine.c +104 -81
- data/ext/ca_sweep_engine.h +7 -20
- data/ext/ca_triop_dispatch.c +4 -1
- data/ext/ca_triop_dispatch.h +5 -2
- data/ext/caframe_csv_reader.c +542 -0
- data/ext/carray.h +241 -92
- data/ext/carray_access.c +608 -276
- data/ext/carray_address_basis.c +494 -97
- data/ext/carray_attribute.c +129 -4
- data/ext/carray_bincount.c +39 -7
- data/ext/carray_broadcast.c +56 -19
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_call_cfunc.c +108 -28
- data/ext/carray_cast.c +1576 -223
- data/ext/carray_cast_func.rb +77 -24
- data/ext/carray_class.c +4 -3
- data/ext/carray_conversion.c +208 -75
- data/ext/carray_copy.c +82 -24
- data/ext/carray_core.c +646 -162
- data/ext/carray_count.c +171 -88
- data/ext/carray_element.c +94 -60
- data/ext/carray_factorize.c +670 -324
- data/ext/carray_generate.c +260 -83
- data/ext/carray_histogram.c +209 -106
- data/ext/carray_hold.c +14 -16
- data/ext/carray_index_classifier.c +422 -243
- data/ext/carray_internal.h +692 -3
- data/ext/carray_kernels_bincmp.c +7 -0
- data/ext/carray_kernels_binop.c +487 -120
- data/ext/carray_kernels_init.c +653 -50
- data/ext/carray_kernels_moncmp.c +15 -8
- data/ext/carray_kernels_monop.c +99 -43
- data/ext/carray_kernels_reduce_aggregate.c +3925 -6112
- data/ext/carray_kernels_reduce_boolean.c +112 -36
- data/ext/carray_kernels_reduce_cumulative.c +2127 -6903
- data/ext/carray_kernels_reduce_extreme.c +3032 -2892
- data/ext/carray_kernels_reduce_variance.c +439 -500
- data/ext/carray_kernels_scan.c +1097 -488
- data/ext/carray_kernels_search.c +3787 -2176
- data/ext/carray_kernels_sort.c +1200 -920
- data/ext/carray_kernels_triop.c +479 -7
- data/ext/carray_key_windows.c +216 -0
- data/ext/carray_lazy.c +967 -153
- data/ext/carray_loop.c +81 -65
- data/ext/carray_mask.c +910 -324
- data/ext/carray_math_kernel.h +1 -0
- data/ext/carray_median_percentile.c +299 -100
- data/ext/carray_memory_view.c +872 -758
- data/ext/carray_operator.c +1171 -364
- data/ext/carray_order.c +505 -133
- data/ext/carray_partition.c +112 -65
- data/ext/carray_random.c +327 -239
- data/ext/carray_round.c +273 -0
- data/ext/carray_scatter.c +346 -78
- data/ext/carray_segment.c +257 -0
- data/ext/carray_slab.c +249 -86
- data/ext/carray_sort.c +224 -227
- data/ext/carray_sort_kernel.c +8 -1
- data/ext/carray_struct.c +118 -39
- data/ext/carray_test.c +193 -69
- data/ext/carray_time_text.c +1182 -0
- data/ext/carray_undef.c +4 -0
- data/ext/carray_utils.c +579 -113
- data/ext/extconf.rb +27 -2
- data/ext/mk_call_cfunc.rb +19 -4
- data/ext/mk_pow5_table.rb +35 -0
- data/ext/mkkernel.rb +1202 -1014
- data/ext/ruby_carray.c +17 -5
- data/ext/version.h +4 -4
- data/ext/version.rb +4 -2
- data/lib/carray/attribute.rb +160 -17
- data/lib/carray/autoload_carray.rb +23 -27
- data/lib/carray/autoload_method_extension.rb +20 -5
- data/lib/carray/axis_group.rb +360 -63
- data/lib/carray/basics.rb +162 -28
- data/lib/carray/bincount_nd.rb +18 -2
- data/lib/carray/block_iterator.rb +231 -58
- data/lib/carray/boolean_reduce.rb +34 -42
- data/lib/carray/categorical.rb +242 -52
- data/lib/carray/categorical_iterator.rb +437 -827
- data/lib/carray/complex.rb +41 -34
- data/lib/carray/conditional.rb +150 -14
- data/lib/carray/const_string.rb +59 -4
- data/lib/carray/construct.rb +15 -11
- data/lib/carray/data_type_extension.rb +1 -0
- data/lib/carray/distance_limit.rb +77 -0
- data/lib/carray/fixlen_string.rb +7 -1
- data/lib/carray/frame/column_verbs.rb +70 -0
- data/lib/carray/frame/concat.rb +157 -6
- data/lib/carray/frame/convert.rb +82 -7
- data/lib/carray/frame/csv_parser.rb +688 -87
- data/lib/carray/frame/describe.rb +80 -0
- data/lib/carray/frame/frame.rb +369 -45
- data/lib/carray/frame/group.rb +631 -90
- data/lib/carray/frame/io.rb +506 -79
- data/lib/carray/frame/join.rb +411 -62
- data/lib/carray/frame/records.rb +116 -19
- data/lib/carray/frame/reshape.rb +455 -0
- data/lib/carray/frame/sort.rb +30 -18
- data/lib/carray/frame/verbs.rb +868 -61
- data/lib/carray/frame/windows.rb +117 -0
- data/lib/carray/frame.rb +4 -0
- data/lib/carray/fuse_source.rb +118 -2
- data/lib/carray/fusion.rb +257 -42
- data/lib/carray/histogram.rb +6 -1
- data/lib/carray/inspect.rb +54 -13
- data/lib/carray/iterator.rb +99 -9
- data/lib/carray/key_windows.rb +620 -0
- data/lib/carray/lazy.rb +216 -193
- data/lib/carray/mask_gap_fill.rb +112 -92
- data/lib/carray/math.rb +32 -19
- data/lib/carray/meld_reduce.rb +52 -21
- data/lib/carray/methods/align_addr.rb +8 -4
- data/lib/carray/methods/bin.rb +26 -18
- data/lib/carray/methods/bincount.rb +18 -2
- data/lib/carray/methods/bit_string.rb +15 -4
- data/lib/carray/methods/choose.rb +16 -3
- data/lib/carray/methods/composition.rb +18 -10
- data/lib/carray/methods/factorize.rb +2 -2
- data/lib/carray/methods/gather_nd.rb +25 -5
- data/lib/carray/methods/insert_block.rb +9 -14
- data/lib/carray/methods/is_between.rb +59 -0
- data/lib/carray/methods/is_in.rb +140 -10
- data/lib/carray/methods/join.rb +2 -7
- data/lib/carray/methods/lag.rb +49 -0
- data/lib/carray/methods/locate_addr.rb +182 -45
- data/lib/carray/methods/mode.rb +18 -1
- data/lib/carray/methods/nunique.rb +11 -6
- data/lib/carray/methods/repeat.rb +1 -20
- data/lib/carray/methods/resize.rb +21 -23
- data/lib/carray/methods/snap.rb +34 -12
- data/lib/carray/methods/unique.rb +11 -2
- data/lib/carray/methods/value_counts.rb +4 -11
- data/lib/carray/record_order.rb +208 -0
- data/lib/carray/rng.rb +3 -0
- data/lib/carray/runtime.rb +65 -17
- data/lib/carray/segment_iterator.rb +1221 -0
- data/lib/carray/serialize.rb +214 -30
- data/lib/carray/slab_iterator.rb +101 -27
- data/lib/carray/stack.rb +10 -4
- data/lib/carray/string.rb +105 -5
- data/lib/carray/string_operation_extension.rb +12 -4
- data/lib/carray/struct.rb +137 -58
- data/lib/carray/struct_builder.rb +124 -17
- data/lib/carray/time.rb +1222 -186
- data/lib/carray/window_iterator.rb +242 -81
- data/lib/carray.rb +9 -2
- data/yard-stubs/ca_obj_array.rb +4 -4
- data/yard-stubs/ca_obj_bitarray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +23 -12
- data/yard-stubs/ca_obj_fake.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +5 -1
- data/yard-stubs/ca_obj_grid.rb +21 -10
- data/yard-stubs/ca_obj_refer.rb +5 -4
- data/yard-stubs/ca_obj_roll.rb +2 -3
- data/yard-stubs/ca_obj_shift.rb +5 -1
- data/yard-stubs/ca_obj_stride.rb +10 -7
- data/yard-stubs/ca_obj_tile.rb +1 -1
- data/yard-stubs/ca_obj_transpose.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +9 -5
- data/yard-stubs/carray_access.rb +11 -5
- data/yard-stubs/carray_attribute.rb +0 -5
- data/yard-stubs/carray_broadcast.rb +1 -1
- data/yard-stubs/carray_cast.rb +88 -43
- data/yard-stubs/carray_conversion.rb +15 -2
- data/yard-stubs/carray_copy.rb +12 -5
- data/yard-stubs/carray_core.rb +3 -3
- data/yard-stubs/carray_count.rb +9 -0
- data/yard-stubs/carray_element.rb +10 -4
- data/yard-stubs/carray_generate.rb +21 -1
- data/yard-stubs/carray_lazy.rb +5 -1
- data/yard-stubs/carray_loop.rb +4 -1
- data/yard-stubs/carray_mask.rb +60 -12
- data/yard-stubs/carray_math.rb +52 -14
- data/yard-stubs/carray_median_percentile.rb +15 -8
- data/yard-stubs/carray_memory_view.rb +40 -23
- data/yard-stubs/carray_order.rb +61 -29
- data/yard-stubs/carray_random.rb +13 -4
- data/yard-stubs/carray_scatter.rb +21 -9
- data/yard-stubs/carray_segment.rb +72 -0
- data/yard-stubs/carray_slab.rb +10 -2
- data/yard-stubs/carray_sort.rb +49 -8
- data/yard-stubs/carray_test.rb +15 -3
- data/yard-stubs/carray_utils.rb +0 -26
- metadata +19 -5
- data/ext/ca_for_each_element.h +0 -239
- data/ext/carray_kernels_map.c +0 -3467
- data/lib/carray/arrow_tensor.rb +0 -401
- data/lib/carray/table.rb +0 -108
data/ext/ca_kernel_iterator.h
CHANGED
|
@@ -1,23 +1,46 @@
|
|
|
1
1
|
/* ---------------------------------------------------------------------------
|
|
2
2
|
|
|
3
|
-
ca_kernel_iterator.h
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
ca_kernel_iterator.h -- the kernel iterator, for C extension authors
|
|
4
|
+
|
|
5
|
+
A kernel written against this header receives the cells of any CArray
|
|
6
|
+
-- an entity, a view of a view, a selection, a lazy expression -- one
|
|
7
|
+
slab at a time, with the mask beside them, and on a write the cells it
|
|
8
|
+
changes reach the array. How the cells are delivered is the engine's
|
|
9
|
+
business (ca_kernel_iterator.c); the kernel only names the axes it
|
|
10
|
+
wants and walks what it is given.
|
|
11
|
+
|
|
12
|
+
Where to start:
|
|
13
|
+
- guides/devel/11_kernel_iterator.md: the surface and its contract,
|
|
14
|
+
with worked kernels
|
|
15
|
+
- docs/authoring/HOW_TO_WRITE_KERNEL.md: the reference, and which
|
|
16
|
+
macro to pick for a reduction, a map, a scan or a whole-array walk
|
|
17
|
+
- the block macros below (CA_FOR_EACH_SLAB / _FIBER / _WHOLE) are
|
|
18
|
+
the usual way in; the raw entry points (ca_iter_state_init_l1 /
|
|
19
|
+
init_l2 ...) are for kernels that handle the return codes
|
|
20
|
+
themselves
|
|
21
|
+
|
|
22
|
+
Freeze contract (3.0 onward). The surface is split
|
|
8
23
|
so the engine can be re-implemented across 3.x (the surface stays fixed)
|
|
9
|
-
without breaking ext-gem kernels.
|
|
10
|
-
|
|
24
|
+
without breaking ext-gem kernels. docs/authoring/HOW_TO_WRITE_KERNEL.md
|
|
25
|
+
states the contract in prose; utils/check_kernel_surface_freeze.rb
|
|
11
26
|
(rake kernel_surface_check) is the mechanical pin.
|
|
12
27
|
|
|
13
28
|
FROZEN (do NOT rename / re-arity / change semantics; additions only):
|
|
14
|
-
- the author macros
|
|
15
|
-
|
|
16
|
-
|
|
29
|
+
- the author macros: the CA_FOR_EACH_SLAB / _FIBER / _WHOLE
|
|
30
|
+
families (including _FIBER_PAIR / _FIBER_PAIR_MASKED), the
|
|
31
|
+
CA_SLAB_REDUCE_* / _MAP_* / _SCAN_* suites except the generator-
|
|
32
|
+
only macros listed under INTERNAL, CA_L2_FOR_EACH, and the mask
|
|
33
|
+
helper CA_FOR_EACH_UNMASKED
|
|
17
34
|
- the raw-API entry points the macros expand to: ca_iter_state_
|
|
18
|
-
init_l2 /
|
|
35
|
+
init_l1 / next_slab / init_l2 / init_l2_paired / next_slab_axes /
|
|
36
|
+
sync_slab / finish / close, ca_iter_check_init,
|
|
37
|
+
ca_iter_check_same_shape
|
|
38
|
+
- the author entry points beside them: ca_iter_ensure (a body that
|
|
39
|
+
can raise) and ca_iter_register_source_kind (an externally
|
|
40
|
+
installed view class)
|
|
19
41
|
- the enum/status tokens authors write literally: CA_SLAB_AXES,
|
|
20
|
-
CA_KERNEL_WRITE, CA_KERNEL_NO_MASK, CA_ITER_OK,
|
|
42
|
+
CA_SLAB_WHOLE, CA_KERNEL_WRITE, CA_KERNEL_NO_MASK, CA_ITER_OK,
|
|
43
|
+
CA_ITER_ERR_*
|
|
21
44
|
- the slab-delivery representation = the ca_iter_state fields a
|
|
22
45
|
kernel reads (marked "FROZEN author contract" at the struct):
|
|
23
46
|
slab_ndim / slab_dims / slab_strides / slab_mask_strides /
|
|
@@ -27,9 +50,15 @@
|
|
|
27
50
|
|
|
28
51
|
INTERNAL (free to refactor across 3.x — engine implementation):
|
|
29
52
|
- the state-machine functions' bodies + init_l1 / next_slab /
|
|
30
|
-
|
|
53
|
+
ca_iter_can_alias
|
|
54
|
+
- the macros only the kernel generator (ext/mkkernel.rb) emits:
|
|
55
|
+
CA_SLAB_REDUCE_SUM8_EX / MIN8_EX / MAX8_EX / STAR8_EX (8-way
|
|
56
|
+
accumulator split; they take EXPR(x) rather than exposing v and
|
|
57
|
+
idx), CA_SLAB_REDUCE_T_MIN_EX / _MAX_EX / _STAR_EX (what it emits
|
|
58
|
+
when a :min / :max / :prod kernel does not take the 8-way form),
|
|
59
|
+
CA_SLAB_SCAN_T_GATED, and CA_SLAB_REDUCE_T_VAR_EX (no caller)
|
|
31
60
|
- alias_mode / src_kind routing (CA_ITER_ALIAS_* / CA_ITER_SRC_* /
|
|
32
|
-
CA_KERNEL_FIBER_CONTIG / CA_SLAB_FREE
|
|
61
|
+
CA_KERNEL_FIBER_CONTIG / CA_SLAB_FREE)
|
|
33
62
|
- every ca_iter_state field NOT marked frozen (scratch / stack /
|
|
34
63
|
fiber / composed / descriptor bookkeeping) + physical layout
|
|
35
64
|
|
|
@@ -37,24 +66,9 @@
|
|
|
37
66
|
_EX macro variant + thin wrapper, new catalog macro) — that is how
|
|
38
67
|
the surface evolves within 3.x. Changing or removing a FROZEN name
|
|
39
68
|
is a 3.x breaking change: update the doc contract AND the guard in
|
|
40
|
-
the same commit.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
Step 1 scope (this commit):
|
|
45
|
-
- struct definition (subset used by step 1; chunk_pos / scratch /
|
|
46
|
-
descs / outer_idx fields are present but unused, reserved for
|
|
47
|
-
later steps per proposal §3.1)
|
|
48
|
-
- CA_SLAB_WHOLE policy only (single slab = whole array)
|
|
49
|
-
- alias path only: ca_attach_is_alias(src) must hold, else init
|
|
50
|
-
returns CA_ITER_ERR_NOT_CHEAP (renamed from ca_attach_is_cheap
|
|
51
|
-
in T1 step 9.4a)
|
|
52
|
-
- READ-only, no mask handling, no WRITE sync
|
|
53
|
-
|
|
54
|
-
Later steps will fill in scratch materialize (step 2), L2 strided
|
|
55
|
-
(step 3), WRITE + sync_slab (step 4), descriptor 6-view connection
|
|
56
|
-
via P3 ca_axis_dispatch_for_each_slab (step 5), mask + macros
|
|
57
|
-
(step 6), NO_MASK enforcement (step 7).
|
|
69
|
+
the same commit. The engine behind these names may be rewritten
|
|
70
|
+
within 3.x; this contract is what keeps such a rewrite invisible to
|
|
71
|
+
kernel authors.
|
|
58
72
|
|
|
59
73
|
--------------------------------------------------------------------------- */
|
|
60
74
|
|
|
@@ -64,174 +78,158 @@
|
|
|
64
78
|
#include "carray.h"
|
|
65
79
|
#include "ca_iter_substrate.h" /* ca_axis_desc_t, ca_op_prefix_axis_t */
|
|
66
80
|
|
|
67
|
-
/* ---- slab policy
|
|
81
|
+
/* ---- slab policy ------------------------------------------------------ */
|
|
68
82
|
typedef enum {
|
|
69
|
-
CA_SLAB_FREE = 0, /* engine-chosen chunk
|
|
70
|
-
|
|
71
|
-
|
|
83
|
+
CA_SLAB_FREE = 0, /* engine-chosen chunk. Not implemented:
|
|
84
|
+
init returns CA_ITER_ERR_POLICY. */
|
|
85
|
+
CA_SLAB_AXES = 1, /* user-pinned axes inside the slab (init_l2) */
|
|
86
|
+
CA_SLAB_WHOLE = 2 /* whole array in one slab */
|
|
72
87
|
} ca_slab_policy_t;
|
|
73
88
|
|
|
74
|
-
/* ---- kernel flags
|
|
89
|
+
/* ---- kernel flags ------------------------------------------------------ */
|
|
75
90
|
#define CA_KERNEL_READ 0x0
|
|
76
|
-
#define CA_KERNEL_WRITE 0x1 /*
|
|
77
|
-
#define CA_KERNEL_NO_MASK 0x2 /*
|
|
78
|
-
|
|
79
|
-
#define
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
91
|
+
#define CA_KERNEL_WRITE 0x1 /* the kernel writes the slab back */
|
|
92
|
+
#define CA_KERNEL_NO_MASK 0x2 /* the kernel cannot take a mask:
|
|
93
|
+
a masked source is refused */
|
|
94
|
+
#define CA_KERNEL_CHUNK_HINT 0x4 /* reserved: init refuses it
|
|
95
|
+
with CA_ITER_ERR_FLAGS */
|
|
96
|
+
#define CA_KERNEL_FIBER_CONTIG 0x8 /* with policy=CA_SLAB_AXES and
|
|
97
|
+
naxes==1, next_slab_axes hands
|
|
98
|
+
out every fiber contiguous
|
|
99
|
+
(gathering a strided one into
|
|
84
100
|
fiber_data_scratch). Set by the
|
|
85
|
-
CA_FOR_EACH_FIBER macro family
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
/*
|
|
93
|
-
#define
|
|
94
|
-
|
|
95
|
-
#define
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
[
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
caches parent->ptr aliases +
|
|
154
|
-
uniform parent-native byte
|
|
155
|
-
strides. next_slab_axes does
|
|
156
|
-
K-fold direct memcpy gather from
|
|
101
|
+
CA_FOR_EACH_FIBER macro family.
|
|
102
|
+
Direct callers that walk the
|
|
103
|
+
slab with slab_strides[0]
|
|
104
|
+
themselves leave it clear. */
|
|
105
|
+
|
|
106
|
+
/* ---- alias mode: where the slabs are cut from (internal) --------------- */
|
|
107
|
+
#define CA_ITER_ALIAS_NONE 0 /* a scratch the walk gathered */
|
|
108
|
+
#define CA_ITER_ALIAS_CONTIG 1 /* the source's own memory */
|
|
109
|
+
#define CA_ITER_ALIAS_STRIDED 2 /* the root's or parent's memory,
|
|
110
|
+
through composed strides */
|
|
111
|
+
#define CA_ITER_ALIAS_ATTACH 3 /* Not set by the engine. A
|
|
112
|
+
SRC_ATTACH walk runs with
|
|
113
|
+
CA_ITER_ALIAS_NONE over an
|
|
114
|
+
iterator-owned scratch, filled with
|
|
115
|
+
ca_xfer_all and written back with
|
|
116
|
+
ki_scratch_push_changed (only the
|
|
117
|
+
changed cells). */
|
|
118
|
+
#define CA_ITER_ALIAS_PER_SLAB 4 /* a descriptor view read one slab at
|
|
119
|
+
a time: ca_axis_dispatch_gather
|
|
120
|
+
over a subset of the descriptors
|
|
121
|
+
with the outer axes pinned, into
|
|
122
|
+
one slab-sized scratch refilled
|
|
123
|
+
in next_slab_axes. Read walks
|
|
124
|
+
only; a write walk takes the
|
|
125
|
+
transfer walk instead. */
|
|
126
|
+
#define CA_ITER_ALIAS_PER_SLAB_HOIST 5 /* PER_SLAB when the innermost slab
|
|
127
|
+
axis is STRIDE and no axis is
|
|
128
|
+
SHIFT: the outer and
|
|
129
|
+
non-innermost slab offsets are
|
|
130
|
+
computed once per slab row and
|
|
131
|
+
the inner run is a memcpy. */
|
|
132
|
+
#define CA_ITER_ALIAS_PER_FIBER_FUSED 6 /* each fiber is fetched (and on a
|
|
133
|
+
write put back) with the view's
|
|
134
|
+
ca_xfer_stride, without
|
|
135
|
+
gathering the whole view: a
|
|
136
|
+
CAWindow / CAShift fiber, or a
|
|
137
|
+
read of the innermost fiber of
|
|
138
|
+
a conversion view. Chosen by
|
|
139
|
+
ca_iter_should_per_fiber_fused.
|
|
140
|
+
The fiber is handed out from
|
|
141
|
+
fiber_data_scratch, its mask
|
|
142
|
+
from fiber_mask_scratch. */
|
|
143
|
+
#define CA_ITER_ALIAS_STACK 7 /* a CAStack read with slab_axes ==
|
|
144
|
+
[k_axis] (the stack axis alone,
|
|
145
|
+
e.g. view.mean(axis: k_axis)).
|
|
146
|
+
init_l2 attaches the K parents
|
|
147
|
+
and caches their pointers and
|
|
148
|
+
their common native strides;
|
|
149
|
+
next_slab_axes hands out each
|
|
150
|
+
fiber's K cells from a tile cache
|
|
151
|
+
(K * tile * bytes, tile 8..64
|
|
152
|
+
fibers) refilled from the parents'
|
|
153
|
+
memory, without ca_xfer_stride.
|
|
154
|
+
Only for slab_axes == [k_axis]
|
|
155
|
+
and an unmasked stack; any other
|
|
156
|
+
stack walk takes STACK_OUTER_K or
|
|
157
|
+
the transfer walk. Delivery
|
|
158
|
+
through ca_xfer_stride was tried
|
|
159
|
+
and measured ~16x slower. */
|
|
160
|
+
#define CA_ITER_ALIAS_STACK_OUTER_K 8 /* CAStack source + CA_SLAB_AXES
|
|
161
|
+
whose slab leaves out the stack
|
|
162
|
+
axis (k_axis), which is then an
|
|
163
|
+
outer axis. Each slab lies inside
|
|
164
|
+
ONE parent, selected by
|
|
165
|
+
outer_idx[stack_k_outer_pos].
|
|
166
|
+
init_l2 attaches the K parents and
|
|
167
|
+
caches their ptrs and parent byte
|
|
168
|
+
strides; next_slab_axes reads
|
|
157
169
|
parents[k]->ptr + parent_off
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
(= slab footprint), not
|
|
165
|
-
K * parent.elements * bytes.
|
|
166
|
-
Scope-narrow to slab_axes == [0]:
|
|
167
|
-
arbitrary slab shapes require
|
|
168
|
-
per-cell index decode which
|
|
169
|
-
loses the inner contig fast
|
|
170
|
-
path; covers the demand-driving
|
|
171
|
-
case (= reduce along the stacked
|
|
172
|
-
K-axis). Mask-bearing CAStack
|
|
173
|
-
falls back to SRC_ATTACH whole-
|
|
174
|
-
view path (per-slab mask gather
|
|
175
|
-
= future extension). Rev1
|
|
176
|
-
explored xfer_stride-based
|
|
177
|
-
delivery (see PROPOSAL
|
|
178
|
-
rev2 §11) and was rejected for
|
|
179
|
-
~16x wall-clock regression. */
|
|
180
|
-
#define CA_ITER_ALIAS_PER_SLAB_HOIST 5 /* Phase C T3 specialised (B-1b,
|
|
181
|
-
C.1b): innermost slab axis is
|
|
182
|
-
STRIDE and no SHIFT axes
|
|
183
|
-
anywhere. Manual gather:
|
|
184
|
-
outer + non-innermost-slab axes
|
|
185
|
-
hoisted (computed once per slab
|
|
186
|
-
row), inner = pure STRIDE
|
|
187
|
-
linear memcpy (= no engine
|
|
188
|
-
per-cell switch, SIMD-friendly
|
|
189
|
-
contig run). Target: 1.5-1.8x
|
|
190
|
-
win vs (A) fallback for INDEX
|
|
191
|
-
slab with innermost STRIDE. */
|
|
170
|
+
directly. The parents' masks are
|
|
171
|
+
copied at init and the cursor
|
|
172
|
+
points into the copy. A fiber
|
|
173
|
+
that is not contiguous in its
|
|
174
|
+
parent is gathered (FIBER_CONTIG).
|
|
175
|
+
READ only. */
|
|
192
176
|
|
|
193
177
|
/* ---- error codes ----------------------------------------------------- */
|
|
194
178
|
#define CA_ITER_OK 0
|
|
195
|
-
#define CA_ITER_ERR_NOT_CHEAP 1 /*
|
|
196
|
-
|
|
197
|
-
|
|
179
|
+
#define CA_ITER_ERR_NOT_CHEAP 1 /* the classifier does not recognise
|
|
180
|
+
the source (a view with no attach
|
|
181
|
+
slot; any other external obj_type
|
|
182
|
+
is delivered by the transfer walk,
|
|
183
|
+
through its xfer slots) */
|
|
184
|
+
#define CA_ITER_ERR_POLICY 2 /* policy or axes refused: a policy
|
|
185
|
+
not implemented, axes NULL / out
|
|
186
|
+
of range / duplicated, AXES at
|
|
187
|
+
L1, descriptor rank != view rank */
|
|
188
|
+
#define CA_ITER_ERR_FLAGS 3 /* flag combination unsupported;
|
|
189
|
+
also returned for a NULL state
|
|
190
|
+
or source */
|
|
198
191
|
#define CA_ITER_ERR_READONLY 4 /* WRITE on readonly view (CARepeat etc.) */
|
|
199
|
-
#define CA_ITER_ERR_MASK 5 /*
|
|
200
|
-
|
|
201
|
-
/*
|
|
192
|
+
#define CA_ITER_ERR_MASK 5 /* reserved: no entry point returns
|
|
193
|
+
it (masked sources are accepted) */
|
|
194
|
+
#define CA_ITER_ERR_MASK_NOT_ALLOWED 6 /* NO_MASK flag set on a masked source */
|
|
195
|
+
/* ---- source kind (internal routing) ------------------------------------ */
|
|
202
196
|
#define CA_ITER_SRC_NONE 0
|
|
203
|
-
#define CA_ITER_SRC_CASTRIDE 1 /* entity / CAStride family
|
|
204
|
-
#define CA_ITER_SRC_DESCRIPTOR 2 /*
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
197
|
+
#define CA_ITER_SRC_CASTRIDE 1 /* entity / CAStride family */
|
|
198
|
+
#define CA_ITER_SRC_DESCRIPTOR 2 /* CASelectAxis / CAGrid / CASelect /
|
|
199
|
+
CAWindow / CAShift: per-axis
|
|
200
|
+
descriptors (describe_axes) */
|
|
201
|
+
#define CA_ITER_SRC_ATTACH 3 /* every other view the classifier
|
|
202
|
+
knows (conversion views, CATile,
|
|
203
|
+
CARoll, CAStack, CAMeld, CARemap,
|
|
204
|
+
CAObject, the lazy views) and any
|
|
205
|
+
registered external obj_type: the
|
|
206
|
+
view's own transfer gathers it */
|
|
208
207
|
#define CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE 4
|
|
209
|
-
/*
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
is yielded with inner_byte_stride). Only
|
|
215
|
-
emitted by ca_iter_route_source after
|
|
216
|
-
describe_axes inspection. */
|
|
208
|
+
/* a descriptor view whose innermost axis
|
|
209
|
+
is STRIDE kind, so an L2 walk can read
|
|
210
|
+
the parent's memory in place. Only
|
|
211
|
+
returned by ca_iter_route_source,
|
|
212
|
+
after describe_axes. */
|
|
217
213
|
|
|
218
214
|
/* ---- source-kind registration for externally installed obj_types -----
|
|
219
215
|
|
|
220
216
|
The classifier recognises the core's own view classes by comparing
|
|
221
217
|
their operation table against a list compiled into the engine. A view
|
|
222
218
|
class installed by a companion gem through ca_install_obj_type matches
|
|
223
|
-
nothing on that list
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
declares the routing here instead, once, in the class's Init:
|
|
219
|
+
nothing on that list; it is delivered as CA_ITER_SRC_ATTACH (gathered
|
|
220
|
+
through the view's own transfer), and so is one that registers here,
|
|
221
|
+
once, in the class's Init:
|
|
227
222
|
|
|
228
223
|
ca_iter_register_source_kind(CA_OBJ_MY_VIEW, CA_ITER_SRC_ATTACH);
|
|
229
224
|
|
|
225
|
+
Registering states the routing rather than changing it.
|
|
230
226
|
CA_ITER_SRC_ATTACH is the only kind that may be registered. It is the
|
|
231
|
-
one whose contract an external class can meet on its own:
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
227
|
+
one whose contract an external class can meet on its own: the engine
|
|
228
|
+
reads the cells with ca_xfer_all GET and, on a WRITE walk, sends back
|
|
229
|
+
the changed cells with ca_xfer_addrs (or ca_xfer_all PUT when every
|
|
230
|
+
cell changed), so the view's transfer slots are all it needs -- the
|
|
231
|
+
slots every view already implements. The view's attach / sync slots
|
|
232
|
+
are not called. The other kinds are not open to registration: SRC_CASTRIDE
|
|
235
233
|
asserts the struct *is* a CAStride (the engine reads its strides
|
|
236
234
|
directly, and a class that really is one is already classified by its
|
|
237
235
|
inherited operation table), and SRC_DESCRIPTOR requires a
|
|
@@ -245,37 +243,45 @@ typedef enum {
|
|
|
245
243
|
materialising path), and consults this table before the built-in list. */
|
|
246
244
|
void ca_iter_register_source_kind (int obj_type, uint8_t kind);
|
|
247
245
|
|
|
248
|
-
/* ---- iter state
|
|
249
|
-
/*
|
|
250
|
-
|
|
251
|
-
|
|
246
|
+
/* ---- iter state ------------------------------------------------------- */
|
|
247
|
+
/* The state lives on the ext author's stack, so its layout is fixed: a
|
|
248
|
+
field is never removed, and the ones no path uses any more (chunk_pos,
|
|
249
|
+
alias_stride) stay as zero-initialised slots. */
|
|
252
250
|
typedef struct {
|
|
253
251
|
/* --- inputs (fixed at init) --- */
|
|
254
252
|
struct _CArray *src;
|
|
255
|
-
uint8_t src_kind; /* CA_ITER_SRC_*
|
|
256
|
-
int8_t level; /* 1=L1 contig, 2=L2 strided
|
|
253
|
+
uint8_t src_kind; /* CA_ITER_SRC_* */
|
|
254
|
+
int8_t level; /* 1=L1 contig, 2=L2 strided */
|
|
257
255
|
ca_slab_policy_t policy;
|
|
258
256
|
int8_t ndim;
|
|
259
|
-
int8_t naxes; /*
|
|
260
|
-
|
|
257
|
+
int8_t naxes; /* AXES policy axis count */
|
|
258
|
+
void *pair; /* the other state of a two-state
|
|
259
|
+
walk (ca_iter_state_init_l2_paired),
|
|
260
|
+
else NULL; finished with this one
|
|
261
|
+
when the walk raises */
|
|
261
262
|
uint32_t flags;
|
|
262
263
|
ca_size_t bytes; /* element size */
|
|
263
264
|
|
|
264
|
-
/* --- CAStride compose-fold cache (
|
|
265
|
+
/* --- CAStride compose-fold cache (L2 alias path) ---
|
|
265
266
|
For CAStride-family sources at L2, init_l2 runs
|
|
266
|
-
ca_stride_compose_to_root once to get the root
|
|
267
|
-
byte strides + base offset
|
|
268
|
-
|
|
269
|
-
Inline CA_RANK_MAX array (no
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
267
|
+
ca_stride_compose_to_root once to get the root + per-axis
|
|
268
|
+
byte strides + base offset, kept here for the author to read. The
|
|
269
|
+
walk itself steps from composed_base by outer_strides (below), which
|
|
270
|
+
init builds from the composed strides. Inline CA_RANK_MAX array (no
|
|
271
|
+
heap alloc). root is the array attached at
|
|
272
|
+
init whose memory the walk reads: the compose-fold root of an L2
|
|
273
|
+
CAStride walk (an entity, or a view that lends memory) or the parent
|
|
274
|
+
of a descriptor view. NULL when the walk reads src itself or a
|
|
275
|
+
scratch of its own. */
|
|
276
|
+
struct _CArray *root; /* attached at init, detached (or
|
|
277
|
+
synced and detached, on WRITE)
|
|
278
|
+
by finish */
|
|
273
279
|
ca_size_t composed_strides[CA_RANK_MAX]; /* byte units */
|
|
274
280
|
ca_size_t composed_base; /* byte offset from root->ptr */
|
|
275
281
|
|
|
276
|
-
/* --- descriptor framework cache (
|
|
277
|
-
For
|
|
278
|
-
|
|
282
|
+
/* --- descriptor framework cache (descriptor sources) ---
|
|
283
|
+
For CASelectAxis / CAGrid / CASelect / CAWindow / CAShift, init runs
|
|
284
|
+
the view's *_describe_axes once then
|
|
279
285
|
reuses the P3 substrate (ca_axis_dispatch_prepare / _layout /
|
|
280
286
|
_classify_prefix) to derive the slab layout. Cached inline so
|
|
281
287
|
next_slab walks the prefix axes without re-doing the analysis. */
|
|
@@ -291,40 +297,44 @@ typedef struct {
|
|
|
291
297
|
ca_size_t total_elements; /* view.elements snapshot */
|
|
292
298
|
|
|
293
299
|
/* --- iteration cursor --- */
|
|
294
|
-
ca_size_t *outer_idx; /* [
|
|
300
|
+
ca_size_t *outer_idx; /* [outer_ndim] */
|
|
295
301
|
ca_size_t slab_n; /* current slab element count */
|
|
296
302
|
ca_size_t total_slabs;
|
|
297
303
|
ca_size_t slabs_emitted;
|
|
298
304
|
|
|
299
|
-
/* --- chunk position (
|
|
305
|
+
/* --- chunk position (chunk_pos is unused, kept for layout) --- */
|
|
300
306
|
ca_size_t chunk_pos;
|
|
301
307
|
ca_size_t chunk_size;
|
|
302
308
|
|
|
303
|
-
/* --- scratch buffer
|
|
309
|
+
/* --- scratch buffer --- */
|
|
304
310
|
char *scratch_ptr;
|
|
305
311
|
boolean8_t *scratch_mask;
|
|
306
312
|
ca_size_t scratch_cap;
|
|
307
313
|
|
|
308
314
|
/* --- alias path --- */
|
|
309
315
|
uint8_t alias_mode; /* CA_ITER_ALIAS_* */
|
|
310
|
-
char *alias_ptr; /*
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
316
|
+
char *alias_ptr; /* base the slabs are yielded from:
|
|
317
|
+
the source or root memory, or the
|
|
318
|
+
scratch under ALIAS_NONE; NULL for
|
|
319
|
+
STACK / STACK_OUTER_K /
|
|
320
|
+
PER_FIBER_FUSED */
|
|
321
|
+
boolean8_t *alias_mask;
|
|
322
|
+
ca_size_t alias_stride; /* unused, kept for layout */
|
|
323
|
+
|
|
324
|
+
/* --- WRITE-path sync timing --- */
|
|
315
325
|
uint8_t write_dirty;
|
|
316
326
|
|
|
317
|
-
/* --- CA_SLAB_AXES policy fields
|
|
318
|
-
Populated when init_l2 is called with policy = CA_SLAB_AXES
|
|
319
|
-
|
|
327
|
+
/* --- CA_SLAB_AXES policy fields ---
|
|
328
|
+
Populated when init_l2 is called with policy = CA_SLAB_AXES, for
|
|
329
|
+
every kind of source. The kernel reads slab metadata directly
|
|
320
330
|
from these fields (= per-walk metadata, unchanged across slabs).
|
|
321
331
|
outer_axes / outer_dims / outer_strides drive the prefix walk;
|
|
322
332
|
slab_axes_buf / slab_dims / slab_strides describe the K-D slab
|
|
323
333
|
handed to the kernel. All strides are byte units. Zero-init for
|
|
324
334
|
other policies (CA_SLAB_WHOLE / FREE).
|
|
325
335
|
|
|
326
|
-
|
|
327
|
-
the
|
|
336
|
+
The slab axis list is "slab_axes_buf" (an inline array); the axes
|
|
337
|
+
the caller passes to init are not kept in the state.
|
|
328
338
|
|
|
329
339
|
>>> FROZEN author contract (see banner, two-tier freeze): the
|
|
330
340
|
slab-delivery representation a kernel reads is exactly
|
|
@@ -346,28 +356,29 @@ typedef struct {
|
|
|
346
356
|
ca_size_t outer_strides[CA_RANK_MAX]; /* per-outer-axis data byte stride */
|
|
347
357
|
ca_size_t outer_mask_strides[CA_RANK_MAX]; /* per-outer-axis mask element stride */
|
|
348
358
|
|
|
349
|
-
/* --- Per-fiber contig scratch
|
|
359
|
+
/* --- Per-fiber contig scratch ---
|
|
350
360
|
For naxes==1 (= per-axis fiber, the catalog CA_FOR_EACH_FIBER target)
|
|
351
|
-
|
|
352
|
-
|
|
361
|
+
on the paths of next_slab_axes that yield a fiber in place (the
|
|
362
|
+
default fall-through and STACK_OUTER_K; ki_fiber_contig_data), when
|
|
363
|
+
slab_strides[0] != bytes the fiber
|
|
353
364
|
is yielded strided. To honor the catalog contract "data contig
|
|
354
365
|
delivery", next_slab_axes lazily allocates this scratch on first
|
|
355
366
|
gather and gathers each fiber here before yielding. Reused across
|
|
356
367
|
fibers; size grows to max slab_dims[0] * bytes.
|
|
357
368
|
|
|
358
|
-
last_data_off is
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
369
|
+
last_data_off is the byte offset of the current slab from alias_ptr,
|
|
370
|
+
set by next_slab_axes on its default path BEFORE the outer_idx
|
|
371
|
+
advance. sync_slab reads it for the fiber it scatters back and for
|
|
372
|
+
the cells of a whole-view scratch it compares; the slab iterator's
|
|
373
|
+
map_slab reads it to mark the output's mask.
|
|
362
374
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
touch these fields. */
|
|
375
|
+
The per-slab paths (CA_ITER_ALIAS_PER_SLAB / _HOIST) and the stack
|
|
376
|
+
paths do not set it. */
|
|
366
377
|
char *fiber_data_scratch;
|
|
367
378
|
ca_size_t fiber_data_scratch_cap;
|
|
368
379
|
ca_size_t last_data_off;
|
|
369
380
|
|
|
370
|
-
/*
|
|
381
|
+
/* Per-fiber contig mask scratch.
|
|
371
382
|
Symmetric to fiber_data_scratch. When the source carries a mask
|
|
372
383
|
(= alias_mask != NULL) and slab_mask_strides[0] != 1, the engine
|
|
373
384
|
gathers the fiber's mask into contig boolean8_t order here so the
|
|
@@ -378,7 +389,7 @@ typedef struct {
|
|
|
378
389
|
boolean8_t *fiber_mask_scratch;
|
|
379
390
|
ca_size_t fiber_mask_scratch_cap;
|
|
380
391
|
|
|
381
|
-
/* ---
|
|
392
|
+
/* --- PER_FIBER_FUSED fields ---
|
|
382
393
|
When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, next_slab_axes
|
|
383
394
|
builds a fiber region from outer_idx + fiber_axis and calls
|
|
384
395
|
ca_xfer_stride(src, ..., GET) into fiber_data_scratch instead of
|
|
@@ -390,38 +401,34 @@ typedef struct {
|
|
|
390
401
|
ca_size_t fiber_native_strides[CA_RANK_MAX];
|
|
391
402
|
/* fiber_region_starts[] cached by next_slab_axes BEFORE outer_idx
|
|
392
403
|
advance so sync_slab can reconstruct the same ca_xfer_stride
|
|
393
|
-
region for WRITE PUT.
|
|
394
|
-
(= F.1a). */
|
|
404
|
+
region for WRITE PUT. The same hazard as last_data_off. */
|
|
395
405
|
ca_size_t fiber_region_starts[CA_RANK_MAX];
|
|
396
406
|
|
|
397
|
-
/* ---
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
going through ca_xfer_stride. Owned by iter (xfree in finish).
|
|
407
|
+
/* --- CAStack read from its parents (STACK / STACK_OUTER_K) --- */
|
|
408
|
+
/* When alias_mode is CA_ITER_ALIAS_STACK or STACK_OUTER_K, init_l2
|
|
409
|
+
attaches all K parents (= O(1) per entity parent) and caches their
|
|
410
|
+
ptr aliases here so next_slab_axes reads them without going through
|
|
411
|
+
ca_xfer_stride. Owned by iter (xfree in finish).
|
|
403
412
|
stack_parent_strides[] are the uniform parent-native byte strides
|
|
404
|
-
(
|
|
413
|
+
(all the parents of a CAStack have one shape). */
|
|
405
414
|
char **stack_parent_ptrs; /* [n_parents] */
|
|
406
415
|
int32_t stack_n_parents; /* = ((CAStack *)src)->n_parents */
|
|
407
416
|
ca_size_t stack_parent_strides[CA_RANK_MAX]; /* parent-space byte strides */
|
|
408
417
|
/* Parent-space element strides (= 1 byte per cell) for parent mask
|
|
409
|
-
addressing;
|
|
418
|
+
addressing; filled by the STACK_OUTER_K init whether or not the
|
|
419
|
+
stack has a mask. Caching
|
|
410
420
|
here lets next_slab_axes compute mask_off without downcasting to
|
|
411
421
|
CAStack (= AC3 layering goal). */
|
|
412
422
|
ca_size_t stack_parent_mask_strides[CA_RANK_MAX];
|
|
413
|
-
/* --- PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18) --- */
|
|
414
423
|
/* When alias_mode == CA_ITER_ALIAS_STACK_OUTER_K and the CAStack
|
|
415
|
-
source carries a mask, init_l2
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
(xfree in finish). */
|
|
424
|
+
source carries a mask, init_l2 copies the K parent masks here, read
|
|
425
|
+
alongside parents[k]->ptr. NULL when the source has no mask. Owned
|
|
426
|
+
by the iterator (xfree in finish). */
|
|
419
427
|
boolean8_t **stack_parent_mask_ptrs; /* [n_parents] or NULL */
|
|
420
428
|
/* K axis position within the outer iter axis list. Set by init_l2
|
|
421
429
|
when alias_mode == CA_ITER_ALIAS_STACK_OUTER_K; next_slab_axes uses
|
|
422
430
|
outer_idx[stack_k_outer_pos] to pick the active parent. */
|
|
423
431
|
int8_t stack_k_outer_pos;
|
|
424
|
-
/* --- pilot/castack-axis0-loop-interchange (2026-06-19) --- */
|
|
425
432
|
/* CA_ITER_ALIAS_STACK tile cache. Refills TILE fibers (= K cells each)
|
|
426
433
|
at once via K contig parent reads, then serves the next TILE next_slab
|
|
427
434
|
calls from L1d-resident buffer. Layout: cache[t][k] so a fiber at
|
|
@@ -434,10 +441,9 @@ typedef struct {
|
|
|
434
441
|
ca_size_t stack_tile_have; /* fibers actually present in current tile */
|
|
435
442
|
} ca_iter_state;
|
|
436
443
|
|
|
437
|
-
/* ---- alias eligibility predicate
|
|
444
|
+
/* ---- alias eligibility predicate ---------------------------------------- */
|
|
438
445
|
|
|
439
|
-
/* Generalises ca_attach_is_alias (carray_core.c
|
|
440
|
-
ca_attach_is_cheap in T1 step 9.4a) so the alias decision can be
|
|
446
|
+
/* Generalises ca_attach_is_alias (carray_core.c) so the alias decision can be
|
|
441
447
|
made level-aware. Level is the dispatch level the caller intends
|
|
442
448
|
to use:
|
|
443
449
|
|
|
@@ -451,28 +457,31 @@ typedef struct {
|
|
|
451
457
|
Alias iff the engine can yield per-outer-axis slabs as
|
|
452
458
|
parent->ptr + offset with a native stride_bytes argument, with
|
|
453
459
|
no scratch allocation. Broader than L1: any CAStride-family
|
|
454
|
-
view qualifies
|
|
460
|
+
view qualifies by its strides (init_l2 still gathers into scratch
|
|
461
|
+
when the root it folds to lends no memory, which this predicate
|
|
462
|
+
does not look at) (the innermost run, even a stride-of-1 of count 1,
|
|
455
463
|
defines a valid strided slab). Entity arrays also qualify
|
|
456
464
|
trivially. Pathological all-strided-no-contig sources still
|
|
457
465
|
qualify here — the engine yields slab_n=1 with the native step;
|
|
458
466
|
L1 fallback is an engine-policy choice, not an eligibility one.
|
|
459
467
|
|
|
460
468
|
level == 3 (L3, multi-d kernel)
|
|
461
|
-
Not implemented
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
Descriptor
|
|
465
|
-
CAShift
|
|
466
|
-
CABitarray) return 0 at every level
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
469
|
+
Not implemented — answered as L1 so the predicate stays
|
|
470
|
+
well-defined for callers that probe ahead.
|
|
471
|
+
|
|
472
|
+
Descriptor views (CASelectAxis / CAGrid / CASelect / CAWindow /
|
|
473
|
+
CAShift) and conversion views (CAFake / CAByteSwap / CABitfield /
|
|
474
|
+
CABitarray) return 0 at every level. init_l2 still reads a
|
|
475
|
+
descriptor view in place when its innermost axis is a stride
|
|
476
|
+
(CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE); this predicate does not
|
|
477
|
+
report that case.
|
|
478
|
+
|
|
479
|
+
ca_attach_is_alias stays the level=1 answer for its existing callers;
|
|
480
|
+
new code in the kernel iterator path should call ca_iter_can_alias
|
|
481
|
+
with an explicit level. */
|
|
473
482
|
int ca_iter_can_alias (void *ap, int level);
|
|
474
483
|
|
|
475
|
-
/* ---- state machine
|
|
484
|
+
/* ---- state machine ------------------------------------------------------ */
|
|
476
485
|
|
|
477
486
|
/* Initialise `st` for an **L1 (contig kernel)** walk of `src`.
|
|
478
487
|
|
|
@@ -481,19 +490,22 @@ int ca_iter_can_alias (void *ap, int level);
|
|
|
481
490
|
src->ptr, stride implicit = bytes)
|
|
482
491
|
- CAStride family non-contig: scratch path (ca_copy_data
|
|
483
492
|
compose-fold gather into a malloc'd buffer)
|
|
484
|
-
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
493
|
+
- descriptor views (CAGrid / CASelect / CAWindow / CAShift /
|
|
494
|
+
CASelectAxis): gathered into a scratch buffer
|
|
495
|
+
- SRC_ATTACH views (overlays, lazy views, CAObject, CAStack, ...,
|
|
496
|
+
and any other view with an attach slot): iterator-owned scratch
|
|
497
|
+
filled through ca_xfer_all
|
|
498
|
+
- a source with no attach slot: CA_ITER_ERR_NOT_CHEAP
|
|
499
|
+
|
|
500
|
+
policy: CA_SLAB_WHOLE only (CA_SLAB_AXES and CA_SLAB_FREE return
|
|
501
|
+
CA_ITER_ERR_POLICY).
|
|
502
|
+
flags: any of CA_KERNEL_WRITE / CA_KERNEL_NO_MASK /
|
|
503
|
+
CA_KERNEL_FIBER_CONTIG; other bits return CA_ITER_ERR_FLAGS.
|
|
488
504
|
|
|
489
505
|
On success returns CA_ITER_OK; on error returns CA_ITER_ERR_*
|
|
490
506
|
without claiming resources (finish need not be called).
|
|
491
507
|
|
|
492
|
-
Pair with ca_iter_state_next_slab.
|
|
493
|
-
level-specific entry points so each setup path stays focused
|
|
494
|
-
(proposal §3.2 rev: original single-init was relaxed when L2 setup
|
|
495
|
-
diverged enough to warrant its own state initialiser; see
|
|
496
|
-
ROADMAP/CHANGELOG rev). */
|
|
508
|
+
Pair with ca_iter_state_next_slab. */
|
|
497
509
|
int ca_iter_state_init_l1 (ca_iter_state *st,
|
|
498
510
|
struct _CArray *src,
|
|
499
511
|
ca_slab_policy_t policy,
|
|
@@ -504,14 +516,31 @@ int ca_iter_state_init_l1 (ca_iter_state *st,
|
|
|
504
516
|
/* Initialise `st` for an **L2 (strided kernel)** walk of `src`.
|
|
505
517
|
|
|
506
518
|
Source routing:
|
|
507
|
-
- entity / CAStride contig: alias_mode = CONTIG,
|
|
508
|
-
|
|
509
|
-
- CAStride non-contig: alias_mode = STRIDED,
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
519
|
+
- entity / CAStride contig: alias_mode = CONTIG, the slab reads
|
|
520
|
+
the source's own memory
|
|
521
|
+
- CAStride non-contig: alias_mode = STRIDED, the slab reads the
|
|
522
|
+
root through the composed strides. No scratch.
|
|
523
|
+
- descriptor views: aliased when the innermost axis is a STRIDE
|
|
524
|
+
axis, otherwise gathered into scratch (per slab when a slab axis
|
|
525
|
+
is an INDEX or SHIFT axis)
|
|
526
|
+
- SRC_ATTACH views: iterator-owned scratch filled through
|
|
527
|
+
ca_xfer_all, written back with ki_scratch_push_changed (only the
|
|
528
|
+
changed cells). A CAStack may instead be walked through its
|
|
529
|
+
parents, and a single FIBER_CONTIG fiber of a cast / byte-swap
|
|
530
|
+
view through ca_xfer_stride (PER_FIBER_FUSED)
|
|
531
|
+
- a source with no attach slot: CA_ITER_ERR_NOT_CHEAP
|
|
532
|
+
|
|
533
|
+
policy: CA_SLAB_AXES only. CA_SLAB_WHOLE and CA_SLAB_FREE return
|
|
534
|
+
CA_ITER_ERR_POLICY, as do out-of-range or duplicate axes,
|
|
535
|
+
and a descriptor view whose descriptor rank differs from the
|
|
536
|
+
view's rank. The whole array as one contiguous run is L1
|
|
537
|
+
(init_l1 with CA_SLAB_WHOLE); one strided run per innermost
|
|
538
|
+
row is CA_SLAB_AXES with the last axis.
|
|
539
|
+
flags: as init_l1. A CA_KERNEL_WRITE walk over a descriptor view
|
|
540
|
+
whose slab has an INDEX or SHIFT axis goes by the transfer
|
|
541
|
+
walk.
|
|
542
|
+
|
|
543
|
+
Pair with ca_iter_state_next_slab_axes. */
|
|
515
544
|
int ca_iter_state_init_l2 (ca_iter_state *st,
|
|
516
545
|
struct _CArray *src,
|
|
517
546
|
ca_slab_policy_t policy,
|
|
@@ -527,11 +556,12 @@ int ca_iter_state_init_l2 (ca_iter_state *st,
|
|
|
527
556
|
*out_mask is set to the per-slab boolean8_t mask pointer when the
|
|
528
557
|
source carries a mask (= ca_has_mask(src)), or NULL otherwise.
|
|
529
558
|
The mask layout matches the value layout (= same iteration order
|
|
530
|
-
and same n).
|
|
531
|
-
|
|
559
|
+
and same n). Kernels use the CA_FOR_EACH_UNMASKED macro family
|
|
560
|
+
(below, in this header) to skip masked cells.
|
|
532
561
|
|
|
533
|
-
The mask is an input. What it points at is a copy taken when the
|
|
534
|
-
walk began,
|
|
562
|
+
The mask is an input. What it points at is a copy -- taken when the
|
|
563
|
+
walk began, or, on the per-fiber fused path, gathered for each fiber
|
|
564
|
+
-- so writing through it changes nothing -- not even under
|
|
535
565
|
CA_KERNEL_WRITE, where the value half of the same yield is often a
|
|
536
566
|
live alias. It also cannot express the thing an author would most
|
|
537
567
|
want it for: an unmasked source yields NULL, so there is nowhere to
|
|
@@ -551,22 +581,6 @@ int ca_iter_state_next_slab (ca_iter_state *st,
|
|
|
551
581
|
boolean8_t **out_mask,
|
|
552
582
|
ca_size_t *out_n);
|
|
553
583
|
|
|
554
|
-
/* Pull the next L2 strided slab. Returns 1 with *out_ptr /
|
|
555
|
-
*out_mask / *out_n / *out_stride_bytes when a slab is yielded;
|
|
556
|
-
returns 0 when the walk is complete. The kernel walks `*out_n`
|
|
557
|
-
elements by stepping `*out_stride_bytes` between consecutive
|
|
558
|
-
elements starting at `*out_ptr`; the mask uses the **same stride
|
|
559
|
-
semantics** when non-NULL (= each mask byte at offset i * stride is
|
|
560
|
-
conceptually paired with the value at ptr + i * stride_bytes, but
|
|
561
|
-
since mask is boolean8_t == 1 byte, mask stride is 1 byte when the
|
|
562
|
-
value stride is bytes, and proportional otherwise). Only valid
|
|
563
|
-
when init_l2 was used. */
|
|
564
|
-
int ca_iter_state_next_slab_strided (ca_iter_state *st,
|
|
565
|
-
char **out_ptr,
|
|
566
|
-
boolean8_t **out_mask,
|
|
567
|
-
ca_size_t *out_n,
|
|
568
|
-
ca_size_t *out_stride_bytes);
|
|
569
|
-
|
|
570
584
|
/* Pull the next CA_SLAB_AXES slab (K-D block). Returns 1 with *out_ptr
|
|
571
585
|
/ *out_mask set to the slab base when a slab is yielded; returns 0
|
|
572
586
|
when the walk is complete. Only valid when init_l2 was called with
|
|
@@ -603,36 +617,115 @@ int ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
|
603
617
|
/* Sync the just-yielded slab back to parent (WRITE path; READ no-op).
|
|
604
618
|
|
|
605
619
|
Caller calls this **unconditionally** after each next_slab /
|
|
606
|
-
|
|
620
|
+
next_slab_axes + kernel invocation; the alias_mode branch lives
|
|
607
621
|
inside the state machine so the caller never has to inspect it.
|
|
608
622
|
|
|
609
|
-
Behaviour
|
|
623
|
+
Behaviour:
|
|
610
624
|
- !(flags & CA_KERNEL_WRITE): no-op (READ walk)
|
|
611
|
-
-
|
|
612
|
-
|
|
613
|
-
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
625
|
+
- per-fiber gather (CA_KERNEL_FIBER_CONTIG on a strided fiber):
|
|
626
|
+
the fiber is scattered back from its scratch first
|
|
627
|
+
- CAStack walked through its parents: READ only, no-op
|
|
628
|
+
- per-fiber fused (CA_ITER_ALIAS_PER_FIBER_FUSED): the fiber is put
|
|
629
|
+
back through the view with ca_xfer_stride
|
|
630
|
+
- alias path (scratch_ptr == NULL): no-op — kernel wrote through
|
|
631
|
+
alias_ptr. When the alias is into a view root's own buffer, that
|
|
632
|
+
buffer is written back once, by finish (ca_sync_detach)
|
|
633
|
+
- scratch path, at L1 or L2: the cells the kernel changed are sent
|
|
634
|
+
back through the source (ki_scratch_push_changed: ca_xfer_addrs,
|
|
635
|
+
or ca_xfer_all PUT when every cell changed); a write over a
|
|
636
|
+
descriptor view takes this path, opened as a transfer walk
|
|
637
|
+
|
|
638
|
+
A view that shows one root cell more than once (a repeated index, a
|
|
639
|
+
tile, a stack or meld holding the same parent twice): which of the
|
|
640
|
+
values written to its positions the root cell ends with is not
|
|
641
|
+
specified. It depends on the route, and a value equal to the one
|
|
642
|
+
delivered is not sent back. The same holds for a view that reads
|
|
643
|
+
several root cells as one (the mask of a refer to a wider type, which
|
|
644
|
+
is the OR of the cells under it): a cell written with the value it
|
|
645
|
+
was delivered is not sent back, so what its root cells end with is
|
|
646
|
+
not specified. */
|
|
617
647
|
void ca_iter_state_sync_slab (ca_iter_state *st);
|
|
618
648
|
|
|
619
|
-
/* Release any resources held by `st` and detach parent.
|
|
620
|
-
|
|
649
|
+
/* Release any resources held by `st` and detach parent. On a WRITE
|
|
650
|
+
walk with a root, the root is synced before it is detached
|
|
651
|
+
(ca_sync_detach); a slab still handed out from scratch is dropped, not
|
|
652
|
+
written back (ca_iter_state_close writes it back first). Idempotent:
|
|
653
|
+
a second call is a no-op. Also safe after a failed init, which holds
|
|
654
|
+
nothing to release (the call is then not required). */
|
|
621
655
|
void ca_iter_state_finish (ca_iter_state *st);
|
|
622
656
|
|
|
657
|
+
/* Open `st` as the second state of a walk whose first state is already
|
|
658
|
+
open -- an input and an output walked together. Takes what
|
|
659
|
+
ca_iter_state_init_l2 takes, plus `first`.
|
|
660
|
+
|
|
661
|
+
The two are then finished together when either walk raises (a gather
|
|
662
|
+
or a write-back through a source's slots): each is open in the
|
|
663
|
+
caller's frame, and the raise would jump over the other's finish. If
|
|
664
|
+
opening `st` raises, or is declined (the code is returned), `first` is
|
|
665
|
+
finished here, so the caller has nothing left to close.
|
|
666
|
+
|
|
667
|
+
When `st` writes (CA_KERNEL_WRITE), `first` reads, both walk by
|
|
668
|
+
CA_SLAB_AXES, and their arrays live in the same root, `first` is
|
|
669
|
+
reopened on a copy of its array taken before anything is written, so
|
|
670
|
+
the kernel reads the values the input held when the walk began. Two
|
|
671
|
+
walks opened separately with ca_iter_state_init_l2 know nothing of
|
|
672
|
+
each other: a read walk over memory a write walk writes into must be
|
|
673
|
+
given a copy by the caller. */
|
|
674
|
+
int ca_iter_state_init_l2_paired (ca_iter_state *st,
|
|
675
|
+
ca_iter_state *first,
|
|
676
|
+
struct _CArray *src,
|
|
677
|
+
ca_slab_policy_t policy,
|
|
678
|
+
int8_t *axes,
|
|
679
|
+
int8_t naxes,
|
|
680
|
+
uint32_t flags);
|
|
681
|
+
|
|
682
|
+
/* Finish a walk as the block macros leave it: when the body left a write
|
|
683
|
+
walk with `break`, the slab it was in is written back before the walk is
|
|
684
|
+
finished. ca_iter_state_finish alone drops such a slab when it sits in
|
|
685
|
+
scratch (writes through an alias are already in place). */
|
|
686
|
+
void ca_iter_state_close (ca_iter_state *st);
|
|
687
|
+
|
|
623
688
|
/* Raise unless rc is CA_ITER_OK. The block macros call this on the value
|
|
624
|
-
|
|
689
|
+
ca_iter_state_init_l2 / _l2_paired returned, so an author who never looks at a
|
|
625
690
|
return code still hears about a request the iterator would not serve.
|
|
626
691
|
Returns rc when there is nothing to report, so it composes inside the
|
|
627
692
|
macros' comma expressions. */
|
|
628
693
|
int ca_iter_check_init (int rc);
|
|
629
694
|
|
|
630
|
-
/*
|
|
631
|
-
|
|
695
|
+
/* Raise ArgumentError unless the two arrays have the same shape. The
|
|
696
|
+
INOUT and PAIR fiber macros call this before opening either walk. */
|
|
697
|
+
int ca_iter_check_same_shape (struct _CArray *a, struct _CArray *b);
|
|
698
|
+
|
|
699
|
+
/* Run body(arg) so that the walks in states[0..n) are finished however the
|
|
700
|
+
body leaves: by returning, by `break`, or by raising.
|
|
701
|
+
|
|
702
|
+
A walk releases what it holds in ca_iter_state_finish, and a body that
|
|
703
|
+
raises jumps over the finish -- the block macros put it in the `for`
|
|
704
|
+
increment, and a hand-written walk has it after the loop. A body that
|
|
705
|
+
calls back into Ruby (an object-lane kernel) can raise at any cell, so
|
|
706
|
+
it runs through this function.
|
|
707
|
+
|
|
708
|
+
The states belong to the caller's frame, not to the body's: after a
|
|
709
|
+
raise, the body's frame is gone, and the cleanup runs where it was.
|
|
710
|
+
Pass them to the body inside `arg` and open each one there, with
|
|
711
|
+
ca_iter_state_init_* or a block macro (`CA_FOR_EACH_SLAB(c->st, ...)`).
|
|
712
|
+
A state declared inside the body is not covered.
|
|
713
|
+
|
|
714
|
+
The states are marked unopened before the body runs, so one the body
|
|
715
|
+
never opens costs nothing, and the body may finish a walk itself:
|
|
716
|
+
finishing is idempotent. Returns what the body returns; a raise propagates
|
|
717
|
+
after the walks are finished. */
|
|
718
|
+
VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
719
|
+
VALUE (*body)(VALUE), VALUE arg);
|
|
720
|
+
|
|
721
|
+
/* ---- kernel author block macros -----------------------------------------
|
|
632
722
|
|
|
633
723
|
Wrap init_l2 / next_slab_axes / [sync_slab] / finish in a single
|
|
634
724
|
block scope so kernel authors don't write lifecycle plumbing.
|
|
635
725
|
|
|
726
|
+
For a function in carray that uses each macro, see the catalog in
|
|
727
|
+
docs/authoring/HOW_TO_WRITE_KERNEL.md §0.1.
|
|
728
|
+
|
|
636
729
|
--- Constraints ---
|
|
637
730
|
|
|
638
731
|
- Author must pre-declare `char *p` and `boolean8_t *m` (or names of
|
|
@@ -652,24 +745,25 @@ int ca_iter_check_init (int rc);
|
|
|
652
745
|
should drop down to the raw API and read the code themselves.
|
|
653
746
|
- `break;` from inside the body exits the loop AND triggers finish
|
|
654
747
|
correctly (= outer for's "increment" clause runs once on natural
|
|
655
|
-
exit; `break` from the inner while breaks both).
|
|
748
|
+
exit; `break` from the inner while breaks both). In a write walk
|
|
749
|
+
the macros close with ca_iter_state_close, which writes back the
|
|
750
|
+
slab the body was in. `return` inside
|
|
656
751
|
the body LEAKS resources (scratch_ptr, parent attach) — drop to
|
|
657
752
|
raw API if early return is needed. A body that raises leaks the
|
|
658
753
|
same way: the engine releases what it holds when the walk itself
|
|
659
754
|
raises (a gather or a write-back through the source's slots), but
|
|
660
755
|
the body runs in the caller's frame, where it has no hold on it.
|
|
661
|
-
|
|
756
|
+
A body that can raise -- one that calls back into Ruby, as an
|
|
757
|
+
object-lane kernel does -- runs through ca_iter_ensure, with the
|
|
758
|
+
state in the caller's frame.
|
|
662
759
|
- Macros are not statement-equivalent (= they expand to nested for
|
|
663
760
|
constructs). Don't follow them with `else` etc. */
|
|
664
761
|
|
|
665
|
-
/* ----
|
|
666
|
-
|
|
762
|
+
/* ---- mask helper macro ---------------------------------------------------
|
|
763
|
+
This macro lets a kernel handle masked sources
|
|
667
764
|
uniformly: pass NULL for `mask` if the source has no mask (= treat
|
|
668
765
|
all cells as unmasked), or a `boolean8_t *` of length n where
|
|
669
766
|
non-zero entries indicate masked (= "skip this cell").
|
|
670
|
-
|
|
671
|
-
PROPOSAL_T1_KERNEL_ITERATOR.md §2.4. GCC statement-expression for
|
|
672
|
-
CA_COUNT_UNMASKED is GCC/Clang only (MSVC not in scope).
|
|
673
767
|
--------------------------------------------------------------------- */
|
|
674
768
|
|
|
675
769
|
#define CA_FOR_EACH_UNMASKED(p, mask, n, body) do { \
|
|
@@ -681,31 +775,9 @@ int ca_iter_check_init (int rc);
|
|
|
681
775
|
} \
|
|
682
776
|
} while (0)
|
|
683
777
|
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
} else { \
|
|
688
|
-
for (ca_size_t i = 0; i < (n); i++) { body } \
|
|
689
|
-
} \
|
|
690
|
-
} while (0)
|
|
691
|
-
|
|
692
|
-
#define CA_COUNT_UNMASKED(mask, n) ({ \
|
|
693
|
-
ca_size_t _ca_cnt = 0; \
|
|
694
|
-
ca_size_t _ca_n = (n); \
|
|
695
|
-
if (mask) { \
|
|
696
|
-
for (ca_size_t _ca_i = 0; _ca_i < _ca_n; _ca_i++) \
|
|
697
|
-
if (!(mask)[_ca_i]) _ca_cnt++; \
|
|
698
|
-
} else { \
|
|
699
|
-
_ca_cnt = _ca_n; \
|
|
700
|
-
} \
|
|
701
|
-
_ca_cnt; \
|
|
702
|
-
})
|
|
703
|
-
|
|
704
|
-
#define CA_MASK_GET(mask, i) ((mask) ? (mask)[i] : 0)
|
|
705
|
-
|
|
706
|
-
/* ---- L2 inner-loop macros (step 8+) -----------------------------------
|
|
707
|
-
`CA_L2_FOR_EACH(T, ptr, n, stride, p, body)` and its unmasked sibling
|
|
708
|
-
wrap the strided callback signature `(ptr, n, stride_bytes)` with a
|
|
778
|
+
/* ---- L2 inner-loop macro ------------------------------------------------
|
|
779
|
+
`CA_L2_FOR_EACH(T, ptr, n, stride, p, body)` wraps the strided
|
|
780
|
+
callback signature `(ptr, n, stride_bytes)` with a
|
|
709
781
|
`stride == sizeof(T)` fast-path branch. When the runtime stride
|
|
710
782
|
matches the element size, `body` runs against a `T *p` that the
|
|
711
783
|
compiler can autovectorise; when it doesn't, `p` is recomputed per
|
|
@@ -714,24 +786,21 @@ int ca_iter_check_init (int rc);
|
|
|
714
786
|
The split is intentional: L2 strided callbacks are deliberately
|
|
715
787
|
universal — they accept arbitrary stride and so the compiler cannot
|
|
716
788
|
prove contig on the kernel-side loop. When the iterator hands a
|
|
717
|
-
contig scratch (
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
789
|
+
contig scratch (a gathered view, stride == bytes), a loop written for
|
|
790
|
+
any stride was measured 18-22% slower than a contiguous one. This
|
|
791
|
+
macro lets kernel authors recover the contig autovectorisation
|
|
792
|
+
without giving up the L2 surface.
|
|
721
793
|
|
|
722
794
|
This is the standard pattern for L2 kernels: write the body once,
|
|
723
|
-
the macro picks the right loop shape.
|
|
724
|
-
|
|
725
|
-
kernel-side macros are the speed knob) is `PROPOSAL_T1_KERNEL_
|
|
726
|
-
ITERATOR.md` §13.1.
|
|
795
|
+
the macro picks the right loop shape. The walk stays the same for
|
|
796
|
+
every source; the loop shape is the kernel's knob for speed.
|
|
727
797
|
|
|
728
798
|
Usage:
|
|
729
799
|
CA_L2_FOR_EACH(double, ptr, n, stride, p, {
|
|
730
800
|
*p = value; // p is `double *`
|
|
731
801
|
});
|
|
732
802
|
|
|
733
|
-
|
|
734
|
-
declarations inside a macro). */
|
|
803
|
+
Plain C99 (a do/while block with for-loop declarations). */
|
|
735
804
|
|
|
736
805
|
#define CA_L2_FOR_EACH(T, ptr, n, stride, p, body) do { \
|
|
737
806
|
ca_size_t _l2_n = (n); \
|
|
@@ -751,39 +820,6 @@ int ca_iter_check_init (int rc);
|
|
|
751
820
|
} \
|
|
752
821
|
} while (0)
|
|
753
822
|
|
|
754
|
-
#define CA_L2_FOR_EACH_UNMASKED(T, ptr, mask, n, stride, p, body) do { \
|
|
755
|
-
ca_size_t _l2_n = (n); \
|
|
756
|
-
ca_size_t _l2_s = (stride); \
|
|
757
|
-
char *_l2_b = (char *)(ptr); \
|
|
758
|
-
boolean8_t *_l2_m = (mask); \
|
|
759
|
-
if (_l2_s == sizeof(T)) { \
|
|
760
|
-
T *p = (T *)_l2_b; \
|
|
761
|
-
if (_l2_m) { \
|
|
762
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
763
|
-
if (!_l2_m[_l2_i]) { body } \
|
|
764
|
-
p++; \
|
|
765
|
-
} \
|
|
766
|
-
} else { \
|
|
767
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
768
|
-
body \
|
|
769
|
-
p++; \
|
|
770
|
-
} \
|
|
771
|
-
} \
|
|
772
|
-
} else { \
|
|
773
|
-
if (_l2_m) { \
|
|
774
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
775
|
-
T *p = (T *)(_l2_b + _l2_i * _l2_s); \
|
|
776
|
-
if (!_l2_m[_l2_i]) { body } \
|
|
777
|
-
} \
|
|
778
|
-
} else { \
|
|
779
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
780
|
-
T *p = (T *)(_l2_b + _l2_i * _l2_s); \
|
|
781
|
-
body \
|
|
782
|
-
} \
|
|
783
|
-
} \
|
|
784
|
-
} \
|
|
785
|
-
} while (0)
|
|
786
|
-
|
|
787
823
|
/* The slab policy is fixed to CA_SLAB_AXES (the only policy compatible
|
|
788
824
|
with next_slab_axes); it is hardcoded inside the macro rather than
|
|
789
825
|
taken as an argument, so block-macro authors never type the policy
|
|
@@ -796,7 +832,7 @@ int ca_iter_check_init (int rc);
|
|
|
796
832
|
(axes), (naxes), (flags))), \
|
|
797
833
|
1); \
|
|
798
834
|
__caf_init; \
|
|
799
|
-
__caf_init = 0,
|
|
835
|
+
__caf_init = 0, ca_iter_state_close(&(st)) ) \
|
|
800
836
|
for ( ; ca_iter_state_next_slab_axes(&(st), &(p), &(m)); \
|
|
801
837
|
ca_iter_state_sync_slab(&(st)) )
|
|
802
838
|
|
|
@@ -806,32 +842,33 @@ int ca_iter_check_init (int rc);
|
|
|
806
842
|
output iter runs WRITE; sync_slab is called on output after each
|
|
807
843
|
body iteration.
|
|
808
844
|
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
845
|
+
ca_in and ca_out must have the same shape (ndim and every dim[k]);
|
|
846
|
+
ca_iter_check_same_shape raises ArgumentError before either walk is
|
|
847
|
+
opened when they do not. Init failure on either iter raises — see
|
|
848
|
+
CA_FOR_EACH_SLAB above. */
|
|
813
849
|
/* Policy fixed to CA_SLAB_AXES internally — see CA_FOR_EACH_SLAB above. */
|
|
814
850
|
#define CA_FOR_EACH_SLAB_INOUT(st_in, st_out, ca_in, ca_out, \
|
|
815
851
|
axes, naxes, \
|
|
816
852
|
p_in, p_out, m_in, m_out) \
|
|
817
853
|
for ( int __cafi_init = ( \
|
|
854
|
+
ca_iter_check_same_shape((ca_in), (ca_out)), \
|
|
818
855
|
ca_iter_check_init( \
|
|
819
856
|
ca_iter_state_init_l2(&(st_in), (ca_in), CA_SLAB_AXES, \
|
|
820
857
|
(axes), (naxes), 0)), \
|
|
821
858
|
ca_iter_check_init( \
|
|
822
|
-
|
|
859
|
+
ca_iter_state_init_l2_paired(&(st_out), &(st_in), (ca_out), \
|
|
860
|
+
CA_SLAB_AXES, \
|
|
823
861
|
(axes), (naxes), CA_KERNEL_WRITE)), \
|
|
824
862
|
1); \
|
|
825
863
|
__cafi_init; \
|
|
826
864
|
__cafi_init = 0, \
|
|
827
|
-
|
|
828
|
-
|
|
865
|
+
ca_iter_state_close(&(st_in)), \
|
|
866
|
+
ca_iter_state_close(&(st_out)) ) \
|
|
829
867
|
for ( ; ca_iter_state_next_slab_axes(&(st_in), &(p_in), &(m_in)) && \
|
|
830
868
|
ca_iter_state_next_slab_axes(&(st_out), &(p_out), &(m_out)); \
|
|
831
869
|
ca_iter_state_sync_slab(&(st_out)) )
|
|
832
870
|
|
|
833
|
-
/* ----
|
|
834
|
-
(rev4 §3 catalog contract)
|
|
871
|
+
/* ---- per-axis fiber macros ----------------------------------------------
|
|
835
872
|
|
|
836
873
|
Author-facing surface for "deliver one contig fiber along `axis` to
|
|
837
874
|
the kernel". Contig delivery is contract:
|
|
@@ -841,23 +878,27 @@ int ca_iter_check_init (int rc);
|
|
|
841
878
|
- output (INOUT forms): contig same as data; CA_KERNEL_WRITE auto-set
|
|
842
879
|
|
|
843
880
|
The CA_KERNEL_FIBER_CONTIG flag is auto-set; engine gathers strided
|
|
844
|
-
fibers into per-state scratch when slab_strides[0] != bytes
|
|
881
|
+
fibers into per-state scratch when slab_strides[0] != bytes.
|
|
845
882
|
|
|
846
|
-
`axis`
|
|
847
|
-
|
|
848
|
-
|
|
883
|
+
`axis`, `ca`, `ca_in` and `ca_out` may be evaluated more than once:
|
|
884
|
+
pass variables, not expressions with side effects. `n` is set to the
|
|
885
|
+
fiber length (= slab_dims[0], constant per walk).
|
|
886
|
+
|
|
887
|
+
The forms without MASKED in the name hand out no mask cursor: a masked
|
|
888
|
+
source is walked with its masked cells read as values. Pass
|
|
889
|
+
CA_KERNEL_NO_MASK to have a masked source refused, or use a MASKED
|
|
890
|
+
form to see the mask.
|
|
849
891
|
|
|
850
892
|
Same constraints as CA_FOR_EACH_SLAB family:
|
|
851
|
-
- `break;` from body exits cleanly (finish runs
|
|
893
|
+
- `break;` from body exits cleanly (finish runs, and a write walk
|
|
894
|
+
writes back the fiber it was in).
|
|
852
895
|
- `return;` from body LEAKS scratch / parent attach -- use raw API.
|
|
853
896
|
- Macros are NOT statement-equivalent (nested for); no trailing else.
|
|
854
897
|
|
|
855
|
-
INOUT forms (form 2 / 4) require
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
macros runtime-assert shape equality and skip body on mismatch.
|
|
860
|
-
Authors that need broadcasting must drop to raw API. */
|
|
898
|
+
INOUT forms (form 2 / 4) require `ca_in` and `ca_out` to have the same
|
|
899
|
+
shape (ndim and every dim[k]), and raise ArgumentError before opening
|
|
900
|
+
either walk when they do not. Authors that need broadcasting must
|
|
901
|
+
drop to raw API. */
|
|
861
902
|
|
|
862
903
|
#define CA_FOR_EACH_FIBER(st, ca, axis, flags, p, n) \
|
|
863
904
|
for ( int __cff_init = ( \
|
|
@@ -868,7 +909,7 @@ int ca_iter_check_init (int rc);
|
|
|
868
909
|
(n) = (st).slab_dims[0], \
|
|
869
910
|
1); \
|
|
870
911
|
__cff_init; \
|
|
871
|
-
__cff_init = 0,
|
|
912
|
+
__cff_init = 0, ca_iter_state_close(&(st)) ) \
|
|
872
913
|
for ( ; ca_iter_state_next_slab_axes(&(st), &(p), NULL); \
|
|
873
914
|
ca_iter_state_sync_slab(&(st)) )
|
|
874
915
|
|
|
@@ -881,31 +922,23 @@ int ca_iter_check_init (int rc);
|
|
|
881
922
|
(n) = (st).slab_dims[0], \
|
|
882
923
|
1); \
|
|
883
924
|
__cffm_init; \
|
|
884
|
-
__cffm_init = 0,
|
|
925
|
+
__cffm_init = 0, ca_iter_state_close(&(st)) ) \
|
|
885
926
|
for ( ; ca_iter_state_next_slab_axes(&(st), &(p), &(m)); \
|
|
886
927
|
ca_iter_state_sync_slab(&(st)) )
|
|
887
928
|
|
|
888
|
-
/* INOUT form 2
|
|
889
|
-
|
|
890
|
-
STRICT FULL SHAPE EQUALITY (rev4 §2.3): the inner for-condition
|
|
891
|
-
re-evaluates ca_in->ndim == ca_out->ndim and dim[axis] equality (=
|
|
892
|
-
minimal seam coverage given the macro can't loop over k). Full
|
|
893
|
-
per-axis equality lives one level up in the init-time short-circuit
|
|
894
|
-
below: we compare elements + ndim + axis dim, which catches the
|
|
895
|
-
common silent-corruption seam (= e.g. (3,5) vs (4,5) axis=1 with
|
|
896
|
-
matching fiber length but different fiber count). Comprehensive
|
|
897
|
-
per-dim check is the caller's responsibility for now (= simpler than
|
|
898
|
-
building a per-dim k loop into a macro; ext authors can drop to raw
|
|
899
|
-
API for arbitrary broadcasting designs). */
|
|
929
|
+
/* INOUT form 2. Output gets CA_KERNEL_WRITE auto-set. The shapes are
|
|
930
|
+
checked first (ca_iter_check_same_shape). */
|
|
900
931
|
#define CA_FOR_EACH_FIBER_INOUT(st_in, st_out, ca_in, ca_out, axis, \
|
|
901
932
|
flags, p_in, p_out, n) \
|
|
902
933
|
for ( int __cffi_init = ( \
|
|
934
|
+
ca_iter_check_same_shape((ca_in), (ca_out)), \
|
|
903
935
|
ca_iter_check_init( \
|
|
904
936
|
ca_iter_state_init_l2(&(st_in), (ca_in), CA_SLAB_AXES, \
|
|
905
937
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
906
938
|
(flags) | CA_KERNEL_FIBER_CONTIG)), \
|
|
907
939
|
ca_iter_check_init( \
|
|
908
|
-
|
|
940
|
+
ca_iter_state_init_l2_paired(&(st_out), &(st_in), (ca_out), \
|
|
941
|
+
CA_SLAB_AXES, \
|
|
909
942
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
910
943
|
((flags) | CA_KERNEL_FIBER_CONTIG \
|
|
911
944
|
| CA_KERNEL_WRITE))), \
|
|
@@ -913,12 +946,9 @@ int ca_iter_check_init (int rc);
|
|
|
913
946
|
1); \
|
|
914
947
|
__cffi_init; \
|
|
915
948
|
__cffi_init = 0, \
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
for ( ; (st_in)
|
|
919
|
-
&& (st_in).src->elements == (st_out).src->elements \
|
|
920
|
-
&& (st_in).slab_dims[0] == (st_out).slab_dims[0] \
|
|
921
|
-
&& ca_iter_state_next_slab_axes(&(st_in), &(p_in), NULL) \
|
|
949
|
+
ca_iter_state_close(&(st_in)), \
|
|
950
|
+
ca_iter_state_close(&(st_out)) ) \
|
|
951
|
+
for ( ; ca_iter_state_next_slab_axes(&(st_in), &(p_in), NULL) \
|
|
922
952
|
&& ca_iter_state_next_slab_axes(&(st_out), &(p_out), NULL); \
|
|
923
953
|
ca_iter_state_sync_slab(&(st_in)), \
|
|
924
954
|
ca_iter_state_sync_slab(&(st_out)) )
|
|
@@ -926,12 +956,14 @@ int ca_iter_check_init (int rc);
|
|
|
926
956
|
#define CA_FOR_EACH_FIBER_INOUT_MASKED(st_in, st_out, ca_in, ca_out, axis, \
|
|
927
957
|
flags, p_in, p_out, n, m) \
|
|
928
958
|
for ( int __cffim_init = ( \
|
|
959
|
+
ca_iter_check_same_shape((ca_in), (ca_out)), \
|
|
929
960
|
ca_iter_check_init( \
|
|
930
961
|
ca_iter_state_init_l2(&(st_in), (ca_in), CA_SLAB_AXES, \
|
|
931
962
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
932
963
|
(flags) | CA_KERNEL_FIBER_CONTIG)), \
|
|
933
964
|
ca_iter_check_init( \
|
|
934
|
-
|
|
965
|
+
ca_iter_state_init_l2_paired(&(st_out), &(st_in), (ca_out), \
|
|
966
|
+
CA_SLAB_AXES, \
|
|
935
967
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
936
968
|
((flags) | CA_KERNEL_FIBER_CONTIG \
|
|
937
969
|
| CA_KERNEL_WRITE))), \
|
|
@@ -939,12 +971,9 @@ int ca_iter_check_init (int rc);
|
|
|
939
971
|
1); \
|
|
940
972
|
__cffim_init; \
|
|
941
973
|
__cffim_init = 0, \
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
for ( ; (st_in)
|
|
945
|
-
&& (st_in).src->elements == (st_out).src->elements \
|
|
946
|
-
&& (st_in).slab_dims[0] == (st_out).slab_dims[0] \
|
|
947
|
-
&& ca_iter_state_next_slab_axes(&(st_in), &(p_in), &(m)) \
|
|
974
|
+
ca_iter_state_close(&(st_in)), \
|
|
975
|
+
ca_iter_state_close(&(st_out)) ) \
|
|
976
|
+
for ( ; ca_iter_state_next_slab_axes(&(st_in), &(p_in), &(m)) \
|
|
948
977
|
&& ca_iter_state_next_slab_axes(&(st_out), &(p_out), NULL); \
|
|
949
978
|
ca_iter_state_sync_slab(&(st_in)), \
|
|
950
979
|
ca_iter_state_sync_slab(&(st_out)) )
|
|
@@ -962,31 +991,32 @@ int ca_iter_check_init (int rc);
|
|
|
962
991
|
cursors: which cells a pair of fibers may be used at is a question about
|
|
963
992
|
both of them, and the INOUT forms answer only about the input.
|
|
964
993
|
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
sources may be the same array.
|
|
994
|
+
The two sources must have the same shape, as for the INOUT forms:
|
|
995
|
+
ca_iter_check_same_shape raises ArgumentError before either walk is
|
|
996
|
+
opened. The two sources may be the same array. The second state is
|
|
997
|
+
opened paired with the first, so a refusal or a raise on either side
|
|
998
|
+
closes both. */
|
|
968
999
|
|
|
969
1000
|
#define CA_FOR_EACH_FIBER_PAIR(st_a, st_b, ca_a, ca_b, axis, \
|
|
970
1001
|
flags, p_a, p_b, n) \
|
|
971
1002
|
for ( int __cffp_init = ( \
|
|
1003
|
+
ca_iter_check_same_shape((ca_a), (ca_b)), \
|
|
972
1004
|
ca_iter_check_init( \
|
|
973
1005
|
ca_iter_state_init_l2(&(st_a), (ca_a), CA_SLAB_AXES, \
|
|
974
1006
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
975
1007
|
(flags) | CA_KERNEL_FIBER_CONTIG)), \
|
|
976
1008
|
ca_iter_check_init( \
|
|
977
|
-
|
|
1009
|
+
ca_iter_state_init_l2_paired(&(st_b), &(st_a), (ca_b), \
|
|
1010
|
+
CA_SLAB_AXES, \
|
|
978
1011
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
979
1012
|
(flags) | CA_KERNEL_FIBER_CONTIG)), \
|
|
980
1013
|
(n) = (st_a).slab_dims[0], \
|
|
981
1014
|
1); \
|
|
982
1015
|
__cffp_init; \
|
|
983
1016
|
__cffp_init = 0, \
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
for ( ; (st_a)
|
|
987
|
-
&& (st_a).src->elements == (st_b).src->elements \
|
|
988
|
-
&& (st_a).slab_dims[0] == (st_b).slab_dims[0] \
|
|
989
|
-
&& ca_iter_state_next_slab_axes(&(st_a), &(p_a), NULL) \
|
|
1017
|
+
ca_iter_state_close(&(st_a)), \
|
|
1018
|
+
ca_iter_state_close(&(st_b)) ) \
|
|
1019
|
+
for ( ; ca_iter_state_next_slab_axes(&(st_a), &(p_a), NULL) \
|
|
990
1020
|
&& ca_iter_state_next_slab_axes(&(st_b), &(p_b), NULL); \
|
|
991
1021
|
ca_iter_state_sync_slab(&(st_a)), \
|
|
992
1022
|
ca_iter_state_sync_slab(&(st_b)) )
|
|
@@ -994,29 +1024,88 @@ int ca_iter_check_init (int rc);
|
|
|
994
1024
|
#define CA_FOR_EACH_FIBER_PAIR_MASKED(st_a, st_b, ca_a, ca_b, axis, \
|
|
995
1025
|
flags, p_a, p_b, n, m_a, m_b) \
|
|
996
1026
|
for ( int __cffpm_init = ( \
|
|
1027
|
+
ca_iter_check_same_shape((ca_a), (ca_b)), \
|
|
997
1028
|
ca_iter_check_init( \
|
|
998
1029
|
ca_iter_state_init_l2(&(st_a), (ca_a), CA_SLAB_AXES, \
|
|
999
1030
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
1000
1031
|
(flags) | CA_KERNEL_FIBER_CONTIG)), \
|
|
1001
1032
|
ca_iter_check_init( \
|
|
1002
|
-
|
|
1033
|
+
ca_iter_state_init_l2_paired(&(st_b), &(st_a), (ca_b), \
|
|
1034
|
+
CA_SLAB_AXES, \
|
|
1003
1035
|
(int8_t[]){(int8_t)(axis)}, 1, \
|
|
1004
1036
|
(flags) | CA_KERNEL_FIBER_CONTIG)), \
|
|
1005
1037
|
(n) = (st_a).slab_dims[0], \
|
|
1006
1038
|
1); \
|
|
1007
1039
|
__cffpm_init; \
|
|
1008
1040
|
__cffpm_init = 0, \
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
for ( ; (st_a)
|
|
1012
|
-
&& (st_a).src->elements == (st_b).src->elements \
|
|
1013
|
-
&& (st_a).slab_dims[0] == (st_b).slab_dims[0] \
|
|
1014
|
-
&& ca_iter_state_next_slab_axes(&(st_a), &(p_a), &(m_a)) \
|
|
1041
|
+
ca_iter_state_close(&(st_a)), \
|
|
1042
|
+
ca_iter_state_close(&(st_b)) ) \
|
|
1043
|
+
for ( ; ca_iter_state_next_slab_axes(&(st_a), &(p_a), &(m_a)) \
|
|
1015
1044
|
&& ca_iter_state_next_slab_axes(&(st_b), &(p_b), &(m_b)); \
|
|
1016
1045
|
ca_iter_state_sync_slab(&(st_a)), \
|
|
1017
1046
|
ca_iter_state_sync_slab(&(st_b)) )
|
|
1018
1047
|
|
|
1019
|
-
/* ----
|
|
1048
|
+
/* ---- whole-array block macros -------------------------------------------
|
|
1049
|
+
|
|
1050
|
+
Hand the kernel the whole array as one contiguous run of cells in
|
|
1051
|
+
row-major order: `p` is the first cell and `n` is the number of cells
|
|
1052
|
+
(src->elements). The body runs once.
|
|
1053
|
+
|
|
1054
|
+
CA_FOR_EACH_WHOLE (st, ca, flags, p, n) data only
|
|
1055
|
+
CA_FOR_EACH_WHOLE_MASKED(st, ca, flags, p, n, m) + mask m[0..n-1],
|
|
1056
|
+
NULL when the
|
|
1057
|
+
source has none
|
|
1058
|
+
|
|
1059
|
+
Use them for a kernel that wants every cell in order and no axes: a
|
|
1060
|
+
whole-array scan, a hash of the values, a copy into a library buffer.
|
|
1061
|
+
A reduction that also runs per axis is better written once with
|
|
1062
|
+
CA_FOR_EACH_SLAB over the chosen axes (every axis for the whole array).
|
|
1063
|
+
|
|
1064
|
+
Contiguity is the promise, and it is kept by copying when it has to be:
|
|
1065
|
+
a contiguous array lends its memory, any other source is gathered into
|
|
1066
|
+
the walk's scratch first (a transpose, a strided block, a selection, a
|
|
1067
|
+
converting view). A kernel that can take strides and wants to avoid
|
|
1068
|
+
that copy uses CA_FOR_EACH_SLAB over every axis instead.
|
|
1069
|
+
|
|
1070
|
+
Same rules as CA_FOR_EACH_SLAB and the CA_FOR_EACH_FIBER family:
|
|
1071
|
+
- `p` and the mask cursor are declared by the caller; `n` receives the
|
|
1072
|
+
cell count. `ca` may be evaluated more than once.
|
|
1073
|
+
- Init failure raises (ca_iter_check_init); the body does not run.
|
|
1074
|
+
- The form without MASKED hands out no mask cursor: masked cells are
|
|
1075
|
+
read as values. Pass CA_KERNEL_NO_MASK to refuse a masked source.
|
|
1076
|
+
The mask cursor of the MASKED form is a copy; writing it reaches
|
|
1077
|
+
nothing.
|
|
1078
|
+
- With CA_KERNEL_WRITE the cells the kernel changed are written back
|
|
1079
|
+
when the body ends, including by `break`.
|
|
1080
|
+
- `return` from the body leaks what the walk holds. So does a body
|
|
1081
|
+
that raises: one that can raise -- one that calls back into Ruby --
|
|
1082
|
+
opens the macro inside ca_iter_ensure, with the state in the
|
|
1083
|
+
caller's frame. A write that raises keeps the cells already
|
|
1084
|
+
written where the walk lends the array's memory, and none of them
|
|
1085
|
+
where it gathered a copy.
|
|
1086
|
+
- Not statement-equivalent (nested `for`); no trailing `else`. */
|
|
1087
|
+
|
|
1088
|
+
#define CA_FOR_EACH_WHOLE(st, ca, flags, p, n) \
|
|
1089
|
+
for ( int __cfw_init = (ca_iter_check_init( \
|
|
1090
|
+
ca_iter_state_init_l1(&(st), (ca), CA_SLAB_WHOLE, \
|
|
1091
|
+
NULL, 0, (flags))), \
|
|
1092
|
+
1); \
|
|
1093
|
+
__cfw_init; \
|
|
1094
|
+
__cfw_init = 0, ca_iter_state_close(&(st)) ) \
|
|
1095
|
+
for ( ; ca_iter_state_next_slab(&(st), &(p), NULL, &(n)); \
|
|
1096
|
+
ca_iter_state_sync_slab(&(st)) )
|
|
1097
|
+
|
|
1098
|
+
#define CA_FOR_EACH_WHOLE_MASKED(st, ca, flags, p, n, m) \
|
|
1099
|
+
for ( int __cfwm_init = (ca_iter_check_init( \
|
|
1100
|
+
ca_iter_state_init_l1(&(st), (ca), CA_SLAB_WHOLE,\
|
|
1101
|
+
NULL, 0, (flags))), \
|
|
1102
|
+
1); \
|
|
1103
|
+
__cfwm_init; \
|
|
1104
|
+
__cfwm_init = 0, ca_iter_state_close(&(st)) ) \
|
|
1105
|
+
for ( ; ca_iter_state_next_slab(&(st), &(p), &(m), &(n)); \
|
|
1106
|
+
ca_iter_state_sync_slab(&(st)) )
|
|
1107
|
+
|
|
1108
|
+
/* ---- per-data_type reduction macro suite ------------------------------- */
|
|
1020
1109
|
|
|
1021
1110
|
/* CA_SLAB_REDUCE_T(T, ...): generic per-data_type slab reduction. T is the
|
|
1022
1111
|
element load type (`double`, `float`, `int32_t`, `int64_t`, ...).
|
|
@@ -1043,10 +1132,8 @@ int ca_iter_check_init (int rc);
|
|
|
1043
1132
|
- p: slab data pointer (char *).
|
|
1044
1133
|
- m: slab mask pointer (boolean8_t *, may be NULL).
|
|
1045
1134
|
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
numerics; use CA_SLAB_REDUCE_T directly for less common types
|
|
1049
|
-
(boolean8_t, int8_t, uint16_t, ...).
|
|
1135
|
+
T is the load type: double, float, int32_t, int64_t, or any other
|
|
1136
|
+
element type (boolean8_t, int8_t, uint16_t, ...).
|
|
1050
1137
|
|
|
1051
1138
|
Mask semantics: when m != NULL, masked cells are skipped (REDUCE is
|
|
1052
1139
|
not invoked). When m == NULL, every cell contributes.
|
|
@@ -1154,9 +1241,9 @@ int ca_iter_check_init (int rc);
|
|
|
1154
1241
|
} while (0)
|
|
1155
1242
|
|
|
1156
1243
|
/* Backward-compatible wrapper that hides masked_cnt from kernels that
|
|
1157
|
-
don't need it.
|
|
1158
|
-
|
|
1159
|
-
|
|
1244
|
+
don't need it. The generated reductions use the _EX forms (or their
|
|
1245
|
+
SIMD / 8-way variants); CA_SLAB_REDUCE_T is for hand-written kernels
|
|
1246
|
+
that do not need the masked count. */
|
|
1160
1247
|
#define CA_SLAB_REDUCE_T(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
1161
1248
|
ca_size_t __sr_throwaway_mc = 0; \
|
|
1162
1249
|
CA_SLAB_REDUCE_T_EX(T, st, p, m, acc, INIT, REDUCE, __sr_throwaway_mc); \
|
|
@@ -1164,24 +1251,18 @@ int ca_iter_check_init (int rc);
|
|
|
1164
1251
|
} while (0)
|
|
1165
1252
|
|
|
1166
1253
|
/* ----------------------------------------------------------------------
|
|
1167
|
-
* SIMD-licensed reduction variants
|
|
1254
|
+
* SIMD-licensed reduction variants
|
|
1168
1255
|
*
|
|
1169
|
-
* The PLUS/MIN/MAX/STAR
|
|
1170
|
-
*
|
|
1171
|
-
*
|
|
1172
|
-
*
|
|
1173
|
-
*
|
|
1174
|
-
*
|
|
1175
|
-
* Subsequent sub-steps (SL.1.1+) will replace each forward with a
|
|
1176
|
-
* contig-branch body carrying `#pragma omp simd reduction(<kind>:acc)`
|
|
1177
|
-
* (and per-acc pragmas for VAR). The pragma is emitted via the
|
|
1256
|
+
* The PLUS/MIN/MAX/STAR _EX variants below carry, in their contiguous
|
|
1257
|
+
* no-mask branch, `#pragma omp simd reduction(<kind>:acc)`, which
|
|
1258
|
+
* licenses the compiler to reassociate the reduction (results are
|
|
1259
|
+
* epsilon-close, not bit-exact). VAR_EX forwards to PLUS_EX and has no
|
|
1260
|
+
* caller. The pragma is emitted via the
|
|
1178
1261
|
* _Pragma + _CA_XSTR substitution helpers below so the acc lvalue
|
|
1179
1262
|
* can be parameterised.
|
|
1180
1263
|
*
|
|
1181
1264
|
* The pragma is a no-op on compilers that don't support `-fopenmp-simd`
|
|
1182
1265
|
* (extconf.rb probe — graceful degradation, code stays correct).
|
|
1183
|
-
*
|
|
1184
|
-
* Q1-Q6 closure (sparring round 1 2026-06-12): see proposal §5.
|
|
1185
1266
|
* --------------------------------------------------------------------*/
|
|
1186
1267
|
|
|
1187
1268
|
#define _CA_STR(x) #x
|
|
@@ -1191,15 +1272,15 @@ int ca_iter_check_init (int rc);
|
|
|
1191
1272
|
#define _CA_SIMD_MAX(var) _Pragma(_CA_XSTR(omp simd reduction(max:var)))
|
|
1192
1273
|
#define _CA_SIMD_STAR(var) _Pragma(_CA_XSTR(omp simd reduction(*:var)))
|
|
1193
1274
|
|
|
1194
|
-
/* CA_SLAB_REDUCE_T_PLUS_EX
|
|
1275
|
+
/* CA_SLAB_REDUCE_T_PLUS_EX:
|
|
1195
1276
|
* Same structure as CA_SLAB_REDUCE_T_EX, but the **no-mask + contig**
|
|
1196
1277
|
* inner loop carries `#pragma omp simd reduction(+:acc)` so clang/gcc
|
|
1197
1278
|
* are licensed to reassoc the accumulator and emit SIMD reduction
|
|
1198
1279
|
* sequences. All other branches (masked-contig, non-contig, masked-
|
|
1199
|
-
* non-contig) are byte-identical to _EX
|
|
1200
|
-
*
|
|
1280
|
+
* non-contig) are byte-identical to _EX: a SIMD reduction there would
|
|
1281
|
+
* need an output-buffered loop interchange.
|
|
1201
1282
|
*
|
|
1202
|
-
*
|
|
1283
|
+
* Measured: N=1M f64 sum 906 us -> 116 us (= 7.8x, 68.8 GB/s).
|
|
1203
1284
|
* Parity: ε-close (relative error < 2e-16 for f64 sum), bit-exact
|
|
1204
1285
|
* not guaranteed (= the SIMD license; see guides/devel/11_kernel_iterator.md).
|
|
1205
1286
|
*
|
|
@@ -1299,7 +1380,7 @@ int ca_iter_check_init (int rc);
|
|
|
1299
1380
|
(void) idx; \
|
|
1300
1381
|
} while (0)
|
|
1301
1382
|
|
|
1302
|
-
/* CA_SLAB_REDUCE_SUM8_EX
|
|
1383
|
+
/* CA_SLAB_REDUCE_SUM8_EX:
|
|
1303
1384
|
* 8-way manual accumulator split for the no-mask + contig inner loop.
|
|
1304
1385
|
*
|
|
1305
1386
|
* Why: GCC 11.5 with -march=native (FMA) does NOT split the pragma-simd
|
|
@@ -1323,8 +1404,9 @@ int ca_iter_check_init (int rc);
|
|
|
1323
1404
|
* branches stay single-accumulator (not the hot path). Position
|
|
1324
1405
|
* counter `idx` is not tracked (variance REDUCE never uses it). ACC_T
|
|
1325
1406
|
* is the accumulator C type (double for Pass 1 sum on numeric/bool and
|
|
1326
|
-
* Pass 2 M2; complex Pass 1 passes its complex type).
|
|
1327
|
-
*
|
|
1407
|
+
* Pass 2 M2; complex Pass 1 passes its complex type). mkkernel.rb
|
|
1408
|
+
* emits it for every 8-way-eligible _PLUS reduction (sum, mean, and
|
|
1409
|
+
* both passes of variance / stddev).
|
|
1328
1410
|
*/
|
|
1329
1411
|
#define CA_SLAB_REDUCE_SUM8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
|
|
1330
1412
|
(acc) = (INIT); \
|
|
@@ -1410,9 +1492,8 @@ int ca_iter_check_init (int rc);
|
|
|
1410
1492
|
} \
|
|
1411
1493
|
} while (0)
|
|
1412
1494
|
|
|
1413
|
-
/* CA_SLAB_REDUCE_MIN8_EX / _MAX8_EX / _STAR8_EX
|
|
1414
|
-
*
|
|
1415
|
-
* 2026-07-19):
|
|
1495
|
+
/* CA_SLAB_REDUCE_MIN8_EX / _MAX8_EX / _STAR8_EX (the SUM8_EX pattern for
|
|
1496
|
+
* the other standard reducers):
|
|
1416
1497
|
*
|
|
1417
1498
|
* Same rationale as SUM8_EX — GCC does not split single-accumulator pragma
|
|
1418
1499
|
* reductions into multiple independent chains on FMA/FADD-latency-bound
|
|
@@ -1703,10 +1784,10 @@ int ca_iter_check_init (int rc);
|
|
|
1703
1784
|
} \
|
|
1704
1785
|
} while (0)
|
|
1705
1786
|
|
|
1706
|
-
/* CA_SLAB_REDUCE_T_MIN_EX (
|
|
1787
|
+
/* CA_SLAB_REDUCE_T_MIN_EX (emitted by the kernel generator; not author surface):
|
|
1707
1788
|
* Same structure as PLUS_EX; no-mask + contig inner loop carries
|
|
1708
|
-
* `#pragma omp simd reduction(min:acc)`.
|
|
1709
|
-
*
|
|
1789
|
+
* `#pragma omp simd reduction(min:acc)`. The :min kernel in
|
|
1790
|
+
* mkkernel.rb writes REDUCE as `acc = (v < acc) ? v : acc`, which
|
|
1710
1791
|
* matches OpenMP `min:` reduction semantics exactly — bit-exact
|
|
1711
1792
|
* parity preserved (no reassoc license needed; min/max are
|
|
1712
1793
|
* associative + commutative under `<` / `>`).
|
|
@@ -1807,7 +1888,7 @@ int ca_iter_check_init (int rc);
|
|
|
1807
1888
|
(void) idx; \
|
|
1808
1889
|
} while (0)
|
|
1809
1890
|
|
|
1810
|
-
/* CA_SLAB_REDUCE_T_MAX_EX (
|
|
1891
|
+
/* CA_SLAB_REDUCE_T_MAX_EX (generator-only): same as MIN_EX with `_CA_SIMD_MAX`. */
|
|
1811
1892
|
#define CA_SLAB_REDUCE_T_MAX_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
|
|
1812
1893
|
(acc) = (INIT); \
|
|
1813
1894
|
int8_t __srK = (st).slab_ndim; \
|
|
@@ -1900,7 +1981,7 @@ int ca_iter_check_init (int rc);
|
|
|
1900
1981
|
(void) idx; \
|
|
1901
1982
|
} while (0)
|
|
1902
1983
|
|
|
1903
|
-
/* CA_SLAB_REDUCE_T_STAR_EX (
|
|
1984
|
+
/* CA_SLAB_REDUCE_T_STAR_EX (emitted by the kernel generator; not author surface):
|
|
1904
1985
|
* Same structure as PLUS_EX; no-mask + contig inner loop carries
|
|
1905
1986
|
* `#pragma omp simd reduction(*:acc)`. Used by :prod kernel
|
|
1906
1987
|
* (acc *= v). f64 / f32 multiplication parity is ε-close (same
|
|
@@ -2000,51 +2081,16 @@ int ca_iter_check_init (int rc);
|
|
|
2000
2081
|
(void) idx; \
|
|
2001
2082
|
} while (0)
|
|
2002
2083
|
|
|
2003
|
-
/*
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
CA_SLAB_REDUCE_T_PLUS_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2007
|
-
__sr_throwaway_mc); \
|
|
2008
|
-
(void) __sr_throwaway_mc; \
|
|
2009
|
-
} while (0)
|
|
2010
|
-
#define CA_SLAB_REDUCE_T_MIN(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
2011
|
-
ca_size_t __sr_throwaway_mc = 0; \
|
|
2012
|
-
CA_SLAB_REDUCE_T_MIN_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2013
|
-
__sr_throwaway_mc); \
|
|
2014
|
-
(void) __sr_throwaway_mc; \
|
|
2015
|
-
} while (0)
|
|
2016
|
-
#define CA_SLAB_REDUCE_T_MAX(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
2017
|
-
ca_size_t __sr_throwaway_mc = 0; \
|
|
2018
|
-
CA_SLAB_REDUCE_T_MAX_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2019
|
-
__sr_throwaway_mc); \
|
|
2020
|
-
(void) __sr_throwaway_mc; \
|
|
2021
|
-
} while (0)
|
|
2022
|
-
#define CA_SLAB_REDUCE_T_STAR(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
2023
|
-
ca_size_t __sr_throwaway_mc = 0; \
|
|
2024
|
-
CA_SLAB_REDUCE_T_STAR_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2025
|
-
__sr_throwaway_mc); \
|
|
2026
|
-
(void) __sr_throwaway_mc; \
|
|
2027
|
-
} while (0)
|
|
2028
|
-
|
|
2029
|
-
/* Multi-acc variant for variance/stddev (sum + sumsq + cnt).
|
|
2030
|
-
*
|
|
2031
|
-
* SL.1.3 measurement (2026-06-12) showed VAR_EX is **NOT NEEDED** for
|
|
2032
|
-
* variance/stddev: a single `#pragma omp simd reduction(+:acc)` on
|
|
2033
|
-
* the primary accumulator (via PLUS_EX dispatch) is sufficient — clang
|
|
2034
|
-
* extends SIMD treatment to the dependent `sumsq` updates via idiom
|
|
2035
|
-
* recognition (the secondary accumulator is recognised as a derived
|
|
2036
|
-
* reduction over `v*v`). variance entity 1-D f64 reached 53.4 GB/s
|
|
2037
|
-
* (gate > 40 PASS, 8.2x speedup vs 6.5 baseline) using PLUS_EX.
|
|
2084
|
+
/* Multi-acc variant (two accumulators). No kernel uses it: variance /
|
|
2085
|
+
* stddev are computed in two passes (Σx for the mean, then Σ(x-mean)^2),
|
|
2086
|
+
* each a single-accumulator reduction through SUM8_EX / PLUS_EX.
|
|
2038
2087
|
*
|
|
2039
|
-
* VAR_EX is
|
|
2040
|
-
*
|
|
2041
|
-
*
|
|
2042
|
-
* to
|
|
2043
|
-
*
|
|
2044
|
-
*
|
|
2045
|
-
* returns "_VAR"; SL.1.0/1.1 reduce DSL has no :var value).
|
|
2046
|
-
*
|
|
2047
|
-
* Removal candidate if no caller materialises by SL.2 close.
|
|
2088
|
+
* VAR_EX is kept as a stub for a future multi-accumulator reduction
|
|
2089
|
+
* that does not share the variance pattern (primary + derived sumsq).
|
|
2090
|
+
* The stub forwards to PLUS_EX on acc1; acc2 is zero-initialised but
|
|
2091
|
+
* otherwise left to the enclosing REDUCE expression. This is safe
|
|
2092
|
+
* because no kernel dispatches to VAR_EX (reduce_macro_suffix never
|
|
2093
|
+
* returns "_VAR"; the reduce DSL has no :var value).
|
|
2048
2094
|
*/
|
|
2049
2095
|
#define CA_SLAB_REDUCE_T_VAR_EX(T, st, p, m, acc1, acc2, \
|
|
2050
2096
|
INIT1, INIT2, REDUCE, masked_cnt) do { \
|
|
@@ -2066,9 +2112,10 @@ int ca_iter_check_init (int rc);
|
|
|
2066
2112
|
enforced by ca_check_same_shape at dispatcher).
|
|
2067
2113
|
|
|
2068
2114
|
Author-supplied (additional to CA_SLAB_REDUCE_T_EX):
|
|
2069
|
-
- T_W: second-operand element C type (load type).
|
|
2070
|
-
|
|
2071
|
-
:
|
|
2115
|
+
- T_W: second-operand element C type (load type). The
|
|
2116
|
+
production kernels read the weight as double
|
|
2117
|
+
(mkkernel array_arg: data_type: :promote), so T_W need
|
|
2118
|
+
not match T. Kept separate in the
|
|
2072
2119
|
macro signature so future heterogeneous-data_type variants
|
|
2073
2120
|
can plug in without macro change.
|
|
2074
2121
|
- st_w: second-operand ca_iter_state, positioned on the
|
|
@@ -2080,7 +2127,7 @@ int ca_iter_check_init (int rc);
|
|
|
2080
2127
|
REDUCE expression binds BOTH `v` (source cell, type T) AND `w` (weights
|
|
2081
2128
|
cell, type T_W), in addition to `acc` and `idx` from the base macro.
|
|
2082
2129
|
The mask `m` applies to the SOURCE only (weights mask is overlaid onto
|
|
2083
|
-
source mask
|
|
2130
|
+
source mask by the dispatcher). */
|
|
2084
2131
|
#define CA_SLAB_REDUCE_ARRAY_T_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, REDUCE, masked_cnt) do { \
|
|
2085
2132
|
(acc) = (INIT); \
|
|
2086
2133
|
int8_t __sraK = (st).slab_ndim; \
|
|
@@ -2187,14 +2234,7 @@ int ca_iter_check_init (int rc);
|
|
|
2187
2234
|
(void) idx; \
|
|
2188
2235
|
} while (0)
|
|
2189
2236
|
|
|
2190
|
-
|
|
2191
|
-
ca_size_t __sra_throwaway_mc = 0; \
|
|
2192
|
-
CA_SLAB_REDUCE_ARRAY_T_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, REDUCE, \
|
|
2193
|
-
__sra_throwaway_mc); \
|
|
2194
|
-
(void) __sra_throwaway_mc; \
|
|
2195
|
-
} while (0)
|
|
2196
|
-
|
|
2197
|
-
/* CA_SLAB_REDUCE_ARRAY_T_PLUS_EX (SL.1.4b):
|
|
2237
|
+
/* CA_SLAB_REDUCE_ARRAY_T_PLUS_EX:
|
|
2198
2238
|
* Parallel-array PLUS variant of CA_SLAB_REDUCE_ARRAY_T_EX.
|
|
2199
2239
|
* no-mask + both-contig inner loop carries `#pragma omp simd
|
|
2200
2240
|
* reduction(+:acc)`. Used by :wsum / :wmean — author REDUCE
|
|
@@ -2203,7 +2243,7 @@ int ca_iter_check_init (int rc);
|
|
|
2203
2243
|
* reduction-tree merge. Other branches (masked, non-contig)
|
|
2204
2244
|
* stay identical to _EX.
|
|
2205
2245
|
*
|
|
2206
|
-
*
|
|
2246
|
+
* Measured:
|
|
2207
2247
|
* wsum f64 N=1M: 908 us -> ~150 us (= the expected 50-60 GB/s)
|
|
2208
2248
|
* wmean f64 N=1M: 1212 us -> ~150 us (same path + 1 extra acc)
|
|
2209
2249
|
*/
|
|
@@ -2315,23 +2355,6 @@ int ca_iter_check_init (int rc);
|
|
|
2315
2355
|
(void) idx; \
|
|
2316
2356
|
} while (0)
|
|
2317
2357
|
|
|
2318
|
-
#define CA_SLAB_REDUCE_ARRAY_T_PLUS(T, T_W, st, p, m, st_w, p_w, acc, \
|
|
2319
|
-
INIT, REDUCE) do { \
|
|
2320
|
-
ca_size_t __sra_throwaway_mc = 0; \
|
|
2321
|
-
CA_SLAB_REDUCE_ARRAY_T_PLUS_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, \
|
|
2322
|
-
REDUCE, __sra_throwaway_mc); \
|
|
2323
|
-
(void) __sra_throwaway_mc; \
|
|
2324
|
-
} while (0)
|
|
2325
|
-
|
|
2326
|
-
#define CA_SLAB_REDUCE_F64(st, p, m, acc, INIT, REDUCE) \
|
|
2327
|
-
CA_SLAB_REDUCE_T(double, st, p, m, acc, INIT, REDUCE)
|
|
2328
|
-
#define CA_SLAB_REDUCE_F32(st, p, m, acc, INIT, REDUCE) \
|
|
2329
|
-
CA_SLAB_REDUCE_T(float, st, p, m, acc, INIT, REDUCE)
|
|
2330
|
-
#define CA_SLAB_REDUCE_I32(st, p, m, acc, INIT, REDUCE) \
|
|
2331
|
-
CA_SLAB_REDUCE_T(int32_t, st, p, m, acc, INIT, REDUCE)
|
|
2332
|
-
#define CA_SLAB_REDUCE_I64(st, p, m, acc, INIT, REDUCE) \
|
|
2333
|
-
CA_SLAB_REDUCE_T(int64_t, st, p, m, acc, INIT, REDUCE)
|
|
2334
|
-
|
|
2335
2358
|
/* CA_SLAB_MAP_T(T_IN, T_OUT, ...): generic per-cell transform from
|
|
2336
2359
|
input slab to output slab, walking both in lockstep. T_IN is the
|
|
2337
2360
|
input element load type, T_OUT is the output element store type.
|
|
@@ -2351,11 +2374,9 @@ int ca_iter_check_init (int rc);
|
|
|
2351
2374
|
Both states must share slab geometry (= same slab_ndim and slab_dims),
|
|
2352
2375
|
typically by initialising both with the same policy + axes on
|
|
2353
2376
|
shape-equal CArrays. Mask handling is **not** done by this macro —
|
|
2354
|
-
if your input is masked, see §
|
|
2377
|
+
if your input is masked, see §8 Mask handling in
|
|
2355
2378
|
docs/authoring/HOW_TO_WRITE_KERNEL.md. Caller is responsible for invoking
|
|
2356
|
-
ca_iter_state_sync_slab on the output state after each slab.
|
|
2357
|
-
|
|
2358
|
-
Convenience aliases below: CA_SLAB_MAP_F64 (T_IN = T_OUT = double). */
|
|
2379
|
+
ca_iter_state_sync_slab on the output state after each slab. */
|
|
2359
2380
|
#define CA_SLAB_MAP_T(T_IN, T_OUT, st_in, p_in, st_out, p_out, MAP_EXPR) do { \
|
|
2360
2381
|
int8_t __mK = (st_in).slab_ndim; \
|
|
2361
2382
|
int8_t __mOuterK = __mK - 1; \
|
|
@@ -2400,9 +2421,6 @@ int ca_iter_check_init (int rc);
|
|
|
2400
2421
|
} \
|
|
2401
2422
|
} while (0)
|
|
2402
2423
|
|
|
2403
|
-
#define CA_SLAB_MAP_F64(st_in, p_in, st_out, p_out, MAP_EXPR) \
|
|
2404
|
-
CA_SLAB_MAP_T(double, double, st_in, p_in, st_out, p_out, MAP_EXPR)
|
|
2405
|
-
|
|
2406
2424
|
/* CA_SLAB_SCAN_T(T_LOAD, T_OUT, ...): cumulative / prefix-scan walk
|
|
2407
2425
|
that combines a reduction (running accumulator) with a map (per-cell
|
|
2408
2426
|
output write). Walks input and output slabs in lockstep; the
|
|
@@ -2694,10 +2712,10 @@ int ca_iter_check_init (int rc);
|
|
|
2694
2712
|
} while (0)
|
|
2695
2713
|
|
|
2696
2714
|
/* ---- Ruby surface ---------------------------------------------------- */
|
|
2697
|
-
/* Called from Init_carray_ext (ruby_carray.c).
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2715
|
+
/* Called from Init_carray_ext (ruby_carray.c). Creates the
|
|
2716
|
+
Ractor-local key for the kept scratch buffer, and in a development
|
|
2717
|
+
build (CARRAY_DEV_BUILD) registers the CArray.t1_* smoke entry points
|
|
2718
|
+
the spec_ai kernel iterator tests use. */
|
|
2701
2719
|
void Init_ca_kernel_iterator (void);
|
|
2702
2720
|
|
|
2703
2721
|
#endif /* CA_KERNEL_ITERATOR_H */
|