carray 3.0.3 → 3.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +469 -0
- data/README.md +3 -2
- data/carray.gemspec +1 -1
- data/ext/ca_array_pool.c +25 -1
- data/ext/ca_axis_descriptor.h +27 -24
- data/ext/ca_axis_dispatch.c +14 -26
- data/ext/ca_axis_group.c +27 -27
- data/ext/ca_binop_dispatch.c +1 -29
- data/ext/ca_binop_dispatch.h +0 -1
- data/ext/ca_categorical_iterator.c +231 -153
- data/ext/ca_compare.c +9 -20
- data/ext/ca_composite_dispatch.c +9 -9
- data/ext/ca_group_iter.c +16 -17
- data/ext/ca_kernel_iterator.c +1240 -2000
- data/ext/ca_kernel_iterator.h +502 -519
- data/ext/ca_monop_dispatch.c +82 -59
- data/ext/ca_monop_dispatch.h +1 -0
- data/ext/ca_obj_array.c +152 -45
- data/ext/ca_obj_bincmp.c +25 -21
- data/ext/ca_obj_binop.c +16 -26
- data/ext/ca_obj_bitarray.c +8 -3
- data/ext/ca_obj_bitfield.c +63 -49
- data/ext/ca_obj_block.c +18 -9
- data/ext/ca_obj_byte_swap.c +7 -7
- data/ext/ca_obj_const_string.c +188 -22
- data/ext/ca_obj_face.c +280 -58
- data/ext/ca_obj_face.h +19 -0
- data/ext/ca_obj_fake.c +15 -29
- data/ext/ca_obj_farray.c +6 -3
- data/ext/ca_obj_field.c +30 -8
- data/ext/ca_obj_fixlen_string.c +2 -3
- data/ext/ca_obj_grid.c +455 -134
- data/ext/ca_obj_meld.c +172 -34
- data/ext/ca_obj_moncmp.c +23 -8
- data/ext/ca_obj_monop.c +113 -67
- data/ext/ca_obj_object.c +210 -52
- data/ext/ca_obj_record.c +15 -7
- data/ext/ca_obj_reduce.c +17 -8
- data/ext/ca_obj_refer.c +42 -35
- data/ext/ca_obj_remap.c +30 -9
- data/ext/ca_obj_repeat.c +9 -10
- data/ext/ca_obj_roll.c +31 -10
- data/ext/ca_obj_select.c +48 -14
- data/ext/ca_obj_select_axis.c +15 -47
- data/ext/ca_obj_shift.c +41 -7
- data/ext/ca_obj_stack.c +180 -34
- data/ext/ca_obj_stride.c +190 -134
- data/ext/ca_obj_string.c +80 -4
- data/ext/ca_obj_tile.c +32 -11
- data/ext/ca_obj_time.c +22 -6
- data/ext/ca_obj_timedelta.c +22 -6
- data/ext/ca_obj_transpose.c +13 -7
- data/ext/ca_obj_triop.c +26 -32
- data/ext/ca_obj_window.c +267 -74
- data/ext/ca_op_ipower.c +4 -4
- data/ext/ca_sort_kernels.h +6 -3
- data/ext/ca_sweep_engine.c +2 -2
- data/ext/caframe_csv_reader.c +80 -1
- data/ext/carray.h +214 -75
- data/ext/carray_access.c +200 -99
- data/ext/carray_address_basis.c +343 -87
- data/ext/carray_attribute.c +113 -6
- data/ext/carray_bincount.c +10 -8
- data/ext/carray_broadcast.c +13 -8
- data/ext/carray_build_flags.h +3 -0
- data/ext/carray_call_cfunc.c +108 -0
- data/ext/carray_cast.c +375 -96
- data/ext/carray_cast_func.rb +90 -20
- data/ext/carray_class.c +4 -3
- data/ext/carray_config.h +28 -0
- data/ext/carray_conversion.c +18 -9
- data/ext/carray_copy.c +51 -14
- data/ext/carray_core.c +150 -55
- data/ext/carray_count.c +56 -20
- data/ext/carray_element.c +81 -33
- data/ext/carray_factorize.c +138 -84
- data/ext/carray_generate.c +199 -52
- data/ext/carray_histogram.c +63 -18
- data/ext/carray_hold.c +12 -15
- data/ext/carray_index_classifier.c +388 -176
- data/ext/carray_internal.h +348 -29
- data/ext/carray_kernels_bincmp.c +4453 -0
- data/ext/carray_kernels_binop.c +11368 -0
- data/ext/carray_kernels_init.c +1734 -0
- data/ext/carray_kernels_moncmp.c +2104 -0
- data/ext/carray_kernels_monop.c +18369 -0
- data/ext/carray_kernels_reduce_aggregate.c +23818 -0
- data/ext/carray_kernels_reduce_boolean.c +406 -0
- data/ext/carray_kernels_reduce_cumulative.c +10086 -0
- data/ext/carray_kernels_reduce_extreme.c +17634 -0
- data/ext/carray_kernels_reduce_variance.c +3849 -0
- data/ext/carray_kernels_scan.c +4302 -0
- data/ext/carray_kernels_search.c +33829 -0
- data/ext/carray_kernels_sort.c +10897 -0
- data/ext/carray_kernels_triop.c +1864 -0
- data/ext/carray_key_windows.c +220 -0
- data/ext/carray_lazy.c +170 -111
- data/ext/carray_mask.c +337 -94
- data/ext/carray_median_percentile.c +82 -28
- data/ext/carray_memory_view.c +669 -561
- data/ext/carray_operator.c +129 -52
- data/ext/carray_order.c +604 -77
- data/ext/carray_partition.c +28 -27
- data/ext/carray_random.c +85 -31
- data/ext/carray_round.c +273 -0
- data/ext/carray_scatter.c +87 -45
- data/ext/carray_segment.c +3 -3
- data/ext/carray_slab.c +123 -37
- data/ext/carray_sort.c +259 -51
- data/ext/carray_sort_kernel.c +5 -7
- data/ext/carray_struct.c +54 -61
- data/ext/carray_test.c +26 -5
- data/ext/carray_time_text.c +4 -2
- data/ext/carray_undef.c +4 -0
- data/ext/carray_utils.c +309 -66
- data/ext/mk_call_cfunc.rb +19 -2
- data/ext/mkkernel.rb +403 -350
- data/ext/ruby_carray.c +10 -4
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +139 -22
- data/lib/carray/autoload_carray.rb +8 -31
- data/lib/carray/autoload_method_extension.rb +14 -5
- data/lib/carray/axis_group.rb +255 -106
- data/lib/carray/basics.rb +105 -17
- data/lib/carray/bincount_nd.rb +12 -2
- data/lib/carray/block_iterator.rb +126 -23
- data/lib/carray/boolean_reduce.rb +17 -17
- data/lib/carray/categorical.rb +195 -59
- data/lib/carray/categorical_iterator.rb +132 -29
- data/lib/carray/complex.rb +5 -5
- data/lib/carray/conditional.rb +67 -18
- data/lib/carray/const_string.rb +24 -5
- data/lib/carray/construct.rb +15 -11
- data/lib/carray/data_type_extension.rb +9 -1
- data/lib/carray/distance_limit.rb +77 -0
- data/lib/carray/fixlen_string.rb +48 -3
- data/lib/carray/frame/column_verbs.rb +71 -0
- data/lib/carray/frame/concat.rb +17 -3
- data/lib/carray/frame/convert.rb +69 -0
- data/lib/carray/frame/csv_parser.rb +111 -17
- data/lib/carray/frame/describe.rb +3 -2
- data/lib/carray/frame/frame.rb +252 -28
- data/lib/carray/frame/group.rb +603 -101
- data/lib/carray/frame/io.rb +117 -28
- data/lib/carray/frame/join.rb +312 -51
- data/lib/carray/frame/records.rb +83 -7
- data/lib/carray/frame/reshape.rb +42 -35
- data/lib/carray/frame/sort.rb +7 -19
- data/lib/carray/frame/verbs.rb +188 -48
- data/lib/carray/frame/windows.rb +117 -0
- data/lib/carray/frame.rb +2 -0
- data/lib/carray/fuse_source.rb +42 -6
- data/lib/carray/fusion.rb +40 -16
- data/lib/carray/histogram.rb +6 -1
- data/lib/carray/inspect.rb +31 -8
- data/lib/carray/iterator.rb +85 -7
- data/lib/carray/key_windows.rb +620 -0
- data/lib/carray/lazy.rb +217 -183
- data/lib/carray/mask_gap_fill.rb +63 -57
- data/lib/carray/meld_reduce.rb +12 -14
- data/lib/carray/methods/bin.rb +26 -18
- data/lib/carray/methods/bincount.rb +16 -0
- data/lib/carray/methods/bit_string.rb +1 -1
- data/lib/carray/methods/choose.rb +16 -3
- data/lib/carray/methods/composition.rb +3 -1
- data/lib/carray/methods/gather_nd.rb +14 -5
- data/lib/carray/methods/insert_block.rb +2 -2
- data/lib/carray/methods/is_between.rb +59 -0
- data/lib/carray/methods/is_in.rb +61 -37
- data/lib/carray/methods/join.rb +1 -6
- data/lib/carray/methods/lag.rb +49 -0
- data/lib/carray/methods/locate_addr.rb +47 -51
- data/lib/carray/methods/mode.rb +1 -1
- data/lib/carray/methods/nunique.rb +11 -6
- data/lib/carray/methods/resize.rb +3 -3
- data/lib/carray/methods/value_counts.rb +4 -11
- data/lib/carray/record_order.rb +3 -3
- data/lib/carray/rng.rb +3 -0
- data/lib/carray/runtime.rb +53 -8
- data/lib/carray/segment_iterator.rb +323 -88
- data/lib/carray/serialize.rb +45 -14
- data/lib/carray/slab_iterator.rb +50 -0
- data/lib/carray/stack.rb +15 -12
- data/lib/carray/string.rb +113 -5
- data/lib/carray/string_operation_extension.rb +21 -6
- data/lib/carray/struct.rb +9 -21
- data/lib/carray/struct_builder.rb +24 -11
- data/lib/carray/time.rb +819 -158
- data/lib/carray/window_iterator.rb +122 -16
- data/lib/carray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +4 -0
- data/yard-stubs/ca_obj_grid.rb +21 -10
- data/yard-stubs/ca_obj_refer.rb +2 -1
- data/yard-stubs/ca_obj_roll.rb +2 -2
- data/yard-stubs/ca_obj_shift.rb +5 -1
- data/yard-stubs/ca_obj_tile.rb +1 -1
- data/yard-stubs/ca_obj_transpose.rb +5 -4
- data/yard-stubs/ca_obj_window.rb +8 -3
- data/yard-stubs/carray_access.rb +6 -1
- data/yard-stubs/carray_attribute.rb +3 -6
- data/yard-stubs/carray_cast.rb +12 -0
- data/yard-stubs/carray_conversion.rb +7 -2
- data/yard-stubs/carray_copy.rb +4 -2
- data/yard-stubs/carray_count.rb +4 -4
- data/yard-stubs/carray_element.rb +17 -10
- data/yard-stubs/carray_generate.rb +21 -1
- data/yard-stubs/carray_mask.rb +32 -3
- data/yard-stubs/carray_math.rb +41 -5
- data/yard-stubs/carray_median_percentile.rb +5 -2
- data/yard-stubs/carray_memory_view.rb +40 -23
- data/yard-stubs/carray_order.rb +129 -22
- data/yard-stubs/carray_random.rb +13 -4
- data/yard-stubs/carray_scatter.rb +17 -10
- data/yard-stubs/carray_slab.rb +9 -2
- data/yard-stubs/carray_sort.rb +6 -5
- data/yard-stubs/carray_test.rb +5 -3
- metadata +25 -3
- data/lib/carray/arrow_tensor.rb +0 -401
- data/lib/carray/table.rb +0 -108
data/ext/ca_kernel_iterator.h
CHANGED
|
@@ -1,24 +1,46 @@
|
|
|
1
1
|
/* ---------------------------------------------------------------------------
|
|
2
2
|
|
|
3
|
-
ca_kernel_iterator.h
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
ca_kernel_iterator.h -- the kernel iterator, for C extension authors
|
|
4
|
+
|
|
5
|
+
A kernel written against this header receives the cells of any CArray
|
|
6
|
+
-- an entity, a view of a view, a selection, a lazy expression -- one
|
|
7
|
+
slab at a time, with the mask beside them, and on a write the cells it
|
|
8
|
+
changes reach the array. How the cells are delivered is the engine's
|
|
9
|
+
business (ca_kernel_iterator.c); the kernel only names the axes it
|
|
10
|
+
wants and walks what it is given.
|
|
11
|
+
|
|
12
|
+
Where to start:
|
|
13
|
+
- guides/devel/11_kernel_iterator.md: the surface and its contract,
|
|
14
|
+
with worked kernels
|
|
15
|
+
- docs/authoring/HOW_TO_WRITE_KERNEL.md: the reference, and which
|
|
16
|
+
macro to pick for a reduction, a map, a scan or a whole-array walk
|
|
17
|
+
- the block macros below (CA_FOR_EACH_SLAB / _FIBER / _WHOLE) are
|
|
18
|
+
the usual way in; the raw entry points (ca_iter_state_init_l1 /
|
|
19
|
+
init_l2 ...) are for kernels that handle the return codes
|
|
20
|
+
themselves
|
|
21
|
+
|
|
22
|
+
Freeze contract (3.0 onward). The surface is split
|
|
8
23
|
so the engine can be re-implemented across 3.x (the surface stays fixed)
|
|
9
|
-
without breaking ext-gem kernels.
|
|
10
|
-
|
|
24
|
+
without breaking ext-gem kernels. docs/authoring/HOW_TO_WRITE_KERNEL.md
|
|
25
|
+
states the contract in prose; utils/check_kernel_surface_freeze.rb
|
|
11
26
|
(rake kernel_surface_check) is the mechanical pin.
|
|
12
27
|
|
|
13
28
|
FROZEN (do NOT rename / re-arity / change semantics; additions only):
|
|
14
|
-
- the author macros
|
|
15
|
-
|
|
16
|
-
|
|
29
|
+
- the author macros: the CA_FOR_EACH_SLAB / _FIBER / _WHOLE
|
|
30
|
+
families (including _FIBER_PAIR / _FIBER_PAIR_MASKED), the
|
|
31
|
+
CA_SLAB_REDUCE_* / _MAP_* / _SCAN_* suites except the generator-
|
|
32
|
+
only macros listed under INTERNAL, CA_L2_FOR_EACH, and the mask
|
|
33
|
+
helper CA_FOR_EACH_UNMASKED
|
|
17
34
|
- the raw-API entry points the macros expand to: ca_iter_state_
|
|
18
|
-
|
|
19
|
-
close, ca_iter_check_init,
|
|
35
|
+
init_l1 / next_slab / init_l2 / init_l2_paired / next_slab_axes /
|
|
36
|
+
sync_slab / finish / close, ca_iter_check_init,
|
|
37
|
+
ca_iter_check_same_shape
|
|
38
|
+
- the author entry points beside them: ca_iter_ensure (a body that
|
|
39
|
+
can raise) and ca_iter_register_source_kind (an externally
|
|
40
|
+
installed view class)
|
|
20
41
|
- the enum/status tokens authors write literally: CA_SLAB_AXES,
|
|
21
|
-
CA_KERNEL_WRITE, CA_KERNEL_NO_MASK, CA_ITER_OK,
|
|
42
|
+
CA_SLAB_WHOLE, CA_KERNEL_WRITE, CA_KERNEL_NO_MASK, CA_ITER_OK,
|
|
43
|
+
CA_ITER_ERR_*
|
|
22
44
|
- the slab-delivery representation = the ca_iter_state fields a
|
|
23
45
|
kernel reads (marked "FROZEN author contract" at the struct):
|
|
24
46
|
slab_ndim / slab_dims / slab_strides / slab_mask_strides /
|
|
@@ -28,9 +50,15 @@
|
|
|
28
50
|
|
|
29
51
|
INTERNAL (free to refactor across 3.x — engine implementation):
|
|
30
52
|
- the state-machine functions' bodies + init_l1 / next_slab /
|
|
31
|
-
|
|
53
|
+
ca_iter_can_alias
|
|
54
|
+
- the macros only the kernel generator (ext/mkkernel.rb) emits:
|
|
55
|
+
CA_SLAB_REDUCE_SUM8_EX / MIN8_EX / MAX8_EX / STAR8_EX (8-way
|
|
56
|
+
accumulator split; they take EXPR(x) rather than exposing v and
|
|
57
|
+
idx), CA_SLAB_REDUCE_T_MIN_EX / _MAX_EX / _STAR_EX (what it emits
|
|
58
|
+
when a :min / :max / :prod kernel does not take the 8-way form),
|
|
59
|
+
CA_SLAB_SCAN_T_GATED, and CA_SLAB_REDUCE_T_VAR_EX (no caller)
|
|
32
60
|
- alias_mode / src_kind routing (CA_ITER_ALIAS_* / CA_ITER_SRC_* /
|
|
33
|
-
CA_KERNEL_FIBER_CONTIG / CA_SLAB_FREE
|
|
61
|
+
CA_KERNEL_FIBER_CONTIG / CA_SLAB_FREE)
|
|
34
62
|
- every ca_iter_state field NOT marked frozen (scratch / stack /
|
|
35
63
|
fiber / composed / descriptor bookkeeping) + physical layout
|
|
36
64
|
|
|
@@ -38,9 +66,9 @@
|
|
|
38
66
|
_EX macro variant + thin wrapper, new catalog macro) — that is how
|
|
39
67
|
the surface evolves within 3.x. Changing or removing a FROZEN name
|
|
40
68
|
is a 3.x breaking change: update the doc contract AND the guard in
|
|
41
|
-
the same commit.
|
|
42
|
-
|
|
43
|
-
|
|
69
|
+
the same commit. The engine behind these names may be rewritten
|
|
70
|
+
within 3.x; this contract is what keeps such a rewrite invisible to
|
|
71
|
+
kernel authors.
|
|
44
72
|
|
|
45
73
|
--------------------------------------------------------------------------- */
|
|
46
74
|
|
|
@@ -50,7 +78,7 @@
|
|
|
50
78
|
#include "carray.h"
|
|
51
79
|
#include "ca_iter_substrate.h" /* ca_axis_desc_t, ca_op_prefix_axis_t */
|
|
52
80
|
|
|
53
|
-
/* ---- slab policy
|
|
81
|
+
/* ---- slab policy ------------------------------------------------------ */
|
|
54
82
|
typedef enum {
|
|
55
83
|
CA_SLAB_FREE = 0, /* engine-chosen chunk. Not implemented:
|
|
56
84
|
init returns CA_ITER_ERR_POLICY. */
|
|
@@ -58,57 +86,84 @@ typedef enum {
|
|
|
58
86
|
CA_SLAB_WHOLE = 2 /* whole array in one slab */
|
|
59
87
|
} ca_slab_policy_t;
|
|
60
88
|
|
|
61
|
-
/* ---- kernel flags
|
|
89
|
+
/* ---- kernel flags ------------------------------------------------------ */
|
|
62
90
|
#define CA_KERNEL_READ 0x0
|
|
63
|
-
#define CA_KERNEL_WRITE 0x1 /*
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
91
|
+
#define CA_KERNEL_WRITE 0x1 /* the kernel writes the slab back.
|
|
92
|
+
The slab is read first (so a
|
|
93
|
+
kernel may change only some cells),
|
|
94
|
+
and a view that cannot be read --
|
|
95
|
+
CAFake of an object array whose
|
|
96
|
+
cells do not convert -- raises
|
|
97
|
+
there, even for a kernel that
|
|
98
|
+
writes every cell */
|
|
99
|
+
#define CA_KERNEL_NO_MASK 0x2 /* the kernel cannot take a mask:
|
|
100
|
+
a masked source is refused */
|
|
101
|
+
#define CA_KERNEL_CHUNK_HINT 0x4 /* reserved: init refuses it
|
|
102
|
+
with CA_ITER_ERR_FLAGS */
|
|
103
|
+
#define CA_KERNEL_FIBER_CONTIG 0x8 /* with policy=CA_SLAB_AXES and
|
|
104
|
+
naxes==1, next_slab_axes hands
|
|
105
|
+
out every fiber contiguous
|
|
106
|
+
(gathering a strided one into
|
|
71
107
|
fiber_data_scratch). Set by the
|
|
72
|
-
CA_FOR_EACH_FIBER macro family
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
/*
|
|
80
|
-
#define
|
|
81
|
-
|
|
82
|
-
#define CA_ITER_ALIAS_STRIDED 2 /* L2 stride-aware. step 3+. */
|
|
108
|
+
CA_FOR_EACH_FIBER macro family.
|
|
109
|
+
Direct callers that walk the
|
|
110
|
+
slab with slab_strides[0]
|
|
111
|
+
themselves leave it clear. */
|
|
112
|
+
|
|
113
|
+
/* ---- alias mode: where the slabs are cut from (internal) --------------- */
|
|
114
|
+
#define CA_ITER_ALIAS_NONE 0 /* a scratch the walk gathered */
|
|
115
|
+
#define CA_ITER_ALIAS_CONTIG 1 /* the source's own memory */
|
|
116
|
+
#define CA_ITER_ALIAS_STRIDED 2 /* the root's or parent's memory,
|
|
117
|
+
through composed strides */
|
|
83
118
|
#define CA_ITER_ALIAS_ATTACH 3 /* Not set by the engine. A
|
|
84
119
|
SRC_ATTACH walk runs with
|
|
85
120
|
CA_ITER_ALIAS_NONE over an
|
|
86
|
-
iterator-owned scratch filled
|
|
87
|
-
written back with
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
121
|
+
iterator-owned scratch, filled with
|
|
122
|
+
ca_xfer_all and written back with
|
|
123
|
+
ki_scratch_push_changed (only the
|
|
124
|
+
changed cells). */
|
|
125
|
+
#define CA_ITER_ALIAS_PER_SLAB 4 /* a descriptor view read one slab at
|
|
126
|
+
a time: ca_axis_dispatch_gather
|
|
127
|
+
over a subset of the descriptors
|
|
128
|
+
with the outer axes pinned, into
|
|
129
|
+
one slab-sized scratch refilled
|
|
130
|
+
in next_slab_axes. Read walks
|
|
131
|
+
only; a write walk takes the
|
|
132
|
+
transfer walk instead. */
|
|
133
|
+
#define CA_ITER_ALIAS_PER_SLAB_HOIST 5 /* PER_SLAB when the innermost slab
|
|
134
|
+
axis is STRIDE and no axis is
|
|
135
|
+
SHIFT: the outer and
|
|
136
|
+
non-innermost slab offsets are
|
|
137
|
+
computed once per slab row and
|
|
138
|
+
the inner run is a memcpy. */
|
|
139
|
+
#define CA_ITER_ALIAS_PER_FIBER_FUSED 6 /* each fiber is fetched (and on a
|
|
140
|
+
write put back) with the view's
|
|
141
|
+
ca_xfer_stride, without
|
|
142
|
+
gathering the whole view: a
|
|
143
|
+
CAWindow / CAShift fiber, or a
|
|
144
|
+
read of the innermost fiber of
|
|
145
|
+
a conversion view. Chosen by
|
|
146
|
+
ca_iter_should_per_fiber_fused.
|
|
147
|
+
The fiber is handed out from
|
|
148
|
+
fiber_data_scratch, its mask
|
|
149
|
+
from fiber_mask_scratch. */
|
|
150
|
+
#define CA_ITER_ALIAS_STACK 7 /* a CAStack read with slab_axes ==
|
|
151
|
+
[k_axis] (the stack axis alone,
|
|
152
|
+
e.g. view.mean(axis: k_axis)).
|
|
153
|
+
init_l2 attaches the K parents
|
|
154
|
+
and caches their pointers and
|
|
155
|
+
their common native strides;
|
|
156
|
+
next_slab_axes hands out each
|
|
157
|
+
fiber's K cells from a tile cache
|
|
158
|
+
(K * tile * bytes, tile 8..64
|
|
159
|
+
fibers) refilled from the parents'
|
|
160
|
+
memory, without ca_xfer_stride.
|
|
161
|
+
Only for slab_axes == [k_axis]
|
|
162
|
+
and an unmasked stack; any other
|
|
163
|
+
stack walk takes STACK_OUTER_K or
|
|
164
|
+
the transfer walk. Delivery
|
|
165
|
+
through ca_xfer_stride was tried
|
|
166
|
+
and measured ~16x slower. */
|
|
112
167
|
#define CA_ITER_ALIAS_STACK_OUTER_K 8 /* CAStack source + CA_SLAB_AXES
|
|
113
168
|
whose slab leaves out the stack
|
|
114
169
|
axis (k_axis), which is then an
|
|
@@ -125,61 +180,18 @@ typedef enum {
|
|
|
125
180
|
that is not contiguous in its
|
|
126
181
|
parent is gathered (FIBER_CONTIG).
|
|
127
182
|
READ only. */
|
|
128
|
-
#define CA_ITER_ALIAS_STACK 7 /* PROPOSAL_CASTACK_LOOP_INTERCHANGE
|
|
129
|
-
Vector A rev2 (direct per-parent
|
|
130
|
-
ptr access): CAStack source +
|
|
131
|
-
CA_SLAB_AXES with slab_axes ==
|
|
132
|
-
[k_axis] (= the stack axis alone,
|
|
133
|
-
e.g. view.mean(axis: k_axis)).
|
|
134
|
-
init_l2 attaches K parents up
|
|
135
|
-
front (= O(1) per entity parent),
|
|
136
|
-
caches parent->ptr aliases +
|
|
137
|
-
uniform parent-native byte
|
|
138
|
-
strides. next_slab_axes does
|
|
139
|
-
K-fold direct memcpy gather from
|
|
140
|
-
parents[k]->ptr + parent_off
|
|
141
|
-
into a slab-sized scratch,
|
|
142
|
-
bypassing ca_xfer_stride
|
|
143
|
-
entirely (= no per-call dispatch
|
|
144
|
-
/ cyclic_check / strided_walk
|
|
145
|
-
function-boundary overhead).
|
|
146
|
-
Peak buffer = K * bytes
|
|
147
|
-
(= slab footprint), not
|
|
148
|
-
K * parent.elements * bytes.
|
|
149
|
-
Scope-narrow to slab_axes == [0]:
|
|
150
|
-
arbitrary slab shapes require
|
|
151
|
-
per-cell index decode which
|
|
152
|
-
loses the inner contig fast
|
|
153
|
-
path; covers the demand-driving
|
|
154
|
-
case (= reduce along the stacked
|
|
155
|
-
K-axis). Mask-bearing CAStack
|
|
156
|
-
falls back to SRC_ATTACH whole-
|
|
157
|
-
view path (per-slab mask gather
|
|
158
|
-
= future extension). Rev1
|
|
159
|
-
explored xfer_stride-based
|
|
160
|
-
delivery (see PROPOSAL
|
|
161
|
-
rev2 §11) and was rejected for
|
|
162
|
-
~16x wall-clock regression. */
|
|
163
|
-
#define CA_ITER_ALIAS_PER_SLAB_HOIST 5 /* Phase C T3 specialised (B-1b,
|
|
164
|
-
C.1b): innermost slab axis is
|
|
165
|
-
STRIDE and no SHIFT axes
|
|
166
|
-
anywhere. Manual gather:
|
|
167
|
-
outer + non-innermost-slab axes
|
|
168
|
-
hoisted (computed once per slab
|
|
169
|
-
row), inner = pure STRIDE
|
|
170
|
-
linear memcpy (= no engine
|
|
171
|
-
per-cell switch, SIMD-friendly
|
|
172
|
-
contig run). Target: 1.5-1.8x
|
|
173
|
-
win vs (A) fallback for INDEX
|
|
174
|
-
slab with innermost STRIDE. */
|
|
175
183
|
|
|
176
184
|
/* ---- error codes ----------------------------------------------------- */
|
|
177
185
|
#define CA_ITER_OK 0
|
|
178
186
|
#define CA_ITER_ERR_NOT_CHEAP 1 /* the classifier does not recognise
|
|
179
187
|
the source (a view with no attach
|
|
180
188
|
slot; any other external obj_type
|
|
181
|
-
is delivered by
|
|
182
|
-
|
|
189
|
+
is delivered by the transfer walk,
|
|
190
|
+
through its xfer slots) */
|
|
191
|
+
#define CA_ITER_ERR_POLICY 2 /* policy or axes refused: a policy
|
|
192
|
+
not implemented, axes NULL / out
|
|
193
|
+
of range / duplicated, AXES at
|
|
194
|
+
L1, descriptor rank != view rank */
|
|
183
195
|
#define CA_ITER_ERR_FLAGS 3 /* flag combination unsupported;
|
|
184
196
|
also returned for a NULL state
|
|
185
197
|
or source */
|
|
@@ -187,9 +199,9 @@ typedef enum {
|
|
|
187
199
|
#define CA_ITER_ERR_MASK 5 /* reserved: no entry point returns
|
|
188
200
|
it (masked sources are accepted) */
|
|
189
201
|
#define CA_ITER_ERR_MASK_NOT_ALLOWED 6 /* NO_MASK flag set on a masked source */
|
|
190
|
-
/* ---- source kind (
|
|
202
|
+
/* ---- source kind (internal routing) ------------------------------------ */
|
|
191
203
|
#define CA_ITER_SRC_NONE 0
|
|
192
|
-
#define CA_ITER_SRC_CASTRIDE 1 /* entity / CAStride family
|
|
204
|
+
#define CA_ITER_SRC_CASTRIDE 1 /* entity / CAStride family */
|
|
193
205
|
#define CA_ITER_SRC_DESCRIPTOR 2 /* CASelectAxis / CAGrid / CASelect /
|
|
194
206
|
CAWindow / CAShift: per-axis
|
|
195
207
|
descriptors (describe_axes) */
|
|
@@ -200,32 +212,31 @@ typedef enum {
|
|
|
200
212
|
registered external obj_type: the
|
|
201
213
|
view's own transfer gathers it */
|
|
202
214
|
#define CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE 4
|
|
203
|
-
/*
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
is yielded with inner_byte_stride). Only
|
|
209
|
-
emitted by ca_iter_route_source after
|
|
210
|
-
describe_axes inspection. */
|
|
215
|
+
/* a descriptor view whose innermost axis
|
|
216
|
+
is STRIDE kind, so an L2 walk can read
|
|
217
|
+
the parent's memory in place. Only
|
|
218
|
+
returned by ca_iter_route_source,
|
|
219
|
+
after describe_axes. */
|
|
211
220
|
|
|
212
221
|
/* ---- source-kind registration for externally installed obj_types -----
|
|
213
222
|
|
|
214
223
|
The classifier recognises the core's own view classes by comparing
|
|
215
224
|
their operation table against a list compiled into the engine. A view
|
|
216
225
|
class installed by a companion gem through ca_install_obj_type matches
|
|
217
|
-
nothing on that list; it is delivered as CA_ITER_SRC_ATTACH (
|
|
218
|
-
|
|
219
|
-
class's Init:
|
|
226
|
+
nothing on that list; it is delivered as CA_ITER_SRC_ATTACH (gathered
|
|
227
|
+
through the view's own transfer), and so is one that registers here,
|
|
228
|
+
once, in the class's Init:
|
|
220
229
|
|
|
221
230
|
ca_iter_register_source_kind(CA_OBJ_MY_VIEW, CA_ITER_SRC_ATTACH);
|
|
222
231
|
|
|
223
232
|
Registering states the routing rather than changing it.
|
|
224
233
|
CA_ITER_SRC_ATTACH is the only kind that may be registered. It is the
|
|
225
|
-
one whose contract an external class can meet on its own:
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
234
|
+
one whose contract an external class can meet on its own: the engine
|
|
235
|
+
reads the cells with ca_xfer_all GET and, on a WRITE walk, sends back
|
|
236
|
+
the changed cells with ca_xfer_addrs (or ca_xfer_all PUT when every
|
|
237
|
+
cell changed), so the view's transfer slots are all it needs -- the
|
|
238
|
+
slots every view already implements. The view's attach / sync slots
|
|
239
|
+
are not called. The other kinds are not open to registration: SRC_CASTRIDE
|
|
229
240
|
asserts the struct *is* a CAStride (the engine reads its strides
|
|
230
241
|
directly, and a class that really is one is already classified by its
|
|
231
242
|
inherited operation table), and SRC_DESCRIPTOR requires a
|
|
@@ -239,18 +250,18 @@ typedef enum {
|
|
|
239
250
|
materialising path), and consults this table before the built-in list. */
|
|
240
251
|
void ca_iter_register_source_kind (int obj_type, uint8_t kind);
|
|
241
252
|
|
|
242
|
-
/* ---- iter state
|
|
243
|
-
/*
|
|
244
|
-
|
|
245
|
-
|
|
253
|
+
/* ---- iter state ------------------------------------------------------- */
|
|
254
|
+
/* The state lives on the ext author's stack, so its layout is fixed: a
|
|
255
|
+
field is never removed, and the ones no path uses any more (chunk_pos,
|
|
256
|
+
alias_stride) stay as zero-initialised slots. */
|
|
246
257
|
typedef struct {
|
|
247
258
|
/* --- inputs (fixed at init) --- */
|
|
248
259
|
struct _CArray *src;
|
|
249
|
-
uint8_t src_kind; /* CA_ITER_SRC_*
|
|
250
|
-
int8_t level; /* 1=L1 contig, 2=L2 strided
|
|
260
|
+
uint8_t src_kind; /* CA_ITER_SRC_* */
|
|
261
|
+
int8_t level; /* 1=L1 contig, 2=L2 strided */
|
|
251
262
|
ca_slab_policy_t policy;
|
|
252
263
|
int8_t ndim;
|
|
253
|
-
int8_t naxes; /*
|
|
264
|
+
int8_t naxes; /* AXES policy axis count */
|
|
254
265
|
void *pair; /* the other state of a two-state
|
|
255
266
|
walk (ca_iter_state_init_l2_paired),
|
|
256
267
|
else NULL; finished with this one
|
|
@@ -258,15 +269,20 @@ typedef struct {
|
|
|
258
269
|
uint32_t flags;
|
|
259
270
|
ca_size_t bytes; /* element size */
|
|
260
271
|
|
|
261
|
-
/* --- CAStride compose-fold cache (
|
|
272
|
+
/* --- CAStride compose-fold cache (L2 alias path) ---
|
|
262
273
|
For CAStride-family sources at L2, init_l2 runs
|
|
263
|
-
ca_stride_compose_to_root once to get the root
|
|
264
|
-
byte strides + base offset
|
|
265
|
-
|
|
266
|
-
Inline CA_RANK_MAX array (no
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
274
|
+
ca_stride_compose_to_root once to get the root + per-axis
|
|
275
|
+
byte strides + base offset, kept here for the author to read. The
|
|
276
|
+
walk itself steps from composed_base by outer_strides (below), which
|
|
277
|
+
init builds from the composed strides. Inline CA_RANK_MAX array (no
|
|
278
|
+
heap alloc). root is the array attached at
|
|
279
|
+
init whose memory the walk reads: the compose-fold root of an L2
|
|
280
|
+
CAStride walk (an entity, or a view that lends memory) or the parent
|
|
281
|
+
of a descriptor view. NULL when the walk reads src itself or a
|
|
282
|
+
scratch of its own. */
|
|
283
|
+
struct _CArray *root; /* attached at init, detached (or
|
|
284
|
+
synced and detached, on WRITE)
|
|
285
|
+
by finish */
|
|
270
286
|
ca_size_t composed_strides[CA_RANK_MAX]; /* byte units */
|
|
271
287
|
ca_size_t composed_base; /* byte offset from root->ptr */
|
|
272
288
|
|
|
@@ -288,30 +304,34 @@ typedef struct {
|
|
|
288
304
|
ca_size_t total_elements; /* view.elements snapshot */
|
|
289
305
|
|
|
290
306
|
/* --- iteration cursor --- */
|
|
291
|
-
ca_size_t *outer_idx; /* [
|
|
307
|
+
ca_size_t *outer_idx; /* [outer_ndim] */
|
|
292
308
|
ca_size_t slab_n; /* current slab element count */
|
|
293
309
|
ca_size_t total_slabs;
|
|
294
310
|
ca_size_t slabs_emitted;
|
|
295
311
|
|
|
296
|
-
/* --- chunk position (
|
|
312
|
+
/* --- chunk position (chunk_pos is unused, kept for layout) --- */
|
|
297
313
|
ca_size_t chunk_pos;
|
|
298
314
|
ca_size_t chunk_size;
|
|
299
315
|
|
|
300
|
-
/* --- scratch buffer
|
|
316
|
+
/* --- scratch buffer --- */
|
|
301
317
|
char *scratch_ptr;
|
|
302
318
|
boolean8_t *scratch_mask;
|
|
303
319
|
ca_size_t scratch_cap;
|
|
304
320
|
|
|
305
321
|
/* --- alias path --- */
|
|
306
322
|
uint8_t alias_mode; /* CA_ITER_ALIAS_* */
|
|
307
|
-
char *alias_ptr; /*
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
323
|
+
char *alias_ptr; /* base the slabs are yielded from:
|
|
324
|
+
the source or root memory, or the
|
|
325
|
+
scratch under ALIAS_NONE; NULL for
|
|
326
|
+
STACK / STACK_OUTER_K /
|
|
327
|
+
PER_FIBER_FUSED */
|
|
328
|
+
boolean8_t *alias_mask;
|
|
329
|
+
ca_size_t alias_stride; /* unused, kept for layout */
|
|
330
|
+
|
|
331
|
+
/* --- WRITE-path sync timing --- */
|
|
312
332
|
uint8_t write_dirty;
|
|
313
333
|
|
|
314
|
-
/* --- CA_SLAB_AXES policy fields
|
|
334
|
+
/* --- CA_SLAB_AXES policy fields ---
|
|
315
335
|
Populated when init_l2 is called with policy = CA_SLAB_AXES, for
|
|
316
336
|
every kind of source. The kernel reads slab metadata directly
|
|
317
337
|
from these fields (= per-walk metadata, unchanged across slabs).
|
|
@@ -320,8 +340,8 @@ typedef struct {
|
|
|
320
340
|
handed to the kernel. All strides are byte units. Zero-init for
|
|
321
341
|
other policies (CA_SLAB_WHOLE / FREE).
|
|
322
342
|
|
|
323
|
-
|
|
324
|
-
the
|
|
343
|
+
The slab axis list is "slab_axes_buf" (an inline array); the axes
|
|
344
|
+
the caller passes to init are not kept in the state.
|
|
325
345
|
|
|
326
346
|
>>> FROZEN author contract (see banner, two-tier freeze): the
|
|
327
347
|
slab-delivery representation a kernel reads is exactly
|
|
@@ -343,10 +363,11 @@ typedef struct {
|
|
|
343
363
|
ca_size_t outer_strides[CA_RANK_MAX]; /* per-outer-axis data byte stride */
|
|
344
364
|
ca_size_t outer_mask_strides[CA_RANK_MAX]; /* per-outer-axis mask element stride */
|
|
345
365
|
|
|
346
|
-
/* --- Per-fiber contig scratch
|
|
366
|
+
/* --- Per-fiber contig scratch ---
|
|
347
367
|
For naxes==1 (= per-axis fiber, the catalog CA_FOR_EACH_FIBER target)
|
|
348
|
-
|
|
349
|
-
|
|
368
|
+
on the paths of next_slab_axes that yield a fiber in place (the
|
|
369
|
+
default fall-through and STACK_OUTER_K; ki_fiber_contig_data), when
|
|
370
|
+
slab_strides[0] != bytes the fiber
|
|
350
371
|
is yielded strided. To honor the catalog contract "data contig
|
|
351
372
|
delivery", next_slab_axes lazily allocates this scratch on first
|
|
352
373
|
gather and gathers each fiber here before yielding. Reused across
|
|
@@ -364,7 +385,7 @@ typedef struct {
|
|
|
364
385
|
ca_size_t fiber_data_scratch_cap;
|
|
365
386
|
ca_size_t last_data_off;
|
|
366
387
|
|
|
367
|
-
/*
|
|
388
|
+
/* Per-fiber contig mask scratch.
|
|
368
389
|
Symmetric to fiber_data_scratch. When the source carries a mask
|
|
369
390
|
(= alias_mask != NULL) and slab_mask_strides[0] != 1, the engine
|
|
370
391
|
gathers the fiber's mask into contig boolean8_t order here so the
|
|
@@ -375,7 +396,7 @@ typedef struct {
|
|
|
375
396
|
boolean8_t *fiber_mask_scratch;
|
|
376
397
|
ca_size_t fiber_mask_scratch_cap;
|
|
377
398
|
|
|
378
|
-
/* ---
|
|
399
|
+
/* --- PER_FIBER_FUSED fields ---
|
|
379
400
|
When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, next_slab_axes
|
|
380
401
|
builds a fiber region from outer_idx + fiber_axis and calls
|
|
381
402
|
ca_xfer_stride(src, ..., GET) into fiber_data_scratch instead of
|
|
@@ -387,23 +408,22 @@ typedef struct {
|
|
|
387
408
|
ca_size_t fiber_native_strides[CA_RANK_MAX];
|
|
388
409
|
/* fiber_region_starts[] cached by next_slab_axes BEFORE outer_idx
|
|
389
410
|
advance so sync_slab can reconstruct the same ca_xfer_stride
|
|
390
|
-
region for WRITE PUT.
|
|
391
|
-
(= F.1a). */
|
|
411
|
+
region for WRITE PUT. The same hazard as last_data_off. */
|
|
392
412
|
ca_size_t fiber_region_starts[CA_RANK_MAX];
|
|
393
413
|
|
|
394
|
-
/* ---
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
going through ca_xfer_stride. Owned by iter (xfree in finish).
|
|
414
|
+
/* --- CAStack read from its parents (STACK / STACK_OUTER_K) --- */
|
|
415
|
+
/* When alias_mode is CA_ITER_ALIAS_STACK or STACK_OUTER_K, init_l2
|
|
416
|
+
attaches all K parents (= O(1) per entity parent) and caches their
|
|
417
|
+
ptr aliases here so next_slab_axes reads them without going through
|
|
418
|
+
ca_xfer_stride. Owned by iter (xfree in finish).
|
|
400
419
|
stack_parent_strides[] are the uniform parent-native byte strides
|
|
401
|
-
(
|
|
420
|
+
(all the parents of a CAStack have one shape). */
|
|
402
421
|
char **stack_parent_ptrs; /* [n_parents] */
|
|
403
422
|
int32_t stack_n_parents; /* = ((CAStack *)src)->n_parents */
|
|
404
423
|
ca_size_t stack_parent_strides[CA_RANK_MAX]; /* parent-space byte strides */
|
|
405
424
|
/* Parent-space element strides (= 1 byte per cell) for parent mask
|
|
406
|
-
addressing;
|
|
425
|
+
addressing; filled by the STACK_OUTER_K init whether or not the
|
|
426
|
+
stack has a mask. Caching
|
|
407
427
|
here lets next_slab_axes compute mask_off without downcasting to
|
|
408
428
|
CAStack (= AC3 layering goal). */
|
|
409
429
|
ca_size_t stack_parent_mask_strides[CA_RANK_MAX];
|
|
@@ -416,7 +436,6 @@ typedef struct {
|
|
|
416
436
|
when alias_mode == CA_ITER_ALIAS_STACK_OUTER_K; next_slab_axes uses
|
|
417
437
|
outer_idx[stack_k_outer_pos] to pick the active parent. */
|
|
418
438
|
int8_t stack_k_outer_pos;
|
|
419
|
-
/* --- pilot/castack-axis0-loop-interchange (2026-06-19) --- */
|
|
420
439
|
/* CA_ITER_ALIAS_STACK tile cache. Refills TILE fibers (= K cells each)
|
|
421
440
|
at once via K contig parent reads, then serves the next TILE next_slab
|
|
422
441
|
calls from L1d-resident buffer. Layout: cache[t][k] so a fiber at
|
|
@@ -429,10 +448,9 @@ typedef struct {
|
|
|
429
448
|
ca_size_t stack_tile_have; /* fibers actually present in current tile */
|
|
430
449
|
} ca_iter_state;
|
|
431
450
|
|
|
432
|
-
/* ---- alias eligibility predicate
|
|
451
|
+
/* ---- alias eligibility predicate ---------------------------------------- */
|
|
433
452
|
|
|
434
|
-
/* Generalises ca_attach_is_alias (carray_core.c
|
|
435
|
-
ca_attach_is_cheap in T1 step 9.4a) so the alias decision can be
|
|
453
|
+
/* Generalises ca_attach_is_alias (carray_core.c) so the alias decision can be
|
|
436
454
|
made level-aware. Level is the dispatch level the caller intends
|
|
437
455
|
to use:
|
|
438
456
|
|
|
@@ -446,15 +464,17 @@ typedef struct {
|
|
|
446
464
|
Alias iff the engine can yield per-outer-axis slabs as
|
|
447
465
|
parent->ptr + offset with a native stride_bytes argument, with
|
|
448
466
|
no scratch allocation. Broader than L1: any CAStride-family
|
|
449
|
-
view qualifies
|
|
467
|
+
view qualifies by its strides (init_l2 still gathers into scratch
|
|
468
|
+
when the root it folds to lends no memory, which this predicate
|
|
469
|
+
does not look at) (the innermost run, even a stride-of-1 of count 1,
|
|
450
470
|
defines a valid strided slab). Entity arrays also qualify
|
|
451
471
|
trivially. Pathological all-strided-no-contig sources still
|
|
452
472
|
qualify here — the engine yields slab_n=1 with the native step;
|
|
453
473
|
L1 fallback is an engine-policy choice, not an eligibility one.
|
|
454
474
|
|
|
455
475
|
level == 3 (L3, multi-d kernel)
|
|
456
|
-
Not implemented
|
|
457
|
-
|
|
476
|
+
Not implemented — answered as L1 so the predicate stays
|
|
477
|
+
well-defined for callers that probe ahead.
|
|
458
478
|
|
|
459
479
|
Descriptor views (CASelectAxis / CAGrid / CASelect / CAWindow /
|
|
460
480
|
CAShift) and conversion views (CAFake / CAByteSwap / CABitfield /
|
|
@@ -463,13 +483,12 @@ typedef struct {
|
|
|
463
483
|
(CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE); this predicate does not
|
|
464
484
|
report that case.
|
|
465
485
|
|
|
466
|
-
ca_attach_is_alias
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
should call ca_iter_can_alias with an explicit level. */
|
|
486
|
+
ca_attach_is_alias stays the level=1 answer for its existing callers;
|
|
487
|
+
new code in the kernel iterator path should call ca_iter_can_alias
|
|
488
|
+
with an explicit level. */
|
|
470
489
|
int ca_iter_can_alias (void *ap, int level);
|
|
471
490
|
|
|
472
|
-
/* ---- state machine
|
|
491
|
+
/* ---- state machine ------------------------------------------------------ */
|
|
473
492
|
|
|
474
493
|
/* Initialise `st` for an **L1 (contig kernel)** walk of `src`.
|
|
475
494
|
|
|
@@ -478,11 +497,12 @@ int ca_iter_can_alias (void *ap, int level);
|
|
|
478
497
|
src->ptr, stride implicit = bytes)
|
|
479
498
|
- CAStride family non-contig: scratch path (ca_copy_data
|
|
480
499
|
compose-fold gather into a malloc'd buffer)
|
|
481
|
-
- descriptor views (CAGrid / CASelect /
|
|
482
|
-
|
|
483
|
-
- SRC_ATTACH views (overlays, lazy views, CAObject, CAStack,
|
|
484
|
-
iterator-owned scratch
|
|
485
|
-
|
|
500
|
+
- descriptor views (CAGrid / CASelect / CAWindow / CAShift /
|
|
501
|
+
CASelectAxis): gathered into a scratch buffer
|
|
502
|
+
- SRC_ATTACH views (overlays, lazy views, CAObject, CAStack, ...,
|
|
503
|
+
and any other view with an attach slot): iterator-owned scratch
|
|
504
|
+
filled through ca_xfer_all
|
|
505
|
+
- a source with no attach slot: CA_ITER_ERR_NOT_CHEAP
|
|
486
506
|
|
|
487
507
|
policy: CA_SLAB_WHOLE only (CA_SLAB_AXES and CA_SLAB_FREE return
|
|
488
508
|
CA_ITER_ERR_POLICY).
|
|
@@ -503,28 +523,31 @@ int ca_iter_state_init_l1 (ca_iter_state *st,
|
|
|
503
523
|
/* Initialise `st` for an **L2 (strided kernel)** walk of `src`.
|
|
504
524
|
|
|
505
525
|
Source routing:
|
|
506
|
-
- entity / CAStride contig: alias_mode = CONTIG,
|
|
507
|
-
|
|
508
|
-
- CAStride non-contig: alias_mode = STRIDED,
|
|
509
|
-
|
|
510
|
-
scratch.
|
|
526
|
+
- entity / CAStride contig: alias_mode = CONTIG, the slab reads
|
|
527
|
+
the source's own memory
|
|
528
|
+
- CAStride non-contig: alias_mode = STRIDED, the slab reads the
|
|
529
|
+
root through the composed strides. No scratch.
|
|
511
530
|
- descriptor views: aliased when the innermost axis is a STRIDE
|
|
512
|
-
axis, otherwise gathered into scratch (per slab
|
|
513
|
-
|
|
531
|
+
axis, otherwise gathered into scratch (per slab when a slab axis
|
|
532
|
+
is an INDEX or SHIFT axis)
|
|
514
533
|
- SRC_ATTACH views: iterator-owned scratch filled through
|
|
515
|
-
ca_xfer_all
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
534
|
+
ca_xfer_all, written back with ki_scratch_push_changed (only the
|
|
535
|
+
changed cells). A CAStack may instead be walked through its
|
|
536
|
+
parents, and a single FIBER_CONTIG fiber of a cast / byte-swap
|
|
537
|
+
view through ca_xfer_stride (PER_FIBER_FUSED)
|
|
538
|
+
- a source with no attach slot: CA_ITER_ERR_NOT_CHEAP
|
|
539
|
+
|
|
540
|
+
policy: CA_SLAB_AXES only. CA_SLAB_WHOLE and CA_SLAB_FREE return
|
|
541
|
+
CA_ITER_ERR_POLICY, as do out-of-range or duplicate axes,
|
|
542
|
+
and a descriptor view whose descriptor rank differs from the
|
|
543
|
+
view's rank. The whole array as one contiguous run is L1
|
|
544
|
+
(init_l1 with CA_SLAB_WHOLE); one strided run per innermost
|
|
545
|
+
row is CA_SLAB_AXES with the last axis.
|
|
546
|
+
flags: as init_l1. A CA_KERNEL_WRITE walk over a descriptor view
|
|
547
|
+
whose slab has an INDEX or SHIFT axis goes by the transfer
|
|
548
|
+
walk.
|
|
549
|
+
|
|
550
|
+
Pair with ca_iter_state_next_slab_axes. */
|
|
528
551
|
int ca_iter_state_init_l2 (ca_iter_state *st,
|
|
529
552
|
struct _CArray *src,
|
|
530
553
|
ca_slab_policy_t policy,
|
|
@@ -540,11 +563,12 @@ int ca_iter_state_init_l2 (ca_iter_state *st,
|
|
|
540
563
|
*out_mask is set to the per-slab boolean8_t mask pointer when the
|
|
541
564
|
source carries a mask (= ca_has_mask(src)), or NULL otherwise.
|
|
542
565
|
The mask layout matches the value layout (= same iteration order
|
|
543
|
-
and same n).
|
|
544
|
-
|
|
566
|
+
and same n). Kernels use the CA_FOR_EACH_UNMASKED macro family
|
|
567
|
+
(below, in this header) to skip masked cells.
|
|
545
568
|
|
|
546
|
-
The mask is an input. What it points at is a copy taken when the
|
|
547
|
-
walk began,
|
|
569
|
+
The mask is an input. What it points at is a copy -- taken when the
|
|
570
|
+
walk began, or, on the per-fiber fused path, gathered for each fiber
|
|
571
|
+
-- so writing through it changes nothing -- not even under
|
|
548
572
|
CA_KERNEL_WRITE, where the value half of the same yield is often a
|
|
549
573
|
live alias. It also cannot express the thing an author would most
|
|
550
574
|
want it for: an unmasked source yields NULL, so there is nowhere to
|
|
@@ -564,19 +588,6 @@ int ca_iter_state_next_slab (ca_iter_state *st,
|
|
|
564
588
|
boolean8_t **out_mask,
|
|
565
589
|
ca_size_t *out_n);
|
|
566
590
|
|
|
567
|
-
/* Pull the next L2 strided slab. Returns 1 with *out_ptr /
|
|
568
|
-
*out_mask / *out_n / *out_stride_bytes when a slab is yielded;
|
|
569
|
-
returns 0 when the walk is complete. The kernel walks `*out_n`
|
|
570
|
-
elements by stepping `*out_stride_bytes` between consecutive
|
|
571
|
-
elements starting at `*out_ptr`. The mask, when non-NULL, is
|
|
572
|
-
contiguous whatever the value stride: mask[i] pairs with the value
|
|
573
|
-
at ptr + i * stride_bytes. Only valid when init_l2 was used. */
|
|
574
|
-
int ca_iter_state_next_slab_strided (ca_iter_state *st,
|
|
575
|
-
char **out_ptr,
|
|
576
|
-
boolean8_t **out_mask,
|
|
577
|
-
ca_size_t *out_n,
|
|
578
|
-
ca_size_t *out_stride_bytes);
|
|
579
|
-
|
|
580
591
|
/* Pull the next CA_SLAB_AXES slab (K-D block). Returns 1 with *out_ptr
|
|
581
592
|
/ *out_mask set to the slab base when a slab is yielded; returns 0
|
|
582
593
|
when the walk is complete. Only valid when init_l2 was called with
|
|
@@ -613,21 +624,39 @@ int ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
|
613
624
|
/* Sync the just-yielded slab back to parent (WRITE path; READ no-op).
|
|
614
625
|
|
|
615
626
|
Caller calls this **unconditionally** after each next_slab /
|
|
616
|
-
|
|
627
|
+
next_slab_axes + kernel invocation; the alias_mode branch lives
|
|
617
628
|
inside the state machine so the caller never has to inspect it.
|
|
618
629
|
|
|
619
630
|
Behaviour:
|
|
620
631
|
- !(flags & CA_KERNEL_WRITE): no-op (READ walk)
|
|
621
632
|
- per-fiber gather (CA_KERNEL_FIBER_CONTIG on a strided fiber):
|
|
622
633
|
the fiber is scattered back from its scratch first
|
|
623
|
-
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
634
|
+
- CAStack walked through its parents: READ only, no-op
|
|
635
|
+
- per-fiber fused (CA_ITER_ALIAS_PER_FIBER_FUSED): the fiber is put
|
|
636
|
+
back through the view with ca_xfer_stride
|
|
637
|
+
- alias path (scratch_ptr == NULL): no-op — kernel wrote through
|
|
638
|
+
alias_ptr. When the alias is into a view root's own buffer, that
|
|
639
|
+
buffer is written back once, by finish (ca_sync_detach)
|
|
640
|
+
- scratch path, at L1 or L2: the cells the kernel changed are sent
|
|
641
|
+
back through the source (ki_scratch_push_changed: ca_xfer_addrs,
|
|
642
|
+
or ca_xfer_all PUT when every cell changed); a write over a
|
|
643
|
+
descriptor view takes this path, opened as a transfer walk
|
|
644
|
+
|
|
645
|
+
A view that shows one root cell more than once (a repeated index, a
|
|
646
|
+
tile, a stack or meld holding the same parent twice): which of the
|
|
647
|
+
values written to its positions the root cell ends with is not
|
|
648
|
+
specified. It depends on the route, and a value equal to the one
|
|
649
|
+
delivered is not sent back. The same holds for a view that reads
|
|
650
|
+
several root cells as one (the mask of a refer to a wider type, which
|
|
651
|
+
is the OR of the cells under it): a cell written with the value it
|
|
652
|
+
was delivered is not sent back, so what its root cells end with is
|
|
653
|
+
not specified. */
|
|
628
654
|
void ca_iter_state_sync_slab (ca_iter_state *st);
|
|
629
655
|
|
|
630
|
-
/* Release any resources held by `st` and detach parent.
|
|
656
|
+
/* Release any resources held by `st` and detach parent. On a WRITE
|
|
657
|
+
walk with a root, the root is synced before it is detached
|
|
658
|
+
(ca_sync_detach); a slab still handed out from scratch is dropped, not
|
|
659
|
+
written back (ca_iter_state_close writes it back first). Idempotent:
|
|
631
660
|
a second call is a no-op. Also safe after a failed init, which holds
|
|
632
661
|
nothing to release (the call is then not required). */
|
|
633
662
|
void ca_iter_state_finish (ca_iter_state *st);
|
|
@@ -659,11 +688,12 @@ int ca_iter_state_init_l2_paired (ca_iter_state *st,
|
|
|
659
688
|
|
|
660
689
|
/* Finish a walk as the block macros leave it: when the body left a write
|
|
661
690
|
walk with `break`, the slab it was in is written back before the walk is
|
|
662
|
-
finished. ca_iter_state_finish
|
|
691
|
+
finished. ca_iter_state_finish alone drops such a slab when it sits in
|
|
692
|
+
scratch (writes through an alias are already in place). */
|
|
663
693
|
void ca_iter_state_close (ca_iter_state *st);
|
|
664
694
|
|
|
665
695
|
/* Raise unless rc is CA_ITER_OK. The block macros call this on the value
|
|
666
|
-
|
|
696
|
+
ca_iter_state_init_l2 / _l2_paired returned, so an author who never looks at a
|
|
667
697
|
return code still hears about a request the iterator would not serve.
|
|
668
698
|
Returns rc when there is nothing to report, so it composes inside the
|
|
669
699
|
macros' comma expressions. */
|
|
@@ -695,12 +725,14 @@ int ca_iter_check_same_shape (struct _CArray *a, struct _CArray *b);
|
|
|
695
725
|
VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
696
726
|
VALUE (*body)(VALUE), VALUE arg);
|
|
697
727
|
|
|
698
|
-
/* ----
|
|
699
|
-
(PROPOSAL_CAPSTONE_PHASE_C.md D3.1 (A) do/while/for + D3.2 (C) 2 kinds)
|
|
728
|
+
/* ---- kernel author block macros -----------------------------------------
|
|
700
729
|
|
|
701
730
|
Wrap init_l2 / next_slab_axes / [sync_slab] / finish in a single
|
|
702
731
|
block scope so kernel authors don't write lifecycle plumbing.
|
|
703
732
|
|
|
733
|
+
For a function in carray that uses each macro, see the catalog in
|
|
734
|
+
docs/authoring/HOW_TO_WRITE_KERNEL.md §0.1.
|
|
735
|
+
|
|
704
736
|
--- Constraints ---
|
|
705
737
|
|
|
706
738
|
- Author must pre-declare `char *p` and `boolean8_t *m` (or names of
|
|
@@ -721,7 +753,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
721
753
|
- `break;` from inside the body exits the loop AND triggers finish
|
|
722
754
|
correctly (= outer for's "increment" clause runs once on natural
|
|
723
755
|
exit; `break` from the inner while breaks both). In a write walk
|
|
724
|
-
|
|
756
|
+
the macros close with ca_iter_state_close, which writes back the
|
|
757
|
+
slab the body was in. `return` inside
|
|
725
758
|
the body LEAKS resources (scratch_ptr, parent attach) — drop to
|
|
726
759
|
raw API if early return is needed. A body that raises leaks the
|
|
727
760
|
same way: the engine releases what it holds when the walk itself
|
|
@@ -733,14 +766,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
733
766
|
- Macros are not statement-equivalent (= they expand to nested for
|
|
734
767
|
constructs). Don't follow them with `else` etc. */
|
|
735
768
|
|
|
736
|
-
/* ----
|
|
737
|
-
|
|
769
|
+
/* ---- mask helper macro ---------------------------------------------------
|
|
770
|
+
This macro lets a kernel handle masked sources
|
|
738
771
|
uniformly: pass NULL for `mask` if the source has no mask (= treat
|
|
739
772
|
all cells as unmasked), or a `boolean8_t *` of length n where
|
|
740
773
|
non-zero entries indicate masked (= "skip this cell").
|
|
741
|
-
|
|
742
|
-
PROPOSAL_T1_KERNEL_ITERATOR.md §2.4. GCC statement-expression for
|
|
743
|
-
CA_COUNT_UNMASKED is GCC/Clang only (MSVC not in scope).
|
|
744
774
|
--------------------------------------------------------------------- */
|
|
745
775
|
|
|
746
776
|
#define CA_FOR_EACH_UNMASKED(p, mask, n, body) do { \
|
|
@@ -752,31 +782,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
752
782
|
} \
|
|
753
783
|
} while (0)
|
|
754
784
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
} else { \
|
|
759
|
-
for (ca_size_t i = 0; i < (n); i++) { body } \
|
|
760
|
-
} \
|
|
761
|
-
} while (0)
|
|
762
|
-
|
|
763
|
-
#define CA_COUNT_UNMASKED(mask, n) ({ \
|
|
764
|
-
ca_size_t _ca_cnt = 0; \
|
|
765
|
-
ca_size_t _ca_n = (n); \
|
|
766
|
-
if (mask) { \
|
|
767
|
-
for (ca_size_t _ca_i = 0; _ca_i < _ca_n; _ca_i++) \
|
|
768
|
-
if (!(mask)[_ca_i]) _ca_cnt++; \
|
|
769
|
-
} else { \
|
|
770
|
-
_ca_cnt = _ca_n; \
|
|
771
|
-
} \
|
|
772
|
-
_ca_cnt; \
|
|
773
|
-
})
|
|
774
|
-
|
|
775
|
-
#define CA_MASK_GET(mask, i) ((mask) ? (mask)[i] : 0)
|
|
776
|
-
|
|
777
|
-
/* ---- L2 inner-loop macros (step 8+) -----------------------------------
|
|
778
|
-
`CA_L2_FOR_EACH(T, ptr, n, stride, p, body)` and its unmasked sibling
|
|
779
|
-
wrap the strided callback signature `(ptr, n, stride_bytes)` with a
|
|
785
|
+
/* ---- L2 inner-loop macro ------------------------------------------------
|
|
786
|
+
`CA_L2_FOR_EACH(T, ptr, n, stride, p, body)` wraps the strided
|
|
787
|
+
callback signature `(ptr, n, stride_bytes)` with a
|
|
780
788
|
`stride == sizeof(T)` fast-path branch. When the runtime stride
|
|
781
789
|
matches the element size, `body` runs against a `T *p` that the
|
|
782
790
|
compiler can autovectorise; when it doesn't, `p` is recomputed per
|
|
@@ -785,76 +793,41 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
785
793
|
The split is intentional: L2 strided callbacks are deliberately
|
|
786
794
|
universal — they accept arbitrary stride and so the compiler cannot
|
|
787
795
|
prove contig on the kernel-side loop. When the iterator hands a
|
|
788
|
-
contig scratch (
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
796
|
+
contig scratch (a gathered view, stride == bytes), a loop written for
|
|
797
|
+
any stride was measured 18-22% slower than a contiguous one. This
|
|
798
|
+
macro lets kernel authors recover the contig autovectorisation
|
|
799
|
+
without giving up the L2 surface.
|
|
792
800
|
|
|
793
801
|
This is the standard pattern for L2 kernels: write the body once,
|
|
794
|
-
the macro picks the right loop shape.
|
|
795
|
-
|
|
796
|
-
kernel-side macros are the speed knob) is `PROPOSAL_T1_KERNEL_
|
|
797
|
-
ITERATOR.md` §13.1.
|
|
802
|
+
the macro picks the right loop shape. The walk stays the same for
|
|
803
|
+
every source; the loop shape is the kernel's knob for speed.
|
|
798
804
|
|
|
799
805
|
Usage:
|
|
800
806
|
CA_L2_FOR_EACH(double, ptr, n, stride, p, {
|
|
801
807
|
*p = value; // p is `double *`
|
|
802
808
|
});
|
|
803
809
|
|
|
804
|
-
|
|
805
|
-
declarations inside a macro). */
|
|
810
|
+
Plain C99 (a do/while block with for-loop declarations). */
|
|
806
811
|
|
|
807
812
|
#define CA_L2_FOR_EACH(T, ptr, n, stride, p, body) do { \
|
|
813
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
808
814
|
ca_size_t _l2_n = (n); \
|
|
809
815
|
ca_size_t _l2_s = (stride); \
|
|
810
816
|
char *_l2_b = (char *)(ptr); \
|
|
811
817
|
if (_l2_s == sizeof(T)) { \
|
|
812
|
-
|
|
818
|
+
__ca_u_T *p = (__ca_u_T *)_l2_b; \
|
|
813
819
|
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
814
820
|
body \
|
|
815
821
|
p++; \
|
|
816
822
|
} \
|
|
817
823
|
} else { \
|
|
818
824
|
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
819
|
-
|
|
825
|
+
__ca_u_T *p = (__ca_u_T *)(_l2_b + _l2_i * _l2_s); \
|
|
820
826
|
body \
|
|
821
827
|
} \
|
|
822
828
|
} \
|
|
823
829
|
} while (0)
|
|
824
830
|
|
|
825
|
-
#define CA_L2_FOR_EACH_UNMASKED(T, ptr, mask, n, stride, p, body) do { \
|
|
826
|
-
ca_size_t _l2_n = (n); \
|
|
827
|
-
ca_size_t _l2_s = (stride); \
|
|
828
|
-
char *_l2_b = (char *)(ptr); \
|
|
829
|
-
boolean8_t *_l2_m = (mask); \
|
|
830
|
-
if (_l2_s == sizeof(T)) { \
|
|
831
|
-
T *p = (T *)_l2_b; \
|
|
832
|
-
if (_l2_m) { \
|
|
833
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
834
|
-
if (!_l2_m[_l2_i]) { body } \
|
|
835
|
-
p++; \
|
|
836
|
-
} \
|
|
837
|
-
} else { \
|
|
838
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
839
|
-
body \
|
|
840
|
-
p++; \
|
|
841
|
-
} \
|
|
842
|
-
} \
|
|
843
|
-
} else { \
|
|
844
|
-
if (_l2_m) { \
|
|
845
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
846
|
-
T *p = (T *)(_l2_b + _l2_i * _l2_s); \
|
|
847
|
-
if (!_l2_m[_l2_i]) { body } \
|
|
848
|
-
} \
|
|
849
|
-
} else { \
|
|
850
|
-
for (ca_size_t _l2_i = 0; _l2_i < _l2_n; _l2_i++) { \
|
|
851
|
-
T *p = (T *)(_l2_b + _l2_i * _l2_s); \
|
|
852
|
-
body \
|
|
853
|
-
} \
|
|
854
|
-
} \
|
|
855
|
-
} \
|
|
856
|
-
} while (0)
|
|
857
|
-
|
|
858
831
|
/* The slab policy is fixed to CA_SLAB_AXES (the only policy compatible
|
|
859
832
|
with next_slab_axes); it is hardcoded inside the macro rather than
|
|
860
833
|
taken as an argument, so block-macro authors never type the policy
|
|
@@ -877,15 +850,16 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
877
850
|
output iter runs WRITE; sync_slab is called on output after each
|
|
878
851
|
body iteration.
|
|
879
852
|
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
853
|
+
ca_in and ca_out must have the same shape (ndim and every dim[k]);
|
|
854
|
+
ca_iter_check_same_shape raises ArgumentError before either walk is
|
|
855
|
+
opened when they do not. Init failure on either iter raises — see
|
|
856
|
+
CA_FOR_EACH_SLAB above. */
|
|
884
857
|
/* Policy fixed to CA_SLAB_AXES internally — see CA_FOR_EACH_SLAB above. */
|
|
885
858
|
#define CA_FOR_EACH_SLAB_INOUT(st_in, st_out, ca_in, ca_out, \
|
|
886
859
|
axes, naxes, \
|
|
887
860
|
p_in, p_out, m_in, m_out) \
|
|
888
861
|
for ( int __cafi_init = ( \
|
|
862
|
+
ca_iter_check_same_shape((ca_in), (ca_out)), \
|
|
889
863
|
ca_iter_check_init( \
|
|
890
864
|
ca_iter_state_init_l2(&(st_in), (ca_in), CA_SLAB_AXES, \
|
|
891
865
|
(axes), (naxes), 0)), \
|
|
@@ -902,8 +876,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
902
876
|
ca_iter_state_next_slab_axes(&(st_out), &(p_out), &(m_out)); \
|
|
903
877
|
ca_iter_state_sync_slab(&(st_out)) )
|
|
904
878
|
|
|
905
|
-
/* ----
|
|
906
|
-
(rev4 §3 catalog contract)
|
|
879
|
+
/* ---- per-axis fiber macros ----------------------------------------------
|
|
907
880
|
|
|
908
881
|
Author-facing surface for "deliver one contig fiber along `axis` to
|
|
909
882
|
the kernel". Contig delivery is contract:
|
|
@@ -913,7 +886,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
913
886
|
- output (INOUT forms): contig same as data; CA_KERNEL_WRITE auto-set
|
|
914
887
|
|
|
915
888
|
The CA_KERNEL_FIBER_CONTIG flag is auto-set; engine gathers strided
|
|
916
|
-
fibers into per-state scratch when slab_strides[0] != bytes
|
|
889
|
+
fibers into per-state scratch when slab_strides[0] != bytes.
|
|
917
890
|
|
|
918
891
|
`axis`, `ca`, `ca_in` and `ca_out` may be evaluated more than once:
|
|
919
892
|
pass variables, not expressions with side effects. `n` is set to the
|
|
@@ -1080,7 +1053,67 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1080
1053
|
ca_iter_state_sync_slab(&(st_a)), \
|
|
1081
1054
|
ca_iter_state_sync_slab(&(st_b)) )
|
|
1082
1055
|
|
|
1083
|
-
/* ----
|
|
1056
|
+
/* ---- whole-array block macros -------------------------------------------
|
|
1057
|
+
|
|
1058
|
+
Hand the kernel the whole array as one contiguous run of cells in
|
|
1059
|
+
row-major order: `p` is the first cell and `n` is the number of cells
|
|
1060
|
+
(src->elements). The body runs once.
|
|
1061
|
+
|
|
1062
|
+
CA_FOR_EACH_WHOLE (st, ca, flags, p, n) data only
|
|
1063
|
+
CA_FOR_EACH_WHOLE_MASKED(st, ca, flags, p, n, m) + mask m[0..n-1],
|
|
1064
|
+
NULL when the
|
|
1065
|
+
source has none
|
|
1066
|
+
|
|
1067
|
+
Use them for a kernel that wants every cell in order and no axes: a
|
|
1068
|
+
whole-array scan, a hash of the values, a copy into a library buffer.
|
|
1069
|
+
A reduction that also runs per axis is better written once with
|
|
1070
|
+
CA_FOR_EACH_SLAB over the chosen axes (every axis for the whole array).
|
|
1071
|
+
|
|
1072
|
+
Contiguity is the promise, and it is kept by copying when it has to be:
|
|
1073
|
+
a contiguous array lends its memory, any other source is gathered into
|
|
1074
|
+
the walk's scratch first (a transpose, a strided block, a selection, a
|
|
1075
|
+
converting view). A kernel that can take strides and wants to avoid
|
|
1076
|
+
that copy uses CA_FOR_EACH_SLAB over every axis instead.
|
|
1077
|
+
|
|
1078
|
+
Same rules as CA_FOR_EACH_SLAB and the CA_FOR_EACH_FIBER family:
|
|
1079
|
+
- `p` and the mask cursor are declared by the caller; `n` receives the
|
|
1080
|
+
cell count. `ca` may be evaluated more than once.
|
|
1081
|
+
- Init failure raises (ca_iter_check_init); the body does not run.
|
|
1082
|
+
- The form without MASKED hands out no mask cursor: masked cells are
|
|
1083
|
+
read as values. Pass CA_KERNEL_NO_MASK to refuse a masked source.
|
|
1084
|
+
The mask cursor of the MASKED form is a copy; writing it reaches
|
|
1085
|
+
nothing.
|
|
1086
|
+
- With CA_KERNEL_WRITE the cells the kernel changed are written back
|
|
1087
|
+
when the body ends, including by `break`.
|
|
1088
|
+
- `return` from the body leaks what the walk holds. So does a body
|
|
1089
|
+
that raises: one that can raise -- one that calls back into Ruby --
|
|
1090
|
+
opens the macro inside ca_iter_ensure, with the state in the
|
|
1091
|
+
caller's frame. A write that raises keeps the cells already
|
|
1092
|
+
written where the walk lends the array's memory, and none of them
|
|
1093
|
+
where it gathered a copy.
|
|
1094
|
+
- Not statement-equivalent (nested `for`); no trailing `else`. */
|
|
1095
|
+
|
|
1096
|
+
#define CA_FOR_EACH_WHOLE(st, ca, flags, p, n) \
|
|
1097
|
+
for ( int __cfw_init = (ca_iter_check_init( \
|
|
1098
|
+
ca_iter_state_init_l1(&(st), (ca), CA_SLAB_WHOLE, \
|
|
1099
|
+
NULL, 0, (flags))), \
|
|
1100
|
+
1); \
|
|
1101
|
+
__cfw_init; \
|
|
1102
|
+
__cfw_init = 0, ca_iter_state_close(&(st)) ) \
|
|
1103
|
+
for ( ; ca_iter_state_next_slab(&(st), &(p), NULL, &(n)); \
|
|
1104
|
+
ca_iter_state_sync_slab(&(st)) )
|
|
1105
|
+
|
|
1106
|
+
#define CA_FOR_EACH_WHOLE_MASKED(st, ca, flags, p, n, m) \
|
|
1107
|
+
for ( int __cfwm_init = (ca_iter_check_init( \
|
|
1108
|
+
ca_iter_state_init_l1(&(st), (ca), CA_SLAB_WHOLE,\
|
|
1109
|
+
NULL, 0, (flags))), \
|
|
1110
|
+
1); \
|
|
1111
|
+
__cfwm_init; \
|
|
1112
|
+
__cfwm_init = 0, ca_iter_state_close(&(st)) ) \
|
|
1113
|
+
for ( ; ca_iter_state_next_slab(&(st), &(p), &(m), &(n)); \
|
|
1114
|
+
ca_iter_state_sync_slab(&(st)) )
|
|
1115
|
+
|
|
1116
|
+
/* ---- per-data_type reduction macro suite ------------------------------- */
|
|
1084
1117
|
|
|
1085
1118
|
/* CA_SLAB_REDUCE_T(T, ...): generic per-data_type slab reduction. T is the
|
|
1086
1119
|
element load type (`double`, `float`, `int32_t`, `int64_t`, ...).
|
|
@@ -1107,10 +1140,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1107
1140
|
- p: slab data pointer (char *).
|
|
1108
1141
|
- m: slab mask pointer (boolean8_t *, may be NULL).
|
|
1109
1142
|
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
numerics; use CA_SLAB_REDUCE_T directly for less common types
|
|
1113
|
-
(boolean8_t, int8_t, uint16_t, ...).
|
|
1143
|
+
T is the load type: double, float, int32_t, int64_t, or any other
|
|
1144
|
+
element type (boolean8_t, int8_t, uint16_t, ...).
|
|
1114
1145
|
|
|
1115
1146
|
Mask semantics: when m != NULL, masked cells are skipped (REDUCE is
|
|
1116
1147
|
not invoked). When m == NULL, every cell contributes.
|
|
@@ -1127,6 +1158,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1127
1158
|
it dead-code-eliminated; the trailing `(void) idx;` silences any
|
|
1128
1159
|
set-but-not-used warnings. */
|
|
1129
1160
|
#define CA_SLAB_REDUCE_T_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
|
|
1161
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1130
1162
|
(acc) = (INIT); \
|
|
1131
1163
|
int8_t __srK = (st).slab_ndim; \
|
|
1132
1164
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1171,7 +1203,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1171
1203
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1172
1204
|
if ( (m) == NULL ) { \
|
|
1173
1205
|
if ( __srContig ) { \
|
|
1174
|
-
const
|
|
1206
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1175
1207
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1176
1208
|
T v = __srSrc[__sj]; \
|
|
1177
1209
|
REDUCE; \
|
|
@@ -1179,7 +1211,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1179
1211
|
} \
|
|
1180
1212
|
} else { \
|
|
1181
1213
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1182
|
-
T v = *(const
|
|
1214
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1183
1215
|
REDUCE; \
|
|
1184
1216
|
idx++; \
|
|
1185
1217
|
} \
|
|
@@ -1187,7 +1219,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1187
1219
|
} else { \
|
|
1188
1220
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1189
1221
|
if ( __srContig && __srMaskU ) { \
|
|
1190
|
-
const
|
|
1222
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1191
1223
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1192
1224
|
if ( ! __srMM[__sj] ) { \
|
|
1193
1225
|
T v = __srSrc[__sj]; \
|
|
@@ -1200,7 +1232,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1200
1232
|
} else { \
|
|
1201
1233
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1202
1234
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1203
|
-
T v = *(const
|
|
1235
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1204
1236
|
REDUCE; \
|
|
1205
1237
|
} else { \
|
|
1206
1238
|
(masked_cnt)++; \
|
|
@@ -1218,9 +1250,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1218
1250
|
} while (0)
|
|
1219
1251
|
|
|
1220
1252
|
/* Backward-compatible wrapper that hides masked_cnt from kernels that
|
|
1221
|
-
don't need it.
|
|
1222
|
-
|
|
1223
|
-
|
|
1253
|
+
don't need it. The generated reductions use the _EX forms (or their
|
|
1254
|
+
SIMD / 8-way variants); CA_SLAB_REDUCE_T is for hand-written kernels
|
|
1255
|
+
that do not need the masked count. */
|
|
1224
1256
|
#define CA_SLAB_REDUCE_T(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
1225
1257
|
ca_size_t __sr_throwaway_mc = 0; \
|
|
1226
1258
|
CA_SLAB_REDUCE_T_EX(T, st, p, m, acc, INIT, REDUCE, __sr_throwaway_mc); \
|
|
@@ -1228,24 +1260,18 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1228
1260
|
} while (0)
|
|
1229
1261
|
|
|
1230
1262
|
/* ----------------------------------------------------------------------
|
|
1231
|
-
* SIMD-licensed reduction variants
|
|
1263
|
+
* SIMD-licensed reduction variants
|
|
1232
1264
|
*
|
|
1233
|
-
* The PLUS/MIN/MAX/STAR
|
|
1234
|
-
*
|
|
1235
|
-
*
|
|
1236
|
-
*
|
|
1237
|
-
*
|
|
1238
|
-
*
|
|
1239
|
-
* Subsequent sub-steps (SL.1.1+) will replace each forward with a
|
|
1240
|
-
* contig-branch body carrying `#pragma omp simd reduction(<kind>:acc)`
|
|
1241
|
-
* (and per-acc pragmas for VAR). The pragma is emitted via the
|
|
1265
|
+
* The PLUS/MIN/MAX/STAR _EX variants below carry, in their contiguous
|
|
1266
|
+
* no-mask branch, `#pragma omp simd reduction(<kind>:acc)`, which
|
|
1267
|
+
* licenses the compiler to reassociate the reduction (results are
|
|
1268
|
+
* epsilon-close, not bit-exact). VAR_EX forwards to PLUS_EX and has no
|
|
1269
|
+
* caller. The pragma is emitted via the
|
|
1242
1270
|
* _Pragma + _CA_XSTR substitution helpers below so the acc lvalue
|
|
1243
1271
|
* can be parameterised.
|
|
1244
1272
|
*
|
|
1245
1273
|
* The pragma is a no-op on compilers that don't support `-fopenmp-simd`
|
|
1246
1274
|
* (extconf.rb probe — graceful degradation, code stays correct).
|
|
1247
|
-
*
|
|
1248
|
-
* Q1-Q6 closure (sparring round 1 2026-06-12): see proposal §5.
|
|
1249
1275
|
* --------------------------------------------------------------------*/
|
|
1250
1276
|
|
|
1251
1277
|
#define _CA_STR(x) #x
|
|
@@ -1255,15 +1281,15 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1255
1281
|
#define _CA_SIMD_MAX(var) _Pragma(_CA_XSTR(omp simd reduction(max:var)))
|
|
1256
1282
|
#define _CA_SIMD_STAR(var) _Pragma(_CA_XSTR(omp simd reduction(*:var)))
|
|
1257
1283
|
|
|
1258
|
-
/* CA_SLAB_REDUCE_T_PLUS_EX
|
|
1284
|
+
/* CA_SLAB_REDUCE_T_PLUS_EX:
|
|
1259
1285
|
* Same structure as CA_SLAB_REDUCE_T_EX, but the **no-mask + contig**
|
|
1260
1286
|
* inner loop carries `#pragma omp simd reduction(+:acc)` so clang/gcc
|
|
1261
1287
|
* are licensed to reassoc the accumulator and emit SIMD reduction
|
|
1262
1288
|
* sequences. All other branches (masked-contig, non-contig, masked-
|
|
1263
|
-
* non-contig) are byte-identical to _EX
|
|
1264
|
-
*
|
|
1289
|
+
* non-contig) are byte-identical to _EX: a SIMD reduction there would
|
|
1290
|
+
* need an output-buffered loop interchange.
|
|
1265
1291
|
*
|
|
1266
|
-
*
|
|
1292
|
+
* Measured: N=1M f64 sum 906 us -> 116 us (= 7.8x, 68.8 GB/s).
|
|
1267
1293
|
* Parity: ε-close (relative error < 2e-16 for f64 sum), bit-exact
|
|
1268
1294
|
* not guaranteed (= the SIMD license; see guides/devel/11_kernel_iterator.md).
|
|
1269
1295
|
*
|
|
@@ -1272,6 +1298,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1272
1298
|
* by the compiler when their update is a simple ++ pattern.
|
|
1273
1299
|
*/
|
|
1274
1300
|
#define CA_SLAB_REDUCE_T_PLUS_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
|
|
1301
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1275
1302
|
(acc) = (INIT); \
|
|
1276
1303
|
int8_t __srK = (st).slab_ndim; \
|
|
1277
1304
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1316,7 +1343,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1316
1343
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1317
1344
|
if ( (m) == NULL ) { \
|
|
1318
1345
|
if ( __srContig ) { \
|
|
1319
|
-
const
|
|
1346
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1320
1347
|
_CA_SIMD_PLUS(acc) \
|
|
1321
1348
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1322
1349
|
T v = __srSrc[__sj]; \
|
|
@@ -1325,7 +1352,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1325
1352
|
} \
|
|
1326
1353
|
} else { \
|
|
1327
1354
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1328
|
-
T v = *(const
|
|
1355
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1329
1356
|
REDUCE; \
|
|
1330
1357
|
idx++; \
|
|
1331
1358
|
} \
|
|
@@ -1333,7 +1360,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1333
1360
|
} else { \
|
|
1334
1361
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1335
1362
|
if ( __srContig && __srMaskU ) { \
|
|
1336
|
-
const
|
|
1363
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1337
1364
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1338
1365
|
if ( ! __srMM[__sj] ) { \
|
|
1339
1366
|
T v = __srSrc[__sj]; \
|
|
@@ -1346,7 +1373,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1346
1373
|
} else { \
|
|
1347
1374
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1348
1375
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1349
|
-
T v = *(const
|
|
1376
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1350
1377
|
REDUCE; \
|
|
1351
1378
|
} else { \
|
|
1352
1379
|
(masked_cnt)++; \
|
|
@@ -1363,7 +1390,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1363
1390
|
(void) idx; \
|
|
1364
1391
|
} while (0)
|
|
1365
1392
|
|
|
1366
|
-
/* CA_SLAB_REDUCE_SUM8_EX
|
|
1393
|
+
/* CA_SLAB_REDUCE_SUM8_EX:
|
|
1367
1394
|
* 8-way manual accumulator split for the no-mask + contig inner loop.
|
|
1368
1395
|
*
|
|
1369
1396
|
* Why: GCC 11.5 with -march=native (FMA) does NOT split the pragma-simd
|
|
@@ -1387,10 +1414,12 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1387
1414
|
* branches stay single-accumulator (not the hot path). Position
|
|
1388
1415
|
* counter `idx` is not tracked (variance REDUCE never uses it). ACC_T
|
|
1389
1416
|
* is the accumulator C type (double for Pass 1 sum on numeric/bool and
|
|
1390
|
-
* Pass 2 M2; complex Pass 1 passes its complex type).
|
|
1391
|
-
*
|
|
1417
|
+
* Pass 2 M2; complex Pass 1 passes its complex type). mkkernel.rb
|
|
1418
|
+
* emits it for every 8-way-eligible _PLUS reduction (sum, mean, and
|
|
1419
|
+
* both passes of variance / stddev).
|
|
1392
1420
|
*/
|
|
1393
1421
|
#define CA_SLAB_REDUCE_SUM8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
|
|
1422
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1394
1423
|
(acc) = (INIT); \
|
|
1395
1424
|
int8_t __srK = (st).slab_ndim; \
|
|
1396
1425
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1427,7 +1456,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1427
1456
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1428
1457
|
if ( (m) == NULL ) { \
|
|
1429
1458
|
if ( __srContig ) { \
|
|
1430
|
-
const
|
|
1459
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1431
1460
|
ACC_T __a0=(ACC_T)0,__a1=(ACC_T)0,__a2=(ACC_T)0,__a3=(ACC_T)0; \
|
|
1432
1461
|
ACC_T __a4=(ACC_T)0,__a5=(ACC_T)0,__a6=(ACC_T)0,__a7=(ACC_T)0; \
|
|
1433
1462
|
ca_size_t __sj = 0, __sN = __srInnerN; \
|
|
@@ -1446,14 +1475,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1446
1475
|
+ ((__a4 + __a5) + (__a6 + __a7)); \
|
|
1447
1476
|
} else { \
|
|
1448
1477
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1449
|
-
T v = *(const
|
|
1478
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1450
1479
|
(acc) += EXPR(v); \
|
|
1451
1480
|
} \
|
|
1452
1481
|
} \
|
|
1453
1482
|
} else { \
|
|
1454
1483
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1455
1484
|
if ( __srContig && __srMaskU ) { \
|
|
1456
|
-
const
|
|
1485
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1457
1486
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1458
1487
|
if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) += EXPR(v); } \
|
|
1459
1488
|
else { (masked_cnt)++; } \
|
|
@@ -1461,7 +1490,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1461
1490
|
} else { \
|
|
1462
1491
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1463
1492
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1464
|
-
T v = *(const
|
|
1493
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1465
1494
|
(acc) += EXPR(v); \
|
|
1466
1495
|
} else { (masked_cnt)++; } \
|
|
1467
1496
|
} \
|
|
@@ -1474,9 +1503,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1474
1503
|
} \
|
|
1475
1504
|
} while (0)
|
|
1476
1505
|
|
|
1477
|
-
/* CA_SLAB_REDUCE_MIN8_EX / _MAX8_EX / _STAR8_EX
|
|
1478
|
-
*
|
|
1479
|
-
* 2026-07-19):
|
|
1506
|
+
/* CA_SLAB_REDUCE_MIN8_EX / _MAX8_EX / _STAR8_EX (the SUM8_EX pattern for
|
|
1507
|
+
* the other standard reducers):
|
|
1480
1508
|
*
|
|
1481
1509
|
* Same rationale as SUM8_EX — GCC does not split single-accumulator pragma
|
|
1482
1510
|
* reductions into multiple independent chains on FMA/FADD-latency-bound
|
|
@@ -1498,6 +1526,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1498
1526
|
* single-accumulator case).
|
|
1499
1527
|
*/
|
|
1500
1528
|
#define CA_SLAB_REDUCE_MIN8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
|
|
1529
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1501
1530
|
(acc) = (INIT); \
|
|
1502
1531
|
int8_t __srK = (st).slab_ndim; \
|
|
1503
1532
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1534,7 +1563,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1534
1563
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1535
1564
|
if ( (m) == NULL ) { \
|
|
1536
1565
|
if ( __srContig ) { \
|
|
1537
|
-
const
|
|
1566
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1538
1567
|
ACC_T __a0=(acc),__a1=(acc),__a2=(acc),__a3=(acc); \
|
|
1539
1568
|
ACC_T __a4=(acc),__a5=(acc),__a6=(acc),__a7=(acc); \
|
|
1540
1569
|
ca_size_t __sj = 0, __sN = __srInnerN; \
|
|
@@ -1560,14 +1589,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1560
1589
|
(acc) = (__c0 < __c1) ? __c0 : __c1; \
|
|
1561
1590
|
} else { \
|
|
1562
1591
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1563
|
-
ACC_T __v = EXPR(*(const
|
|
1592
|
+
ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
|
|
1564
1593
|
if (__v < (acc)) (acc) = __v; \
|
|
1565
1594
|
} \
|
|
1566
1595
|
} \
|
|
1567
1596
|
} else { \
|
|
1568
1597
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1569
1598
|
if ( __srContig && __srMaskU ) { \
|
|
1570
|
-
const
|
|
1599
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1571
1600
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1572
1601
|
if ( ! __srMM[__sj] ) { \
|
|
1573
1602
|
ACC_T __v = EXPR(__srSrc[__sj]); \
|
|
@@ -1577,7 +1606,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1577
1606
|
} else { \
|
|
1578
1607
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1579
1608
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1580
|
-
ACC_T __v = EXPR(*(const
|
|
1609
|
+
ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
|
|
1581
1610
|
if (__v < (acc)) (acc) = __v; \
|
|
1582
1611
|
} else { (masked_cnt)++; } \
|
|
1583
1612
|
} \
|
|
@@ -1591,6 +1620,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1591
1620
|
} while (0)
|
|
1592
1621
|
|
|
1593
1622
|
#define CA_SLAB_REDUCE_MAX8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
|
|
1623
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1594
1624
|
(acc) = (INIT); \
|
|
1595
1625
|
int8_t __srK = (st).slab_ndim; \
|
|
1596
1626
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1627,7 +1657,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1627
1657
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1628
1658
|
if ( (m) == NULL ) { \
|
|
1629
1659
|
if ( __srContig ) { \
|
|
1630
|
-
const
|
|
1660
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1631
1661
|
ACC_T __a0=(acc),__a1=(acc),__a2=(acc),__a3=(acc); \
|
|
1632
1662
|
ACC_T __a4=(acc),__a5=(acc),__a6=(acc),__a7=(acc); \
|
|
1633
1663
|
ca_size_t __sj = 0, __sN = __srInnerN; \
|
|
@@ -1653,14 +1683,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1653
1683
|
(acc) = (__c0 > __c1) ? __c0 : __c1; \
|
|
1654
1684
|
} else { \
|
|
1655
1685
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1656
|
-
ACC_T __v = EXPR(*(const
|
|
1686
|
+
ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
|
|
1657
1687
|
if (__v > (acc)) (acc) = __v; \
|
|
1658
1688
|
} \
|
|
1659
1689
|
} \
|
|
1660
1690
|
} else { \
|
|
1661
1691
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1662
1692
|
if ( __srContig && __srMaskU ) { \
|
|
1663
|
-
const
|
|
1693
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1664
1694
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1665
1695
|
if ( ! __srMM[__sj] ) { \
|
|
1666
1696
|
ACC_T __v = EXPR(__srSrc[__sj]); \
|
|
@@ -1670,7 +1700,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1670
1700
|
} else { \
|
|
1671
1701
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1672
1702
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1673
|
-
ACC_T __v = EXPR(*(const
|
|
1703
|
+
ACC_T __v = EXPR(*(const __ca_u_T *)(__srQ + __sj * __srInnerS)); \
|
|
1674
1704
|
if (__v > (acc)) (acc) = __v; \
|
|
1675
1705
|
} else { (masked_cnt)++; } \
|
|
1676
1706
|
} \
|
|
@@ -1684,6 +1714,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1684
1714
|
} while (0)
|
|
1685
1715
|
|
|
1686
1716
|
#define CA_SLAB_REDUCE_STAR8_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
|
|
1717
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1687
1718
|
(acc) = (INIT); \
|
|
1688
1719
|
int8_t __srK = (st).slab_ndim; \
|
|
1689
1720
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1720,7 +1751,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1720
1751
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1721
1752
|
if ( (m) == NULL ) { \
|
|
1722
1753
|
if ( __srContig ) { \
|
|
1723
|
-
const
|
|
1754
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1724
1755
|
ACC_T __a0=(ACC_T)1,__a1=(ACC_T)1,__a2=(ACC_T)1,__a3=(ACC_T)1; \
|
|
1725
1756
|
ACC_T __a4=(ACC_T)1,__a5=(ACC_T)1,__a6=(ACC_T)1,__a7=(ACC_T)1; \
|
|
1726
1757
|
ca_size_t __sj = 0, __sN = __srInnerN; \
|
|
@@ -1739,14 +1770,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1739
1770
|
* ((__a4 * __a5) * (__a6 * __a7)); \
|
|
1740
1771
|
} else { \
|
|
1741
1772
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1742
|
-
T v = *(const
|
|
1773
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1743
1774
|
(acc) *= EXPR(v); \
|
|
1744
1775
|
} \
|
|
1745
1776
|
} \
|
|
1746
1777
|
} else { \
|
|
1747
1778
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1748
1779
|
if ( __srContig && __srMaskU ) { \
|
|
1749
|
-
const
|
|
1780
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1750
1781
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1751
1782
|
if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) *= EXPR(v); } \
|
|
1752
1783
|
else { (masked_cnt)++; } \
|
|
@@ -1754,7 +1785,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1754
1785
|
} else { \
|
|
1755
1786
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1756
1787
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1757
|
-
T v = *(const
|
|
1788
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1758
1789
|
(acc) *= EXPR(v); \
|
|
1759
1790
|
} else { (masked_cnt)++; } \
|
|
1760
1791
|
} \
|
|
@@ -1767,10 +1798,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1767
1798
|
} \
|
|
1768
1799
|
} while (0)
|
|
1769
1800
|
|
|
1770
|
-
/* CA_SLAB_REDUCE_T_MIN_EX (
|
|
1801
|
+
/* CA_SLAB_REDUCE_T_MIN_EX (emitted by the kernel generator; not author surface):
|
|
1771
1802
|
* Same structure as PLUS_EX; no-mask + contig inner loop carries
|
|
1772
|
-
* `#pragma omp simd reduction(min:acc)`.
|
|
1773
|
-
*
|
|
1803
|
+
* `#pragma omp simd reduction(min:acc)`. The :min kernel in
|
|
1804
|
+
* mkkernel.rb writes REDUCE as `acc = (v < acc) ? v : acc`, which
|
|
1774
1805
|
* matches OpenMP `min:` reduction semantics exactly — bit-exact
|
|
1775
1806
|
* parity preserved (no reassoc license needed; min/max are
|
|
1776
1807
|
* associative + commutative under `<` / `>`).
|
|
@@ -1780,6 +1811,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1780
1811
|
* (T_LIMIT_HI), matching the pre-SIMD path byte-identically.
|
|
1781
1812
|
*/
|
|
1782
1813
|
#define CA_SLAB_REDUCE_T_MIN_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
|
|
1814
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1783
1815
|
(acc) = (INIT); \
|
|
1784
1816
|
int8_t __srK = (st).slab_ndim; \
|
|
1785
1817
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1824,7 +1856,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1824
1856
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1825
1857
|
if ( (m) == NULL ) { \
|
|
1826
1858
|
if ( __srContig ) { \
|
|
1827
|
-
const
|
|
1859
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1828
1860
|
_CA_SIMD_MIN(acc) \
|
|
1829
1861
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1830
1862
|
T v = __srSrc[__sj]; \
|
|
@@ -1833,7 +1865,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1833
1865
|
} \
|
|
1834
1866
|
} else { \
|
|
1835
1867
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1836
|
-
T v = *(const
|
|
1868
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1837
1869
|
REDUCE; \
|
|
1838
1870
|
idx++; \
|
|
1839
1871
|
} \
|
|
@@ -1841,7 +1873,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1841
1873
|
} else { \
|
|
1842
1874
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1843
1875
|
if ( __srContig && __srMaskU ) { \
|
|
1844
|
-
const
|
|
1876
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1845
1877
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1846
1878
|
if ( ! __srMM[__sj] ) { \
|
|
1847
1879
|
T v = __srSrc[__sj]; \
|
|
@@ -1854,7 +1886,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1854
1886
|
} else { \
|
|
1855
1887
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1856
1888
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1857
|
-
T v = *(const
|
|
1889
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1858
1890
|
REDUCE; \
|
|
1859
1891
|
} else { \
|
|
1860
1892
|
(masked_cnt)++; \
|
|
@@ -1871,8 +1903,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1871
1903
|
(void) idx; \
|
|
1872
1904
|
} while (0)
|
|
1873
1905
|
|
|
1874
|
-
/* CA_SLAB_REDUCE_T_MAX_EX (
|
|
1906
|
+
/* CA_SLAB_REDUCE_T_MAX_EX (generator-only): same as MIN_EX with `_CA_SIMD_MAX`. */
|
|
1875
1907
|
#define CA_SLAB_REDUCE_T_MAX_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
|
|
1908
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1876
1909
|
(acc) = (INIT); \
|
|
1877
1910
|
int8_t __srK = (st).slab_ndim; \
|
|
1878
1911
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -1917,7 +1950,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1917
1950
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1918
1951
|
if ( (m) == NULL ) { \
|
|
1919
1952
|
if ( __srContig ) { \
|
|
1920
|
-
const
|
|
1953
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1921
1954
|
_CA_SIMD_MAX(acc) \
|
|
1922
1955
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1923
1956
|
T v = __srSrc[__sj]; \
|
|
@@ -1926,7 +1959,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1926
1959
|
} \
|
|
1927
1960
|
} else { \
|
|
1928
1961
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1929
|
-
T v = *(const
|
|
1962
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1930
1963
|
REDUCE; \
|
|
1931
1964
|
idx++; \
|
|
1932
1965
|
} \
|
|
@@ -1934,7 +1967,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1934
1967
|
} else { \
|
|
1935
1968
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1936
1969
|
if ( __srContig && __srMaskU ) { \
|
|
1937
|
-
const
|
|
1970
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1938
1971
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1939
1972
|
if ( ! __srMM[__sj] ) { \
|
|
1940
1973
|
T v = __srSrc[__sj]; \
|
|
@@ -1947,7 +1980,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1947
1980
|
} else { \
|
|
1948
1981
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1949
1982
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1950
|
-
T v = *(const
|
|
1983
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1951
1984
|
REDUCE; \
|
|
1952
1985
|
} else { \
|
|
1953
1986
|
(masked_cnt)++; \
|
|
@@ -1964,7 +1997,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1964
1997
|
(void) idx; \
|
|
1965
1998
|
} while (0)
|
|
1966
1999
|
|
|
1967
|
-
/* CA_SLAB_REDUCE_T_STAR_EX (
|
|
2000
|
+
/* CA_SLAB_REDUCE_T_STAR_EX (emitted by the kernel generator; not author surface):
|
|
1968
2001
|
* Same structure as PLUS_EX; no-mask + contig inner loop carries
|
|
1969
2002
|
* `#pragma omp simd reduction(*:acc)`. Used by :prod kernel
|
|
1970
2003
|
* (acc *= v). f64 / f32 multiplication parity is ε-close (same
|
|
@@ -1973,6 +2006,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1973
2006
|
* integers).
|
|
1974
2007
|
*/
|
|
1975
2008
|
#define CA_SLAB_REDUCE_T_STAR_EX(T, st, p, m, acc, INIT, REDUCE, masked_cnt) do { \
|
|
2009
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1976
2010
|
(acc) = (INIT); \
|
|
1977
2011
|
int8_t __srK = (st).slab_ndim; \
|
|
1978
2012
|
int8_t __srOuterK = __srK - 1; \
|
|
@@ -2017,7 +2051,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2017
2051
|
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
2018
2052
|
if ( (m) == NULL ) { \
|
|
2019
2053
|
if ( __srContig ) { \
|
|
2020
|
-
const
|
|
2054
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
2021
2055
|
_CA_SIMD_STAR(acc) \
|
|
2022
2056
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
2023
2057
|
T v = __srSrc[__sj]; \
|
|
@@ -2026,7 +2060,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2026
2060
|
} \
|
|
2027
2061
|
} else { \
|
|
2028
2062
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
2029
|
-
T v = *(const
|
|
2063
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
2030
2064
|
REDUCE; \
|
|
2031
2065
|
idx++; \
|
|
2032
2066
|
} \
|
|
@@ -2034,7 +2068,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2034
2068
|
} else { \
|
|
2035
2069
|
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
2036
2070
|
if ( __srContig && __srMaskU ) { \
|
|
2037
|
-
const
|
|
2071
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
2038
2072
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
2039
2073
|
if ( ! __srMM[__sj] ) { \
|
|
2040
2074
|
T v = __srSrc[__sj]; \
|
|
@@ -2047,7 +2081,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2047
2081
|
} else { \
|
|
2048
2082
|
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
2049
2083
|
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
2050
|
-
T v = *(const
|
|
2084
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
2051
2085
|
REDUCE; \
|
|
2052
2086
|
} else { \
|
|
2053
2087
|
(masked_cnt)++; \
|
|
@@ -2064,51 +2098,16 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2064
2098
|
(void) idx; \
|
|
2065
2099
|
} while (0)
|
|
2066
2100
|
|
|
2067
|
-
/*
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
CA_SLAB_REDUCE_T_PLUS_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2071
|
-
__sr_throwaway_mc); \
|
|
2072
|
-
(void) __sr_throwaway_mc; \
|
|
2073
|
-
} while (0)
|
|
2074
|
-
#define CA_SLAB_REDUCE_T_MIN(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
2075
|
-
ca_size_t __sr_throwaway_mc = 0; \
|
|
2076
|
-
CA_SLAB_REDUCE_T_MIN_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2077
|
-
__sr_throwaway_mc); \
|
|
2078
|
-
(void) __sr_throwaway_mc; \
|
|
2079
|
-
} while (0)
|
|
2080
|
-
#define CA_SLAB_REDUCE_T_MAX(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
2081
|
-
ca_size_t __sr_throwaway_mc = 0; \
|
|
2082
|
-
CA_SLAB_REDUCE_T_MAX_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2083
|
-
__sr_throwaway_mc); \
|
|
2084
|
-
(void) __sr_throwaway_mc; \
|
|
2085
|
-
} while (0)
|
|
2086
|
-
#define CA_SLAB_REDUCE_T_STAR(T, st, p, m, acc, INIT, REDUCE) do { \
|
|
2087
|
-
ca_size_t __sr_throwaway_mc = 0; \
|
|
2088
|
-
CA_SLAB_REDUCE_T_STAR_EX(T, st, p, m, acc, INIT, REDUCE, \
|
|
2089
|
-
__sr_throwaway_mc); \
|
|
2090
|
-
(void) __sr_throwaway_mc; \
|
|
2091
|
-
} while (0)
|
|
2092
|
-
|
|
2093
|
-
/* Multi-acc variant for variance/stddev (sum + sumsq + cnt).
|
|
2094
|
-
*
|
|
2095
|
-
* SL.1.3 measurement (2026-06-12) showed VAR_EX is **NOT NEEDED** for
|
|
2096
|
-
* variance/stddev: a single `#pragma omp simd reduction(+:acc)` on
|
|
2097
|
-
* the primary accumulator (via PLUS_EX dispatch) is sufficient — clang
|
|
2098
|
-
* extends SIMD treatment to the dependent `sumsq` updates via idiom
|
|
2099
|
-
* recognition (the secondary accumulator is recognised as a derived
|
|
2100
|
-
* reduction over `v*v`). variance entity 1-D f64 reached 53.4 GB/s
|
|
2101
|
-
* (gate > 40 PASS, 8.2x speedup vs 6.5 baseline) using PLUS_EX.
|
|
2102
|
-
*
|
|
2103
|
-
* VAR_EX is **retained as a stub** here for future SL.2.x output-
|
|
2104
|
-
* buffered work or other multi-acc reductions that don't share the
|
|
2105
|
-
* variance pattern (= primary + derived sumsq). The stub forwards
|
|
2106
|
-
* to PLUS_EX on acc1; acc2 is zero-initialised but otherwise left
|
|
2107
|
-
* to the enclosing REDUCE expression. This is safe because no
|
|
2108
|
-
* kernel currently dispatches to VAR_EX (= reduce_macro_suffix never
|
|
2109
|
-
* returns "_VAR"; SL.1.0/1.1 reduce DSL has no :var value).
|
|
2101
|
+
/* Multi-acc variant (two accumulators). No kernel uses it: variance /
|
|
2102
|
+
* stddev are computed in two passes (Σx for the mean, then Σ(x-mean)^2),
|
|
2103
|
+
* each a single-accumulator reduction through SUM8_EX / PLUS_EX.
|
|
2110
2104
|
*
|
|
2111
|
-
*
|
|
2105
|
+
* VAR_EX is kept as a stub for a future multi-accumulator reduction
|
|
2106
|
+
* that does not share the variance pattern (primary + derived sumsq).
|
|
2107
|
+
* The stub forwards to PLUS_EX on acc1; acc2 is zero-initialised but
|
|
2108
|
+
* otherwise left to the enclosing REDUCE expression. This is safe
|
|
2109
|
+
* because no kernel dispatches to VAR_EX (reduce_macro_suffix never
|
|
2110
|
+
* returns "_VAR"; the reduce DSL has no :var value).
|
|
2112
2111
|
*/
|
|
2113
2112
|
#define CA_SLAB_REDUCE_T_VAR_EX(T, st, p, m, acc1, acc2, \
|
|
2114
2113
|
INIT1, INIT2, REDUCE, masked_cnt) do { \
|
|
@@ -2130,9 +2129,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2130
2129
|
enforced by ca_check_same_shape at dispatcher).
|
|
2131
2130
|
|
|
2132
2131
|
Author-supplied (additional to CA_SLAB_REDUCE_T_EX):
|
|
2133
|
-
- T_W: second-operand element C type (load type).
|
|
2134
|
-
|
|
2135
|
-
:
|
|
2132
|
+
- T_W: second-operand element C type (load type). The
|
|
2133
|
+
production kernels read the weight as double
|
|
2134
|
+
(mkkernel array_arg: data_type: :promote), so T_W need
|
|
2135
|
+
not match T. Kept separate in the
|
|
2136
2136
|
macro signature so future heterogeneous-data_type variants
|
|
2137
2137
|
can plug in without macro change.
|
|
2138
2138
|
- st_w: second-operand ca_iter_state, positioned on the
|
|
@@ -2144,8 +2144,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2144
2144
|
REDUCE expression binds BOTH `v` (source cell, type T) AND `w` (weights
|
|
2145
2145
|
cell, type T_W), in addition to `acc` and `idx` from the base macro.
|
|
2146
2146
|
The mask `m` applies to the SOURCE only (weights mask is overlaid onto
|
|
2147
|
-
source mask
|
|
2147
|
+
source mask by the dispatcher). */
|
|
2148
2148
|
#define CA_SLAB_REDUCE_ARRAY_T_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, REDUCE, masked_cnt) do { \
|
|
2149
|
+
typedef T_W __ca_u_T_W CA_UNALIGNED; \
|
|
2150
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
2149
2151
|
(acc) = (INIT); \
|
|
2150
2152
|
int8_t __sraK = (st).slab_ndim; \
|
|
2151
2153
|
int8_t __sraOuterK = __sraK - 1; \
|
|
@@ -2199,8 +2201,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2199
2201
|
const char *__sraWQ = (const char *)(p_w) + __sraWoff; \
|
|
2200
2202
|
if ( (m) == NULL ) { \
|
|
2201
2203
|
if ( __sraContig && __sraContigW ) { \
|
|
2202
|
-
const
|
|
2203
|
-
const
|
|
2204
|
+
const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
|
|
2205
|
+
const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
|
|
2204
2206
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2205
2207
|
T v = __sraSrc[__saj]; \
|
|
2206
2208
|
T_W w = __sraWgt[__saj]; \
|
|
@@ -2209,8 +2211,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2209
2211
|
} \
|
|
2210
2212
|
} else { \
|
|
2211
2213
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2212
|
-
T v = *(const
|
|
2213
|
-
T_W w = *(const
|
|
2214
|
+
T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
|
|
2215
|
+
T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
|
|
2214
2216
|
REDUCE; \
|
|
2215
2217
|
idx++; \
|
|
2216
2218
|
} \
|
|
@@ -2218,8 +2220,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2218
2220
|
} else { \
|
|
2219
2221
|
const boolean8_t *__sraMM = (const boolean8_t *)(m) + __sraMoff; \
|
|
2220
2222
|
if ( __sraContig && __sraContigW && __sraMaskU ) { \
|
|
2221
|
-
const
|
|
2222
|
-
const
|
|
2223
|
+
const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
|
|
2224
|
+
const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
|
|
2223
2225
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2224
2226
|
if ( ! __sraMM[__saj] ) { \
|
|
2225
2227
|
T v = __sraSrc[__saj]; \
|
|
@@ -2233,8 +2235,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2233
2235
|
} else { \
|
|
2234
2236
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2235
2237
|
if ( ! __sraMM[__saj * __sraInnerMS] ) { \
|
|
2236
|
-
T v = *(const
|
|
2237
|
-
T_W w = *(const
|
|
2238
|
+
T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
|
|
2239
|
+
T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
|
|
2238
2240
|
REDUCE; \
|
|
2239
2241
|
} else { \
|
|
2240
2242
|
(masked_cnt)++; \
|
|
@@ -2251,14 +2253,7 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2251
2253
|
(void) idx; \
|
|
2252
2254
|
} while (0)
|
|
2253
2255
|
|
|
2254
|
-
|
|
2255
|
-
ca_size_t __sra_throwaway_mc = 0; \
|
|
2256
|
-
CA_SLAB_REDUCE_ARRAY_T_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, REDUCE, \
|
|
2257
|
-
__sra_throwaway_mc); \
|
|
2258
|
-
(void) __sra_throwaway_mc; \
|
|
2259
|
-
} while (0)
|
|
2260
|
-
|
|
2261
|
-
/* CA_SLAB_REDUCE_ARRAY_T_PLUS_EX (SL.1.4b):
|
|
2256
|
+
/* CA_SLAB_REDUCE_ARRAY_T_PLUS_EX:
|
|
2262
2257
|
* Parallel-array PLUS variant of CA_SLAB_REDUCE_ARRAY_T_EX.
|
|
2263
2258
|
* no-mask + both-contig inner loop carries `#pragma omp simd
|
|
2264
2259
|
* reduction(+:acc)`. Used by :wsum / :wmean — author REDUCE
|
|
@@ -2267,12 +2262,14 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2267
2262
|
* reduction-tree merge. Other branches (masked, non-contig)
|
|
2268
2263
|
* stay identical to _EX.
|
|
2269
2264
|
*
|
|
2270
|
-
*
|
|
2265
|
+
* Measured:
|
|
2271
2266
|
* wsum f64 N=1M: 908 us -> ~150 us (= the expected 50-60 GB/s)
|
|
2272
2267
|
* wmean f64 N=1M: 1212 us -> ~150 us (same path + 1 extra acc)
|
|
2273
2268
|
*/
|
|
2274
2269
|
#define CA_SLAB_REDUCE_ARRAY_T_PLUS_EX(T, T_W, st, p, m, st_w, p_w, acc, \
|
|
2275
2270
|
INIT, REDUCE, masked_cnt) do { \
|
|
2271
|
+
typedef T_W __ca_u_T_W CA_UNALIGNED; \
|
|
2272
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
2276
2273
|
(acc) = (INIT); \
|
|
2277
2274
|
int8_t __sraK = (st).slab_ndim; \
|
|
2278
2275
|
int8_t __sraOuterK = __sraK - 1; \
|
|
@@ -2326,8 +2323,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2326
2323
|
const char *__sraWQ = (const char *)(p_w) + __sraWoff; \
|
|
2327
2324
|
if ( (m) == NULL ) { \
|
|
2328
2325
|
if ( __sraContig && __sraContigW ) { \
|
|
2329
|
-
const
|
|
2330
|
-
const
|
|
2326
|
+
const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
|
|
2327
|
+
const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
|
|
2331
2328
|
_CA_SIMD_PLUS(acc) \
|
|
2332
2329
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2333
2330
|
T v = __sraSrc[__saj]; \
|
|
@@ -2337,8 +2334,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2337
2334
|
} \
|
|
2338
2335
|
} else { \
|
|
2339
2336
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2340
|
-
T v = *(const
|
|
2341
|
-
T_W w = *(const
|
|
2337
|
+
T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
|
|
2338
|
+
T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
|
|
2342
2339
|
REDUCE; \
|
|
2343
2340
|
idx++; \
|
|
2344
2341
|
} \
|
|
@@ -2346,8 +2343,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2346
2343
|
} else { \
|
|
2347
2344
|
const boolean8_t *__sraMM = (const boolean8_t *)(m) + __sraMoff; \
|
|
2348
2345
|
if ( __sraContig && __sraContigW && __sraMaskU ) { \
|
|
2349
|
-
const
|
|
2350
|
-
const
|
|
2346
|
+
const __ca_u_T *__sraSrc = (const __ca_u_T *) __sraQ; \
|
|
2347
|
+
const __ca_u_T_W *__sraWgt = (const __ca_u_T_W *) __sraWQ; \
|
|
2351
2348
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2352
2349
|
if ( ! __sraMM[__saj] ) { \
|
|
2353
2350
|
T v = __sraSrc[__saj]; \
|
|
@@ -2361,8 +2358,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2361
2358
|
} else { \
|
|
2362
2359
|
for ( ca_size_t __saj = 0; __saj < __sraInnerN; __saj++ ) { \
|
|
2363
2360
|
if ( ! __sraMM[__saj * __sraInnerMS] ) { \
|
|
2364
|
-
T v = *(const
|
|
2365
|
-
T_W w = *(const
|
|
2361
|
+
T v = *(const __ca_u_T *)(__sraQ + __saj * __sraInnerS); \
|
|
2362
|
+
T_W w = *(const __ca_u_T_W *)(__sraWQ + __saj * __sraInnerWS); \
|
|
2366
2363
|
REDUCE; \
|
|
2367
2364
|
} else { \
|
|
2368
2365
|
(masked_cnt)++; \
|
|
@@ -2379,23 +2376,6 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2379
2376
|
(void) idx; \
|
|
2380
2377
|
} while (0)
|
|
2381
2378
|
|
|
2382
|
-
#define CA_SLAB_REDUCE_ARRAY_T_PLUS(T, T_W, st, p, m, st_w, p_w, acc, \
|
|
2383
|
-
INIT, REDUCE) do { \
|
|
2384
|
-
ca_size_t __sra_throwaway_mc = 0; \
|
|
2385
|
-
CA_SLAB_REDUCE_ARRAY_T_PLUS_EX(T, T_W, st, p, m, st_w, p_w, acc, INIT, \
|
|
2386
|
-
REDUCE, __sra_throwaway_mc); \
|
|
2387
|
-
(void) __sra_throwaway_mc; \
|
|
2388
|
-
} while (0)
|
|
2389
|
-
|
|
2390
|
-
#define CA_SLAB_REDUCE_F64(st, p, m, acc, INIT, REDUCE) \
|
|
2391
|
-
CA_SLAB_REDUCE_T(double, st, p, m, acc, INIT, REDUCE)
|
|
2392
|
-
#define CA_SLAB_REDUCE_F32(st, p, m, acc, INIT, REDUCE) \
|
|
2393
|
-
CA_SLAB_REDUCE_T(float, st, p, m, acc, INIT, REDUCE)
|
|
2394
|
-
#define CA_SLAB_REDUCE_I32(st, p, m, acc, INIT, REDUCE) \
|
|
2395
|
-
CA_SLAB_REDUCE_T(int32_t, st, p, m, acc, INIT, REDUCE)
|
|
2396
|
-
#define CA_SLAB_REDUCE_I64(st, p, m, acc, INIT, REDUCE) \
|
|
2397
|
-
CA_SLAB_REDUCE_T(int64_t, st, p, m, acc, INIT, REDUCE)
|
|
2398
|
-
|
|
2399
2379
|
/* CA_SLAB_MAP_T(T_IN, T_OUT, ...): generic per-cell transform from
|
|
2400
2380
|
input slab to output slab, walking both in lockstep. T_IN is the
|
|
2401
2381
|
input element load type, T_OUT is the output element store type.
|
|
@@ -2417,10 +2397,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2417
2397
|
shape-equal CArrays. Mask handling is **not** done by this macro —
|
|
2418
2398
|
if your input is masked, see §8 Mask handling in
|
|
2419
2399
|
docs/authoring/HOW_TO_WRITE_KERNEL.md. Caller is responsible for invoking
|
|
2420
|
-
ca_iter_state_sync_slab on the output state after each slab.
|
|
2421
|
-
|
|
2422
|
-
Convenience aliases below: CA_SLAB_MAP_F64 (T_IN = T_OUT = double). */
|
|
2400
|
+
ca_iter_state_sync_slab on the output state after each slab. */
|
|
2423
2401
|
#define CA_SLAB_MAP_T(T_IN, T_OUT, st_in, p_in, st_out, p_out, MAP_EXPR) do { \
|
|
2402
|
+
typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
|
|
2403
|
+
typedef T_IN __ca_u_T_IN CA_UNALIGNED; \
|
|
2424
2404
|
int8_t __mK = (st_in).slab_ndim; \
|
|
2425
2405
|
int8_t __mOuterK = __mK - 1; \
|
|
2426
2406
|
ca_size_t __mInN = (st_in).slab_dims[__mK - 1]; \
|
|
@@ -2441,8 +2421,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2441
2421
|
const char *__mQi = (const char *)(p_in) + __mIOff; \
|
|
2442
2422
|
char *__mQo = (char *) (p_out) + __mOOff; \
|
|
2443
2423
|
if ( __mIContig && __mOContig ) { \
|
|
2444
|
-
const
|
|
2445
|
-
|
|
2424
|
+
const __ca_u_T_IN *__mSi = (const __ca_u_T_IN *) __mQi; \
|
|
2425
|
+
__ca_u_T_OUT *__mSo = (__ca_u_T_OUT *) __mQo; \
|
|
2446
2426
|
for ( ca_size_t __mj = 0; __mj < __mInN; __mj++ ) { \
|
|
2447
2427
|
T_IN v = __mSi[__mj]; \
|
|
2448
2428
|
T_OUT r; \
|
|
@@ -2451,10 +2431,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2451
2431
|
} \
|
|
2452
2432
|
} else { \
|
|
2453
2433
|
for ( ca_size_t __mj = 0; __mj < __mInN; __mj++ ) { \
|
|
2454
|
-
T_IN v = *(const
|
|
2434
|
+
T_IN v = *(const __ca_u_T_IN *)(__mQi + __mj * __mInS); \
|
|
2455
2435
|
T_OUT r; \
|
|
2456
2436
|
MAP_EXPR; \
|
|
2457
|
-
*(
|
|
2437
|
+
*(__ca_u_T_OUT *)(__mQo + __mj * __mOutS) = r; \
|
|
2458
2438
|
} \
|
|
2459
2439
|
} \
|
|
2460
2440
|
for ( int8_t __mk = __mOuterK - 1; __mk >= 0; __mk-- ) { \
|
|
@@ -2464,9 +2444,6 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2464
2444
|
} \
|
|
2465
2445
|
} while (0)
|
|
2466
2446
|
|
|
2467
|
-
#define CA_SLAB_MAP_F64(st_in, p_in, st_out, p_out, MAP_EXPR) \
|
|
2468
|
-
CA_SLAB_MAP_T(double, double, st_in, p_in, st_out, p_out, MAP_EXPR)
|
|
2469
|
-
|
|
2470
2447
|
/* CA_SLAB_SCAN_T(T_LOAD, T_OUT, ...): cumulative / prefix-scan walk
|
|
2471
2448
|
that combines a reduction (running accumulator) with a map (per-cell
|
|
2472
2449
|
output write). Walks input and output slabs in lockstep; the
|
|
@@ -2506,6 +2483,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2506
2483
|
CA_SLAB_REDUCE_T / CA_SLAB_MAP_T. */
|
|
2507
2484
|
#define CA_SLAB_SCAN_T(T_LOAD, T_OUT, st_in, p_in, m_in, \
|
|
2508
2485
|
st_out, p_out, INIT, STEP) do { \
|
|
2486
|
+
typedef T_LOAD __ca_u_T_LOAD CA_UNALIGNED; \
|
|
2487
|
+
typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
|
|
2509
2488
|
T_OUT acc = (INIT); \
|
|
2510
2489
|
int8_t __ssK = (st_in).slab_ndim; \
|
|
2511
2490
|
int8_t __ssOuterK = __ssK - 1; \
|
|
@@ -2531,8 +2510,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2531
2510
|
char *__ssQo = (char *) (p_out) + __ssOOff; \
|
|
2532
2511
|
if ( (m_in) == NULL ) { \
|
|
2533
2512
|
if ( __ssIContig && __ssOContig ) { \
|
|
2534
|
-
const
|
|
2535
|
-
|
|
2513
|
+
const __ca_u_T_LOAD *__ssSi = (const __ca_u_T_LOAD *) __ssQi; \
|
|
2514
|
+
__ca_u_T_OUT *__ssSo = (__ca_u_T_OUT *) __ssQo; \
|
|
2536
2515
|
for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
|
|
2537
2516
|
T_LOAD v = __ssSi[__sj]; \
|
|
2538
2517
|
T_OUT r; \
|
|
@@ -2541,10 +2520,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2541
2520
|
} \
|
|
2542
2521
|
} else { \
|
|
2543
2522
|
for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
|
|
2544
|
-
T_LOAD v = *(const
|
|
2523
|
+
T_LOAD v = *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
|
|
2545
2524
|
T_OUT r; \
|
|
2546
2525
|
STEP; \
|
|
2547
|
-
*(
|
|
2526
|
+
*(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
|
|
2548
2527
|
} \
|
|
2549
2528
|
} \
|
|
2550
2529
|
} else { \
|
|
@@ -2553,16 +2532,16 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2553
2532
|
T_OUT r; \
|
|
2554
2533
|
if ( ! __ssMM[__ssMaskU ? __sj : __sj * __ssInMS] ) { \
|
|
2555
2534
|
T_LOAD v = __ssIContig \
|
|
2556
|
-
? ((const
|
|
2557
|
-
: *(const
|
|
2535
|
+
? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
|
|
2536
|
+
: *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
|
|
2558
2537
|
STEP; \
|
|
2559
2538
|
} else { \
|
|
2560
2539
|
r = acc; /* masked: write current running aggregate (legacy parity) */ \
|
|
2561
2540
|
} \
|
|
2562
2541
|
if ( __ssOContig ) \
|
|
2563
|
-
((
|
|
2542
|
+
((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
|
|
2564
2543
|
else \
|
|
2565
|
-
*(
|
|
2544
|
+
*(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
|
|
2566
2545
|
} \
|
|
2567
2546
|
} \
|
|
2568
2547
|
for ( int8_t __sk = __ssOuterK - 1; __sk >= 0; __sk-- ) { \
|
|
@@ -2599,6 +2578,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2599
2578
|
(defensive: falls back to holding the sentinel unmasked). */
|
|
2600
2579
|
#define CA_SLAB_SCAN_T_GATED(T_LOAD, T_OUT, st_in, p_in, m_in, \
|
|
2601
2580
|
st_out, p_out, m_out, INIT, STEP) do { \
|
|
2581
|
+
typedef T_LOAD __ca_u_T_LOAD CA_UNALIGNED; \
|
|
2582
|
+
typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
|
|
2602
2583
|
T_OUT acc = (INIT); \
|
|
2603
2584
|
int8_t __ssK = (st_in).slab_ndim; \
|
|
2604
2585
|
int8_t __ssOuterK = __ssK - 1; \
|
|
@@ -2627,8 +2608,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2627
2608
|
int __ssSeen = 0; \
|
|
2628
2609
|
if ( (m_in) == NULL ) { \
|
|
2629
2610
|
if ( __ssIContig && __ssOContig ) { \
|
|
2630
|
-
const
|
|
2631
|
-
|
|
2611
|
+
const __ca_u_T_LOAD *__ssSi = (const __ca_u_T_LOAD *) __ssQi; \
|
|
2612
|
+
__ca_u_T_OUT *__ssSo = (__ca_u_T_OUT *) __ssQo; \
|
|
2632
2613
|
for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
|
|
2633
2614
|
T_LOAD v = __ssSi[__sj]; \
|
|
2634
2615
|
T_OUT r; \
|
|
@@ -2637,10 +2618,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2637
2618
|
} \
|
|
2638
2619
|
} else { \
|
|
2639
2620
|
for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
|
|
2640
|
-
T_LOAD v = *(const
|
|
2621
|
+
T_LOAD v = *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
|
|
2641
2622
|
T_OUT r; \
|
|
2642
2623
|
STEP; \
|
|
2643
|
-
*(
|
|
2624
|
+
*(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
|
|
2644
2625
|
} \
|
|
2645
2626
|
} \
|
|
2646
2627
|
} else { \
|
|
@@ -2649,8 +2630,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2649
2630
|
T_OUT r; \
|
|
2650
2631
|
if ( ! __ssMM[__ssMaskU ? __sj : __sj * __ssInMS] ) { \
|
|
2651
2632
|
T_LOAD v = __ssIContig \
|
|
2652
|
-
? ((const
|
|
2653
|
-
: *(const
|
|
2633
|
+
? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
|
|
2634
|
+
: *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
|
|
2654
2635
|
STEP; \
|
|
2655
2636
|
__ssSeen = 1; \
|
|
2656
2637
|
} else if ( ! __ssSeen && (m_out) != NULL ) { \
|
|
@@ -2660,9 +2641,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2660
2641
|
r = acc; /* masked after first present: hold running extremum */\
|
|
2661
2642
|
} \
|
|
2662
2643
|
if ( __ssOContig ) \
|
|
2663
|
-
((
|
|
2644
|
+
((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
|
|
2664
2645
|
else \
|
|
2665
|
-
*(
|
|
2646
|
+
*(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
|
|
2666
2647
|
} \
|
|
2667
2648
|
} \
|
|
2668
2649
|
for ( int8_t __sk = __ssOuterK - 1; __sk >= 0; __sk-- ) { \
|
|
@@ -2692,6 +2673,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2692
2673
|
"last live value" trail. */
|
|
2693
2674
|
#define CA_SLAB_SCAN_TA(T_LOAD, T_OUT, T_ACC, st_in, p_in, m_in, \
|
|
2694
2675
|
st_out, p_out, INIT, STEP) do { \
|
|
2676
|
+
typedef T_LOAD __ca_u_T_LOAD CA_UNALIGNED; \
|
|
2677
|
+
typedef T_OUT __ca_u_T_OUT CA_UNALIGNED; \
|
|
2695
2678
|
T_ACC acc = (INIT); \
|
|
2696
2679
|
int8_t __ssK = (st_in).slab_ndim; \
|
|
2697
2680
|
int8_t __ssOuterK = __ssK - 1; \
|
|
@@ -2719,16 +2702,16 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2719
2702
|
if ( (m_in) == NULL ) { \
|
|
2720
2703
|
for ( ca_size_t __sj = 0; __sj < __ssInnerN; __sj++ ) { \
|
|
2721
2704
|
T_LOAD v = __ssIContig \
|
|
2722
|
-
? ((const
|
|
2723
|
-
: *(const
|
|
2705
|
+
? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
|
|
2706
|
+
: *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
|
|
2724
2707
|
T_OUT r; \
|
|
2725
2708
|
int first = __ssFirst; \
|
|
2726
2709
|
STEP; \
|
|
2727
2710
|
__ssFirst = 0; \
|
|
2728
2711
|
if ( __ssOContig ) \
|
|
2729
|
-
((
|
|
2712
|
+
((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
|
|
2730
2713
|
else \
|
|
2731
|
-
*(
|
|
2714
|
+
*(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
|
|
2732
2715
|
} \
|
|
2733
2716
|
} else { \
|
|
2734
2717
|
const boolean8_t *__ssMM = (const boolean8_t *)(m_in) + __ssMoff; \
|
|
@@ -2736,8 +2719,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2736
2719
|
T_OUT r; \
|
|
2737
2720
|
if ( ! __ssMM[__ssMaskU ? __sj : __sj * __ssInMS] ) { \
|
|
2738
2721
|
T_LOAD v = __ssIContig \
|
|
2739
|
-
? ((const
|
|
2740
|
-
: *(const
|
|
2722
|
+
? ((const __ca_u_T_LOAD *) __ssQi)[__sj] \
|
|
2723
|
+
: *(const __ca_u_T_LOAD *)(__ssQi + __sj * __ssInS); \
|
|
2741
2724
|
int first = __ssFirst; \
|
|
2742
2725
|
STEP; \
|
|
2743
2726
|
__ssFirst = 0; \
|
|
@@ -2745,9 +2728,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2745
2728
|
r = (T_OUT) 0; /* masked: neutral output for downstream OR */ \
|
|
2746
2729
|
} \
|
|
2747
2730
|
if ( __ssOContig ) \
|
|
2748
|
-
((
|
|
2731
|
+
((__ca_u_T_OUT *) __ssQo)[__sj] = r; \
|
|
2749
2732
|
else \
|
|
2750
|
-
*(
|
|
2733
|
+
*(__ca_u_T_OUT *)(__ssQo + __sj * __ssOutS) = r; \
|
|
2751
2734
|
} \
|
|
2752
2735
|
} \
|
|
2753
2736
|
for ( int8_t __sk = __ssOuterK - 1; __sk >= 0; __sk-- ) { \
|
|
@@ -2758,10 +2741,10 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2758
2741
|
} while (0)
|
|
2759
2742
|
|
|
2760
2743
|
/* ---- Ruby surface ---------------------------------------------------- */
|
|
2761
|
-
/* Called from Init_carray_ext (ruby_carray.c).
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2744
|
+
/* Called from Init_carray_ext (ruby_carray.c). Creates the
|
|
2745
|
+
Ractor-local key for the kept scratch buffer, and in a development
|
|
2746
|
+
build (CARRAY_DEV_BUILD) registers the CArray.t1_* smoke entry points
|
|
2747
|
+
the spec_ai kernel iterator tests use. */
|
|
2765
2748
|
void Init_ca_kernel_iterator (void);
|
|
2766
2749
|
|
|
2767
2750
|
#endif /* CA_KERNEL_ITERATOR_H */
|