carray 3.0.3 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +264 -0
- data/README.md +3 -2
- data/carray.gemspec +1 -1
- data/ext/ca_axis_descriptor.h +27 -24
- data/ext/ca_axis_dispatch.c +1 -6
- data/ext/ca_axis_group.c +27 -27
- data/ext/ca_binop_dispatch.c +1 -29
- data/ext/ca_binop_dispatch.h +0 -1
- data/ext/ca_categorical_iterator.c +205 -134
- data/ext/ca_compare.c +7 -18
- data/ext/ca_group_iter.c +16 -17
- data/ext/ca_kernel_iterator.c +1234 -1994
- data/ext/ca_kernel_iterator.h +391 -437
- data/ext/ca_monop_dispatch.c +82 -59
- data/ext/ca_monop_dispatch.h +1 -0
- data/ext/ca_obj_array.c +96 -39
- data/ext/ca_obj_bincmp.c +24 -21
- data/ext/ca_obj_binop.c +15 -26
- data/ext/ca_obj_bitarray.c +3 -2
- data/ext/ca_obj_bitfield.c +5 -6
- data/ext/ca_obj_block.c +18 -9
- data/ext/ca_obj_byte_swap.c +5 -6
- data/ext/ca_obj_const_string.c +173 -19
- data/ext/ca_obj_face.c +238 -57
- data/ext/ca_obj_face.h +19 -0
- data/ext/ca_obj_fake.c +15 -29
- data/ext/ca_obj_farray.c +6 -3
- data/ext/ca_obj_field.c +17 -5
- data/ext/ca_obj_fixlen_string.c +2 -3
- data/ext/ca_obj_grid.c +406 -132
- data/ext/ca_obj_meld.c +172 -34
- data/ext/ca_obj_moncmp.c +23 -8
- data/ext/ca_obj_monop.c +67 -57
- data/ext/ca_obj_object.c +157 -51
- data/ext/ca_obj_record.c +12 -7
- data/ext/ca_obj_reduce.c +13 -8
- data/ext/ca_obj_refer.c +25 -27
- data/ext/ca_obj_remap.c +30 -9
- data/ext/ca_obj_repeat.c +9 -10
- data/ext/ca_obj_roll.c +6 -5
- data/ext/ca_obj_select.c +14 -9
- data/ext/ca_obj_select_axis.c +15 -47
- data/ext/ca_obj_shift.c +11 -1
- data/ext/ca_obj_stack.c +179 -34
- data/ext/ca_obj_stride.c +162 -115
- data/ext/ca_obj_string.c +71 -4
- data/ext/ca_obj_tile.c +6 -5
- data/ext/ca_obj_time.c +22 -6
- data/ext/ca_obj_timedelta.c +22 -6
- data/ext/ca_obj_transpose.c +5 -3
- data/ext/ca_obj_triop.c +23 -32
- data/ext/ca_obj_window.c +190 -26
- data/ext/ca_sweep_engine.c +2 -2
- data/ext/caframe_csv_reader.c +80 -1
- data/ext/carray.h +181 -75
- data/ext/carray_access.c +184 -96
- data/ext/carray_address_basis.c +343 -87
- data/ext/carray_attribute.c +89 -6
- data/ext/carray_bincount.c +10 -8
- data/ext/carray_broadcast.c +13 -8
- data/ext/carray_build_flags.h +3 -0
- data/ext/carray_call_cfunc.c +108 -0
- data/ext/carray_cast.c +142 -24
- data/ext/carray_cast_func.rb +46 -6
- data/ext/carray_class.c +4 -3
- data/ext/carray_config.h +28 -0
- data/ext/carray_conversion.c +16 -7
- data/ext/carray_copy.c +51 -14
- data/ext/carray_core.c +147 -55
- data/ext/carray_count.c +50 -20
- data/ext/carray_element.c +76 -33
- data/ext/carray_factorize.c +118 -64
- data/ext/carray_generate.c +191 -50
- data/ext/carray_histogram.c +62 -17
- data/ext/carray_hold.c +12 -15
- data/ext/carray_index_classifier.c +388 -176
- data/ext/carray_internal.h +336 -29
- data/ext/carray_kernels_bincmp.c +4453 -0
- data/ext/carray_kernels_binop.c +11368 -0
- data/ext/carray_kernels_init.c +1734 -0
- data/ext/carray_kernels_moncmp.c +2104 -0
- data/ext/carray_kernels_monop.c +18369 -0
- data/ext/carray_kernels_reduce_aggregate.c +23818 -0
- data/ext/carray_kernels_reduce_boolean.c +406 -0
- data/ext/carray_kernels_reduce_cumulative.c +10086 -0
- data/ext/carray_kernels_reduce_extreme.c +17634 -0
- data/ext/carray_kernels_reduce_variance.c +3849 -0
- data/ext/carray_kernels_scan.c +4302 -0
- data/ext/carray_kernels_search.c +33829 -0
- data/ext/carray_kernels_sort.c +10906 -0
- data/ext/carray_kernels_triop.c +1864 -0
- data/ext/carray_key_windows.c +216 -0
- data/ext/carray_lazy.c +170 -111
- data/ext/carray_mask.c +292 -72
- data/ext/carray_median_percentile.c +82 -28
- data/ext/carray_memory_view.c +669 -561
- data/ext/carray_operator.c +124 -47
- data/ext/carray_order.c +384 -76
- data/ext/carray_partition.c +28 -27
- data/ext/carray_random.c +67 -13
- data/ext/carray_round.c +273 -0
- data/ext/carray_scatter.c +63 -38
- data/ext/carray_segment.c +3 -3
- data/ext/carray_slab.c +91 -28
- data/ext/carray_sort.c +35 -35
- data/ext/carray_struct.c +19 -18
- data/ext/carray_test.c +26 -5
- data/ext/carray_undef.c +4 -0
- data/ext/carray_utils.c +246 -45
- data/ext/mk_call_cfunc.rb +19 -2
- data/ext/mkkernel.rb +198 -240
- data/ext/ruby_carray.c +10 -4
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +126 -22
- data/lib/carray/autoload_carray.rb +8 -31
- data/lib/carray/autoload_method_extension.rb +14 -5
- data/lib/carray/axis_group.rb +255 -106
- data/lib/carray/basics.rb +78 -10
- data/lib/carray/bincount_nd.rb +12 -2
- data/lib/carray/block_iterator.rb +126 -23
- data/lib/carray/boolean_reduce.rb +17 -17
- data/lib/carray/categorical.rb +165 -43
- data/lib/carray/categorical_iterator.rb +130 -27
- data/lib/carray/complex.rb +5 -5
- data/lib/carray/conditional.rb +67 -18
- data/lib/carray/const_string.rb +14 -5
- data/lib/carray/construct.rb +15 -11
- data/lib/carray/data_type_extension.rb +1 -0
- data/lib/carray/distance_limit.rb +77 -0
- data/lib/carray/fixlen_string.rb +7 -1
- data/lib/carray/frame/column_verbs.rb +70 -0
- data/lib/carray/frame/convert.rb +69 -0
- data/lib/carray/frame/csv_parser.rb +111 -17
- data/lib/carray/frame/describe.rb +3 -2
- data/lib/carray/frame/frame.rb +246 -21
- data/lib/carray/frame/group.rb +510 -95
- data/lib/carray/frame/io.rb +115 -28
- data/lib/carray/frame/join.rb +312 -51
- data/lib/carray/frame/records.rb +83 -7
- data/lib/carray/frame/reshape.rb +42 -35
- data/lib/carray/frame/sort.rb +7 -19
- data/lib/carray/frame/verbs.rb +188 -48
- data/lib/carray/frame/windows.rb +117 -0
- data/lib/carray/frame.rb +2 -0
- data/lib/carray/fuse_source.rb +11 -4
- data/lib/carray/fusion.rb +40 -16
- data/lib/carray/histogram.rb +6 -1
- data/lib/carray/inspect.rb +31 -8
- data/lib/carray/iterator.rb +85 -7
- data/lib/carray/key_windows.rb +620 -0
- data/lib/carray/lazy.rb +182 -183
- data/lib/carray/mask_gap_fill.rb +63 -57
- data/lib/carray/meld_reduce.rb +12 -14
- data/lib/carray/methods/bin.rb +26 -18
- data/lib/carray/methods/bincount.rb +16 -0
- data/lib/carray/methods/bit_string.rb +1 -1
- data/lib/carray/methods/choose.rb +16 -3
- data/lib/carray/methods/composition.rb +3 -1
- data/lib/carray/methods/gather_nd.rb +14 -5
- data/lib/carray/methods/insert_block.rb +2 -2
- data/lib/carray/methods/is_between.rb +59 -0
- data/lib/carray/methods/is_in.rb +53 -36
- data/lib/carray/methods/join.rb +1 -6
- data/lib/carray/methods/lag.rb +49 -0
- data/lib/carray/methods/locate_addr.rb +47 -51
- data/lib/carray/methods/mode.rb +1 -1
- data/lib/carray/methods/nunique.rb +11 -6
- data/lib/carray/methods/resize.rb +3 -3
- data/lib/carray/methods/value_counts.rb +4 -11
- data/lib/carray/record_order.rb +3 -3
- data/lib/carray/rng.rb +3 -0
- data/lib/carray/runtime.rb +53 -8
- data/lib/carray/segment_iterator.rb +313 -82
- data/lib/carray/serialize.rb +24 -10
- data/lib/carray/slab_iterator.rb +50 -0
- data/lib/carray/string.rb +105 -5
- data/lib/carray/string_operation_extension.rb +8 -1
- data/lib/carray/struct.rb +9 -21
- data/lib/carray/struct_builder.rb +19 -8
- data/lib/carray/time.rb +726 -104
- data/lib/carray/window_iterator.rb +122 -16
- data/lib/carray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +4 -0
- data/yard-stubs/ca_obj_grid.rb +21 -10
- data/yard-stubs/ca_obj_refer.rb +2 -1
- data/yard-stubs/ca_obj_roll.rb +2 -2
- data/yard-stubs/ca_obj_shift.rb +5 -1
- data/yard-stubs/ca_obj_tile.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +8 -3
- data/yard-stubs/carray_access.rb +6 -1
- data/yard-stubs/carray_attribute.rb +0 -5
- data/yard-stubs/carray_cast.rb +10 -0
- data/yard-stubs/carray_conversion.rb +2 -1
- data/yard-stubs/carray_copy.rb +4 -2
- data/yard-stubs/carray_count.rb +4 -4
- data/yard-stubs/carray_element.rb +10 -4
- data/yard-stubs/carray_generate.rb +21 -1
- data/yard-stubs/carray_mask.rb +26 -2
- data/yard-stubs/carray_math.rb +27 -1
- data/yard-stubs/carray_median_percentile.rb +5 -2
- data/yard-stubs/carray_memory_view.rb +40 -23
- data/yard-stubs/carray_order.rb +29 -18
- data/yard-stubs/carray_random.rb +13 -4
- data/yard-stubs/carray_scatter.rb +14 -9
- data/yard-stubs/carray_slab.rb +9 -2
- data/yard-stubs/carray_sort.rb +6 -5
- data/yard-stubs/carray_test.rb +5 -3
- metadata +25 -3
- data/lib/carray/arrow_tensor.rb +0 -401
- data/lib/carray/table.rb +0 -108
data/ext/ca_kernel_iterator.c
CHANGED
|
@@ -1,32 +1,45 @@
|
|
|
1
1
|
/* ---------------------------------------------------------------------------
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
-
|
|
3
|
+
ca_kernel_iterator.c -- the kernel iterator engine
|
|
4
|
+
|
|
5
|
+
A C kernel asks for the cells of an array, one slab at a time, and gets
|
|
6
|
+
them whatever the array is: an entity, a view of a view, a selection, a
|
|
7
|
+
lazy expression. This file decides, for each source, how to deliver
|
|
8
|
+
them (borrow its memory, gather a copy, walk a stack's parents) and, on
|
|
9
|
+
a write, how the cells the kernel changed get back to the array.
|
|
10
|
+
|
|
11
|
+
Where to read first:
|
|
12
|
+
- writing a kernel: guides/devel/11_kernel_iterator.md and
|
|
13
|
+
docs/authoring/HOW_TO_WRITE_KERNEL.md; the surface itself is
|
|
14
|
+
ca_kernel_iterator.h
|
|
15
|
+
- how the engine works (its routes, the promises every route keeps,
|
|
16
|
+
and which part keeps each): docs/internal/KernelIteratorEngine.md
|
|
17
|
+
|
|
18
|
+
Map of this file:
|
|
19
|
+
- scratch buffers (ki_data_alloc / ki_data_free) and helpers
|
|
20
|
+
- classification: ca_iter_classify_source, ca_iter_route_source
|
|
21
|
+
- the promise parts every route calls: ki_take_mask_copy,
|
|
22
|
+
ki_gather_view, ki_scratch_push_changed, ki_attach_stack_parents,
|
|
23
|
+
ki_fiber_contig_data / _mask
|
|
24
|
+
- opening a walk: init_l1, and for L2 ki_choose_l2_route (decides)
|
|
25
|
+
then ki_open_l2 (builds the state, with ki_lay_out_slabs)
|
|
26
|
+
- handing out slabs: next_slab, next_slab_axes
|
|
27
|
+
- writing back: sync_slab
|
|
28
|
+
- closing: finish, close, and the raise-safe entry points
|
|
29
|
+
(ca_iter_protect, ca_iter_ensure)
|
|
30
|
+
- development-build test surface (CARRAY_DEV_BUILD), at the end
|
|
31
|
+
|
|
32
|
+
A route added for speed is a shortcut, tagged next to its code with
|
|
33
|
+
"Shortcut NAME" and a fixed set of fields; each one is switched off by
|
|
34
|
+
ca_fast_paths_enabled().
|
|
22
35
|
|
|
23
36
|
---------------------------------------------------------------------------- */
|
|
24
37
|
|
|
25
38
|
#include "carray.h"
|
|
26
39
|
#include "carray_internal.h" /* per-obj_type view constructors */
|
|
27
40
|
#include "ca_kernel_iterator.h"
|
|
28
|
-
#include "ca_monop_dispatch.h" /*
|
|
29
|
-
#include "ca_obj_face.h" /*
|
|
41
|
+
#include "ca_monop_dispatch.h" /* ca_monop_view_is_single_cast */
|
|
42
|
+
#include "ca_obj_face.h" /* ca_strip_face, to walk what a Face wraps */
|
|
30
43
|
|
|
31
44
|
#include <assert.h>
|
|
32
45
|
#include <string.h>
|
|
@@ -47,7 +60,24 @@
|
|
|
47
60
|
reduction itself. The kept buffer is owned by nobody, and a walk that
|
|
48
61
|
takes it owns it alone until it frees it again. Each Ractor keeps its
|
|
49
62
|
own (ractor-local storage), since walks in different Ractors run at
|
|
50
|
-
the same time.
|
|
63
|
+
the same time.
|
|
64
|
+
|
|
65
|
+
Shortcut KEEP_SCRATCH (name tag; docs/internal/KernelIteratorEngine.md
|
|
66
|
+
section 3.4 has the table of all shortcuts).
|
|
67
|
+
Layer data: where a scratch comes from. The walk is unchanged.
|
|
68
|
+
Speeds up walks with a scratch of 1 MiB or more, run one after
|
|
69
|
+
another.
|
|
70
|
+
Why no difference measured on macOS (a per-axis sum of a
|
|
71
|
+
2000x2000 fake view); the page faults it
|
|
72
|
+
avoids are expected to cost more on Linux (unmeasured).
|
|
73
|
+
Enters ki_data_alloc: bytes >= KI_KEEP_MIN and a kept buffer
|
|
74
|
+
that fits; ki_data_free: KI_KEEP_MIN <= bytes <=
|
|
75
|
+
KI_KEEP_MAX.
|
|
76
|
+
Promises 4 (a kept buffer has no owner, so no exit can leave it
|
|
77
|
+
held); 7 (object cells are cleared and held either way).
|
|
78
|
+
Switch ca_fast_paths_enabled(): off, every scratch is fresh.
|
|
79
|
+
Checked by test_iter_ractor_scratch.rb; the buffer count in
|
|
80
|
+
test_iter_delivery_detectors.rb. */
|
|
51
81
|
|
|
52
82
|
#define KI_KEEP_MIN ((ca_size_t) 1 << 20)
|
|
53
83
|
#define KI_KEEP_MAX ((ca_size_t) 64 << 20)
|
|
@@ -86,11 +116,24 @@ ki_kept_slot (void)
|
|
|
86
116
|
return k;
|
|
87
117
|
}
|
|
88
118
|
|
|
119
|
+
/* Development build: the number of buffers ki_data_alloc has handed out
|
|
120
|
+
that ki_data_free has not taken back (CArray.__iter_buffers__). A walk
|
|
121
|
+
that ends leaves the count where it found it; the spec suite compares it
|
|
122
|
+
around each walk. */
|
|
123
|
+
#ifdef CARRAY_DEV_BUILD
|
|
124
|
+
static long ki_buffers_out = 0;
|
|
125
|
+
/* atomic: walks run in several Ractors at once */
|
|
126
|
+
#define KI_BUFFERS_OUT(d) ( (void) __atomic_add_fetch(&ki_buffers_out, (d), __ATOMIC_RELAXED) )
|
|
127
|
+
#else
|
|
128
|
+
#define KI_BUFFERS_OUT(d) ((void) 0)
|
|
129
|
+
#endif
|
|
130
|
+
|
|
89
131
|
static void *
|
|
90
132
|
ki_data_alloc (int8_t data_type, ca_size_t bytes)
|
|
91
133
|
{
|
|
92
134
|
void *p = NULL;
|
|
93
|
-
|
|
135
|
+
KI_BUFFERS_OUT(1);
|
|
136
|
+
if ( bytes >= KI_KEEP_MIN && ca_fast_paths_enabled() ) {
|
|
94
137
|
ki_kept_t *k = ki_kept_slot();
|
|
95
138
|
if ( k->p && bytes <= k->bytes ) {
|
|
96
139
|
p = k->p;
|
|
@@ -102,10 +145,8 @@ ki_data_alloc (int8_t data_type, ca_size_t bytes)
|
|
|
102
145
|
p = xmalloc(bytes > 0 ? bytes : 1);
|
|
103
146
|
}
|
|
104
147
|
if ( data_type == CA_OBJECT ) {
|
|
105
|
-
VALUE *v = (VALUE *) p;
|
|
106
148
|
ca_size_t n = bytes / (ca_size_t) sizeof(VALUE);
|
|
107
|
-
|
|
108
|
-
for ( i = 0; i < n; i++ ) v[i] = Qnil;
|
|
149
|
+
ca_object_cells_clear(p, n);
|
|
109
150
|
ca_gc_hold_buffer(p, n);
|
|
110
151
|
}
|
|
111
152
|
return p;
|
|
@@ -115,8 +156,9 @@ ki_data_alloc (int8_t data_type, ca_size_t bytes)
|
|
|
115
156
|
static void
|
|
116
157
|
ki_data_free (void *p, ca_size_t bytes)
|
|
117
158
|
{
|
|
159
|
+
KI_BUFFERS_OUT(-1);
|
|
118
160
|
ca_gc_release_buffer(p);
|
|
119
|
-
if ( bytes >= KI_KEEP_MIN && bytes <= KI_KEEP_MAX ) {
|
|
161
|
+
if ( bytes >= KI_KEEP_MIN && bytes <= KI_KEEP_MAX && ca_fast_paths_enabled() ) {
|
|
120
162
|
ki_kept_t *k = ki_kept_slot();
|
|
121
163
|
if ( bytes > k->bytes ) {
|
|
122
164
|
void *old = k->p;
|
|
@@ -136,26 +178,28 @@ ki_data_free (void *p, ca_size_t bytes)
|
|
|
136
178
|
extern int ca_attach_is_alias (void *ap);
|
|
137
179
|
extern int ca_root_lends_no_memory (void *ap);
|
|
138
180
|
|
|
139
|
-
/* CAStride family attach-fn marker
|
|
181
|
+
/* CAStride family attach-fn marker. */
|
|
140
182
|
extern ca_operation_function_t ca_stride_func;
|
|
141
183
|
|
|
142
|
-
/* Descriptor framework view ops
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
(
|
|
146
|
-
|
|
147
|
-
|
|
184
|
+
/* Descriptor framework view ops. Each view's describe_axes emits the
|
|
185
|
+
per-axis descriptor + parent dim snapshot consumed by the
|
|
186
|
+
ca_axis_dispatch_* engine. CAShift shares CAWindow's operation table
|
|
187
|
+
(ca_shift_func is a copy with only free / clone / create_mask
|
|
188
|
+
overridden), so it is recognised by its attach pointer matching
|
|
189
|
+
ca_window_func.attach. */
|
|
148
190
|
extern ca_operation_function_t ca_select_axis_func;
|
|
149
191
|
extern ca_operation_function_t ca_grid_func;
|
|
150
192
|
extern ca_operation_function_t ca_select_func;
|
|
151
193
|
extern ca_operation_function_t ca_window_func;
|
|
152
|
-
/* ca_mapping_func retired in R.3 (PROPOSAL_CAMAPPING_REMOVAL). */
|
|
153
194
|
|
|
154
|
-
/*
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
195
|
+
/* Operation tables the classifier compares a source's attach slot
|
|
196
|
+
against. Every view below is delivered as CA_ITER_SRC_ATTACH: the
|
|
197
|
+
engine gathers its cells into a scratch buffer of its own through the
|
|
198
|
+
view's transfer (ki_gather_view = ca_xfer_all GET) and, on a WRITE
|
|
199
|
+
walk, sends back the cells the kernel changed (ki_scratch_push_changed
|
|
200
|
+
= ca_xfer_addrs, or ca_xfer_all PUT when every cell changed). The
|
|
201
|
+
view's attach / sync slots are not called. Cast / byte swap / bit
|
|
202
|
+
unpack / reduction views: */
|
|
159
203
|
extern ca_operation_function_t ca_fake_func;
|
|
160
204
|
extern ca_operation_function_t ca_byte_swap_func;
|
|
161
205
|
extern ca_operation_function_t ca_bitfield_func;
|
|
@@ -163,42 +207,24 @@ extern ca_operation_function_t ca_bitarray_func;
|
|
|
163
207
|
extern ca_operation_function_t ca_reduce_func;
|
|
164
208
|
extern ca_operation_function_t ca_object_func;
|
|
165
209
|
|
|
166
|
-
/*
|
|
167
|
-
|
|
168
|
-
descriptor framework's kind enum {STRIDE, INDEX, SHIFT}, so the
|
|
169
|
-
innermost-STRIDE L2 alias path is not used; but the view-specific
|
|
170
|
-
func_attach (embed-region gather) materialises src->ptr and func_sync
|
|
171
|
-
scatters back — structurally identical to SRC_ATTACH. Fills the gap
|
|
172
|
-
in the "deliver" principle (= these were previously rejected via
|
|
173
|
-
SRC_NONE). */
|
|
210
|
+
/* CATile / CARoll: the per-cell modulo wrap has no STRIDE / INDEX /
|
|
211
|
+
SHIFT descriptor, so they are delivered as SRC_ATTACH (above). */
|
|
174
212
|
extern ca_operation_function_t ca_tile_func;
|
|
175
213
|
extern ca_operation_function_t ca_roll_func;
|
|
176
214
|
|
|
177
|
-
/*
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
func_sync scatters back via xfer_all PUT. kernel sees a flat
|
|
181
|
-
contig slab over the stacked output of shape (K, *parent_shape). */
|
|
215
|
+
/* CAStack / CAMeld: SRC_ATTACH (above). A stack can also be walked
|
|
216
|
+
through its parents directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K,
|
|
217
|
+
see init_l2). */
|
|
182
218
|
extern ca_operation_function_t ca_stack_func;
|
|
183
219
|
extern ca_operation_function_t ca_meld_func;
|
|
184
220
|
|
|
185
|
-
/*
|
|
186
|
-
|
|
187
|
-
SRC_ATTACH source. Per-element gather has no STRIDE structure to
|
|
188
|
-
preserve, so SRC_ATTACH (scratch materialise via ca_remap_func_attach)
|
|
189
|
-
is the natural acceptance path. func_attach allocates a fresh
|
|
190
|
-
buffer and runs xfer_all(GET); kernel sees a flat contig slab.
|
|
191
|
-
On WRITE, ca_sync routes through xfer_all(PUT). */
|
|
221
|
+
/* CARemap (internal, no Ruby class constant): a per-element gather with
|
|
222
|
+
no STRIDE structure, delivered as SRC_ATTACH (above). */
|
|
192
223
|
extern ca_operation_function_t ca_remap_func;
|
|
193
224
|
|
|
194
|
-
/*
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
the lazy tree's materialise into a fresh contig buffer; kernel sees
|
|
198
|
-
a flat slab. Same SRC_ATTACH structural pattern as CAFake et al,
|
|
199
|
-
so a 4-line addition to the classify_source list opens all 22
|
|
200
|
-
mkkernel-generated reduction ops (sum / count / mean / variance /
|
|
201
|
-
argmin / ...) to lazy operands — `(a.lazy + b).sum` now works. */
|
|
225
|
+
/* Lazy element-wise views (CAMonOp / CABinOp / CABinCmp / CAMonCmp /
|
|
226
|
+
CATriOp): SRC_ATTACH (above); gathering evaluates the expression into
|
|
227
|
+
the scratch, so every generated reduction accepts a lazy operand. */
|
|
202
228
|
extern ca_operation_function_t ca_monop_func;
|
|
203
229
|
extern ca_operation_function_t ca_binop_func;
|
|
204
230
|
extern ca_operation_function_t ca_bincmp_func;
|
|
@@ -255,16 +281,27 @@ ca_iter_register_source_kind (int obj_type, uint8_t kind)
|
|
|
255
281
|
ca_iter_registered_source_kind[obj_type] = kind;
|
|
256
282
|
}
|
|
257
283
|
|
|
258
|
-
/* Classify the source by
|
|
259
|
-
|
|
284
|
+
/* Classify the source by how its cells are delivered. Returns
|
|
285
|
+
CA_ITER_SRC_NONE only for a NULL source or one with no attach slot;
|
|
286
|
+
every other view is delivered, as SRC_ATTACH if nothing more specific
|
|
287
|
+
applies. */
|
|
260
288
|
static uint8_t
|
|
261
289
|
ca_iter_classify_source (CArray *src)
|
|
262
290
|
{
|
|
263
291
|
if ( src == NULL ) return CA_ITER_SRC_NONE;
|
|
292
|
+
|
|
293
|
+
/* Development build, CARRAY_GENERAL_PATH=2: every source takes the
|
|
294
|
+
transfer walk (see ki_open_by_transfer), the reference the other
|
|
295
|
+
routes are checked against. */
|
|
296
|
+
if ( ca_iter_by_transfer_only() ) {
|
|
297
|
+
return ca_func[src->obj_type].attach != NULL ? CA_ITER_SRC_ATTACH
|
|
298
|
+
: CA_ITER_SRC_NONE;
|
|
299
|
+
}
|
|
300
|
+
|
|
264
301
|
if ( ca_is_entity(src) ) return CA_ITER_SRC_CASTRIDE;
|
|
265
302
|
if ( ca_iter_is_castride_family(src) ) return CA_ITER_SRC_CASTRIDE;
|
|
266
303
|
|
|
267
|
-
/* Externally installed obj_types declare their routing
|
|
304
|
+
/* Externally installed obj_types declare their routing.
|
|
268
305
|
Placed after the two structural cases and before the built-in list:
|
|
269
306
|
entity and CAStride-family sources are recognised from the struct
|
|
270
307
|
itself and are read directly, so a registration must not be able to
|
|
@@ -276,95 +313,73 @@ ca_iter_classify_source (CArray *src)
|
|
|
276
313
|
if ( kind != CA_ITER_SRC_NONE ) return kind;
|
|
277
314
|
}
|
|
278
315
|
|
|
279
|
-
/* Descriptor
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
check below. */
|
|
316
|
+
/* Descriptor views: CASelectAxis / CAGrid / CASelect / CAWindow.
|
|
317
|
+
CAShift shares ca_window_func's attach, so it matches the CAWindow
|
|
318
|
+
check. */
|
|
283
319
|
void *attach = ca_func[src->obj_type].attach;
|
|
284
320
|
if ( attach == ca_select_axis_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
285
321
|
if ( attach == ca_grid_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
286
322
|
if ( attach == ca_select_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
287
323
|
if ( attach == ca_window_func.attach ) return CA_ITER_SRC_DESCRIPTOR; /* + CAShift */
|
|
288
324
|
|
|
289
|
-
/*
|
|
290
|
-
|
|
325
|
+
/* The views below are SRC_ATTACH: gathered into scratch through their
|
|
326
|
+
own transfer (ki_gather_view) and, on WRITE, sent back the changed
|
|
327
|
+
cells (ki_scratch_push_changed). Their attach / sync slots are not
|
|
328
|
+
called by the engine. */
|
|
291
329
|
if ( attach == ca_fake_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
292
330
|
if ( attach == ca_byte_swap_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
293
331
|
if ( attach == ca_bitfield_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
294
332
|
if ( attach == ca_bitarray_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
295
333
|
if ( attach == ca_reduce_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
296
|
-
/*
|
|
297
|
-
|
|
298
|
-
copy_data, func_sync scatters back via Ruby sync_data, CA_FLAG_
|
|
299
|
-
READ_ONLY auto-rejects WRITE via ca_is_readonly). Bench gate
|
|
300
|
-
n/a (Ruby callback overhead structurally dominant). */
|
|
334
|
+
/* CAObject: each transfer calls the Ruby callback. A read-only
|
|
335
|
+
CAObject is refused for WRITE by the ca_is_readonly check. */
|
|
301
336
|
if ( attach == ca_object_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
302
337
|
|
|
303
|
-
/* F-2 follow-up: CATile / CARoll via SRC_ATTACH (embed-region gather
|
|
304
|
-
materialises src->ptr; ca_sync scatters back per view-specific
|
|
305
|
-
semantics — CATile tile decomposition, CARoll cyclic permutation). */
|
|
306
338
|
if ( attach == ca_tile_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
307
339
|
if ( attach == ca_roll_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
308
340
|
|
|
309
|
-
/*
|
|
310
|
-
|
|
311
|
-
bytes alloc, caller responsibility per MEMO §3.4); func_sync
|
|
312
|
-
scatters back via xfer_all PUT. kernel sees a flat contig slab
|
|
313
|
-
over the stacked output. Routine ndim mismatch is irrelevant
|
|
314
|
-
here (= attach delivers a self-owned buffer of shape (K, *parent
|
|
315
|
-
shape), kernel reads/writes that buffer). */
|
|
341
|
+
/* CAStack: SRC_ATTACH here; init_l2 may instead walk its parents
|
|
342
|
+
directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K). */
|
|
316
343
|
if ( attach == ca_stack_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
317
344
|
|
|
318
|
-
/* CAMeld
|
|
319
|
-
|
|
320
|
-
seg_offset[k] * tail_bytes offsets; func_sync scatters back. Same
|
|
321
|
-
SRC_ATTACH structural pattern as CAStack; reduce hot paths bypass
|
|
322
|
-
this via the per-parent decompose in lib/carray/meld_reduce.rb. */
|
|
345
|
+
/* CAMeld: reduce hot paths avoid the gather through the per-parent
|
|
346
|
+
decompose in lib/carray/meld_reduce.rb. */
|
|
323
347
|
if ( attach == ca_meld_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
324
348
|
|
|
325
|
-
/* M.6: CARemap — per-element gather, internal-only. Same SRC_ATTACH
|
|
326
|
-
structural pattern (func_attach materialises via xfer_all GET,
|
|
327
|
-
func_sync via xfer_all PUT). */
|
|
328
349
|
if ( attach == ca_remap_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
329
350
|
|
|
330
|
-
/*
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
slab. Read-only (= CA_FLAG_READ_ONLY) so WRITE auto-rejects at
|
|
334
|
-
the ca_is_readonly check. Opens 22 mkkernel-generated reduction
|
|
335
|
-
ops to lazy operands. */
|
|
351
|
+
/* Lazy views (CAMonOp / CABinOp / CABinCmp / CAMonCmp / CATriOp):
|
|
352
|
+
gathering evaluates the expression. They are read-only, so WRITE is
|
|
353
|
+
refused by the ca_is_readonly check. */
|
|
336
354
|
if ( attach == ca_monop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
337
355
|
if ( attach == ca_binop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
338
356
|
if ( attach == ca_bincmp_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
339
357
|
if ( attach == ca_moncmp_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
340
358
|
if ( attach == ca_triop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
341
359
|
|
|
342
|
-
/* Anything else
|
|
343
|
-
register
|
|
344
|
-
|
|
345
|
-
|
|
360
|
+
/* Anything else -- a view class installed from outside that did not
|
|
361
|
+
register, or an in-tree class with no case above (such as the mask
|
|
362
|
+
views CAMaskOfOperands and CAObjectMask). It is classified as a
|
|
363
|
+
registration would declare it, SRC_ATTACH, and delivered by the
|
|
364
|
+
transfer walk through its xfer slots rather than refused. */
|
|
346
365
|
if ( attach != NULL ) return CA_ITER_SRC_ATTACH;
|
|
347
366
|
|
|
348
367
|
return CA_ITER_SRC_NONE;
|
|
349
368
|
}
|
|
350
369
|
|
|
351
|
-
/*
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
Per-walk overhead = 1 describe_axes call (descriptor sources only),
|
|
365
|
-
which init was going to do anyway -- so the routing is net zero cost
|
|
366
|
-
compared to the pre-rev6 path where classify_source + init both
|
|
367
|
-
computed describe_axes redundantly. See prep doc rev2 §4.1.1. */
|
|
370
|
+
/* Routes a source by running classify_source and, for descriptor-routed views, also calling
|
|
371
|
+
describe_axes to inspect the innermost axis kind. A descriptor view
|
|
372
|
+
comes back as SRC_DESCRIPTOR_L2_ALIASABLE (innermost axis STRIDE) or
|
|
373
|
+
SRC_DESCRIPTOR, and then out_descs / out_parent_dims / out_ndim are
|
|
374
|
+
filled so the caller does not re-call describe_axes. A grid with holes
|
|
375
|
+
and a view whose parent lends no memory come back as SRC_ATTACH, and
|
|
376
|
+
the out_* arguments are not touched; nor are they for any other kind,
|
|
377
|
+
whose classify_source verdict is returned as-is.
|
|
378
|
+
|
|
379
|
+
Cost: classify_source is O(1) pointer compares; a descriptor view
|
|
380
|
+
adds one describe_axes call (O(ndim)), which init would make anyway,
|
|
381
|
+
and init reads the descriptors from here instead of describing the
|
|
382
|
+
axes again. */
|
|
368
383
|
static uint8_t ca_iter_classify_source (CArray *src);
|
|
369
384
|
static void ca_iter_describe_axes (CArray *src, ca_axis_desc_t *,
|
|
370
385
|
ca_size_t *, int8_t *);
|
|
@@ -378,6 +393,13 @@ ca_iter_route_source (CArray *src,
|
|
|
378
393
|
uint8_t kind = ca_iter_classify_source(src);
|
|
379
394
|
if ( kind != CA_ITER_SRC_DESCRIPTOR ) return kind;
|
|
380
395
|
|
|
396
|
+
/* A grid with holes (masked index cells) is not described by the engine
|
|
397
|
+
either: its holes are delivered by its own transfers. */
|
|
398
|
+
if ( ca_func[src->obj_type].attach == ca_grid_func.attach
|
|
399
|
+
&& ca_grid_has_holes(src) ) {
|
|
400
|
+
return CA_ITER_SRC_ATTACH;
|
|
401
|
+
}
|
|
402
|
+
|
|
381
403
|
/* The descriptor walk reads the parent's memory, which such a parent
|
|
382
404
|
only has after a whole materialise -- however few cells the view
|
|
383
405
|
selects. Deliver through the view's own transfer instead: SRC_ATTACH
|
|
@@ -398,9 +420,8 @@ ca_iter_route_source (CArray *src,
|
|
|
398
420
|
return CA_ITER_SRC_DESCRIPTOR;
|
|
399
421
|
}
|
|
400
422
|
|
|
401
|
-
/* Dispatch to the view's describe_axes
|
|
402
|
-
|
|
403
|
-
typedef). */
|
|
423
|
+
/* Dispatch to the view's describe_axes, keyed on the attach pointer
|
|
424
|
+
(CAShift shares CAWindow's). */
|
|
404
425
|
static void
|
|
405
426
|
ca_iter_describe_axes (CArray *src,
|
|
406
427
|
ca_axis_desc_t *out_descs,
|
|
@@ -435,7 +456,8 @@ ca_iter_describe_axes (CArray *src,
|
|
|
435
456
|
*out_ndim = 0;
|
|
436
457
|
}
|
|
437
458
|
|
|
438
|
-
/* Public alias eligibility predicate
|
|
459
|
+
/* Public alias eligibility predicate: whether a walk at `level` could
|
|
460
|
+
read the source's own memory in place. */
|
|
439
461
|
int
|
|
440
462
|
ca_iter_can_alias (void *ap, int level)
|
|
441
463
|
{
|
|
@@ -451,9 +473,8 @@ ca_iter_can_alias (void *ap, int level)
|
|
|
451
473
|
return ca_iter_is_castride_family(ca);
|
|
452
474
|
|
|
453
475
|
default:
|
|
454
|
-
/* L3 (and any future level)
|
|
455
|
-
|
|
456
|
-
probe ahead. */
|
|
476
|
+
/* L3 (and any future level) is not implemented; answer as L1 so
|
|
477
|
+
the predicate stays well-defined for callers that probe ahead. */
|
|
457
478
|
return ca_attach_is_alias(ca);
|
|
458
479
|
}
|
|
459
480
|
}
|
|
@@ -473,24 +494,24 @@ ca_iter_build_rowmajor_strides (ca_size_t *strides,
|
|
|
473
494
|
}
|
|
474
495
|
}
|
|
475
496
|
|
|
476
|
-
/* ----
|
|
497
|
+
/* ---- PER_FIBER_FUSED: which views take it ----------------------------- */
|
|
477
498
|
|
|
478
|
-
/*
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
499
|
+
/* A src_kind branch, then a check of the view's operation table, then of
|
|
500
|
+
the fiber axis's stride. Returns nonzero for
|
|
501
|
+
CAFake / CAByteSwap / a single-cast CAMonOp under SRC_ATTACH when the
|
|
502
|
+
fiber is contiguous in the parent, and for CAWindow / CAShift under
|
|
503
|
+
SRC_DESCRIPTOR; zero otherwise.
|
|
483
504
|
|
|
484
505
|
The fiber_axis_stride argument is the effective byte stride of the
|
|
485
506
|
fiber axis on the view (= what next_slab_axes would yield as
|
|
486
|
-
slab_strides[0]).
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
Called from
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
state
|
|
507
|
+
slab_strides[0]). fiber_axis_stride == src->bytes means the fiber is
|
|
508
|
+
a contiguous run of the parent's memory, which is what lets the view's
|
|
509
|
+
region transfer deliver it in one pass.
|
|
510
|
+
|
|
511
|
+
Called from two places in init_l2: the CA_SLAB_AXES descriptor branch
|
|
512
|
+
and the SRC_ATTACH branch, each before it would materialise. When it
|
|
513
|
+
returns nonzero the caller skips the materialise and sets alias_mode =
|
|
514
|
+
CA_ITER_ALIAS_PER_FIBER_FUSED with the fiber dispatch state filled. */
|
|
494
515
|
static int
|
|
495
516
|
ca_iter_should_per_fiber_fused (CArray *src,
|
|
496
517
|
int src_kind,
|
|
@@ -499,27 +520,22 @@ ca_iter_should_per_fiber_fused (CArray *src,
|
|
|
499
520
|
uint32_t flags)
|
|
500
521
|
{
|
|
501
522
|
(void) fiber_axis;
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
materialise (CAFake) / 16-18% (CAByteSwap). Non-innermost (=
|
|
512
|
-
fiber_axis_stride != bytes) was a loser, so the stride gate
|
|
513
|
-
keeps current path for that case.
|
|
514
|
-
|
|
515
|
-
Phase 6 P.6.2.d (Q13 α): single-cast CAMonOp (= post-migration
|
|
516
|
-
successor of CAFake numeric path) is recognised via
|
|
517
|
-
ca_monop_view_is_single_cast and routed through the same F.6.2
|
|
518
|
-
fast path. Chain CAMonOp (depth ≥ 2) is excluded — its
|
|
519
|
-
materialise path (= arena-pooled scratches from Phase 3) is
|
|
520
|
-
structurally different from the X.4 inner-contig fast path. */
|
|
523
|
+
|
|
524
|
+
/* CAFake / CAByteSwap: when the fiber is a contiguous run of the
|
|
525
|
+
parent's memory, ca_xfer_stride takes the view's converting fast
|
|
526
|
+
path (ca_xfer_stride_transform_fused) and delivers the fiber in one
|
|
527
|
+
pass. A fiber along another axis was measured slower this way, so
|
|
528
|
+
the stride gate keeps it on the gathered route. A single-cast
|
|
529
|
+
CAMonOp (ca_monop_view_is_single_cast) is the lazy form of the same
|
|
530
|
+
conversion and takes the same route; a chain of lazy operations
|
|
531
|
+
gathers through the lazy arena instead, which this does not fit. */
|
|
521
532
|
void *attach = ca_func[src->obj_type].attach;
|
|
522
|
-
|
|
533
|
+
/* A conversion view is read-only on this path. Its put-back sends the
|
|
534
|
+
whole fiber, and a cell the kernel did not write does not come back
|
|
535
|
+
unchanged through a conversion that loses information (an int64
|
|
536
|
+
above 2^53 seen as float64), so a write takes the transfer walk,
|
|
537
|
+
which sends back only the changed cells (promise 3). */
|
|
538
|
+
if ( src_kind == CA_ITER_SRC_ATTACH && !(flags & CA_KERNEL_WRITE) ) {
|
|
523
539
|
if ( attach == ca_fake_func.attach
|
|
524
540
|
|| attach == ca_byte_swap_func.attach
|
|
525
541
|
|| ca_monop_view_is_single_cast(src) ) {
|
|
@@ -527,64 +543,26 @@ ca_iter_should_per_fiber_fused (CArray *src,
|
|
|
527
543
|
}
|
|
528
544
|
}
|
|
529
545
|
|
|
530
|
-
/*
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
(
|
|
536
|
-
STRIDE promotion), the existing alias path is already optimal
|
|
537
|
-
(parent.ptr + composed offsets, zero materialise). Per-fiber
|
|
538
|
-
dispatch in that case adds ca_xfer_stride per-call overhead with
|
|
539
|
-
no payoff -- F.6.3 bench measured 0.35-0.82x (regression). Gate
|
|
540
|
-
on SRC_DESCRIPTOR only. */
|
|
546
|
+
/* CAShift / CAWindow: the view's region transfer fills out-of-bounds
|
|
547
|
+
cells itself, so a fiber can be fetched in one call instead of a
|
|
548
|
+
per-slab or whole-view gather (SRC_DESCRIPTOR). A view that can
|
|
549
|
+
lend the parent's memory (SRC_DESCRIPTOR_L2_ALIASABLE, an interior
|
|
550
|
+
CAWindow) is already read in place; fetching it fiber by fiber was
|
|
551
|
+
measured slower (0.35-0.82x), so only SRC_DESCRIPTOR takes it. */
|
|
541
552
|
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
542
|
-
/*
|
|
543
|
-
typedef pattern: ca_shift_func = ca_window_func). */
|
|
553
|
+
/* CAWindow and CAShift share ca_window_func.attach. */
|
|
544
554
|
if ( attach == ca_window_func.attach ) {
|
|
545
555
|
return 1;
|
|
546
556
|
}
|
|
547
557
|
}
|
|
548
558
|
|
|
549
|
-
/*
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
batched throughput. Structurally different from
|
|
557
|
-
CAFake/CAByteSwap (transform-fused at byte level, X.4) and
|
|
558
|
-
CAShift/CAWindow OOB (per-region fused with bound check, X.1):
|
|
559
|
-
descriptor framework views do not have a per-region fused fast
|
|
560
|
-
path designed for per-fiber entry.
|
|
561
|
-
|
|
562
|
-
Note (rev4): bench split fiber-axis kind shows STRIDE fiber
|
|
563
|
-
1.25x slower vs INDEX fiber 4-5x slower. STRIDE-fiber case
|
|
564
|
-
is close to parity and could conceivably reach win with finer
|
|
565
|
-
gating (= bypass per-axis kind classification when the outer
|
|
566
|
-
prefix is homogeneous STRIDE). Not pursued at current 1.25x
|
|
567
|
-
deficit; recorded as a future possible refinement should an
|
|
568
|
-
application motivate it. See proposal §3.1 SRC_DESCRIPTOR
|
|
569
|
-
(CSA, CAGrid, ...) inline comment.
|
|
570
|
-
|
|
571
|
-
F.6.6 audit (devel/bench_f6_6_bit_audit.rb): CABitarray /
|
|
572
|
-
CABitfield kept on current path. Bench measured via
|
|
573
|
-
bits.fake(:float64) (= production CAFake-wrap hot path) shows
|
|
574
|
-
per-fiber 1.23-5.86x slower on the F.6.2-gate-off axes.
|
|
575
|
-
|
|
576
|
-
The CAFake wrap overhead applies equally to both per-fiber and
|
|
577
|
-
whole-view paths (same total cell-cast cost), so the 5x ratio
|
|
578
|
-
is essentially the CABitarray whole-view-vs-per-fiber ratio
|
|
579
|
-
itself. Phase A bench note "xfer_stride 0.98x parity" is a
|
|
580
|
-
single-call comparison: N per-fiber calls cumulatively cost
|
|
581
|
-
N x parity > 1 x xfer_all (+ N dispatch overhead + per-call
|
|
582
|
-
bit alignment math). This reasoning generalises to direct
|
|
583
|
-
CABitarray / CABitfield source (= when a non-f64 fiber smoke
|
|
584
|
-
is added in a future phase) -- no separate audit needed for
|
|
585
|
-
that case. Predicate stays unchanged.
|
|
586
|
-
|
|
587
|
-
F.6.7 will audit CAReduce. */
|
|
559
|
+
/* Not taken by the other views, each measured slower fiber by fiber:
|
|
560
|
+
CASelectAxis / CAGrid / CASelect (1.25-4.7x), whose descriptor
|
|
561
|
+
engine already gathers the whole view in one batched pass and has no
|
|
562
|
+
per-region fast path to enter; CABitarray / CABitfield (1.2-5.9x,
|
|
563
|
+
through a fake over them), where N per-fiber calls each pay the bit
|
|
564
|
+
alignment that one whole-view transfer pays once. CAReduce is not
|
|
565
|
+
routed here either. */
|
|
588
566
|
return 0;
|
|
589
567
|
}
|
|
590
568
|
|
|
@@ -600,8 +578,8 @@ ca_iter_validate_inputs (ca_iter_state *st,
|
|
|
600
578
|
uint32_t flags)
|
|
601
579
|
{
|
|
602
580
|
if ( st == NULL || src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
603
|
-
/*
|
|
604
|
-
|
|
581
|
+
/* CA_SLAB_AXES is accepted here and refused by init_l1 itself; init_l2
|
|
582
|
+
refuses CA_SLAB_WHOLE (see each init's policy gate). */
|
|
605
583
|
if ( policy != CA_SLAB_WHOLE && policy != CA_SLAB_AXES )
|
|
606
584
|
return CA_ITER_ERR_POLICY;
|
|
607
585
|
|
|
@@ -724,6 +702,45 @@ ca_iter_check_init (int rc)
|
|
|
724
702
|
return rc; /* not reached */
|
|
725
703
|
}
|
|
726
704
|
|
|
705
|
+
/* Attaches a CAStack's parents and keeps their buffers. stack_n_parents
|
|
706
|
+
counts the parents attached so far, so an attach that raises leaves
|
|
707
|
+
finish with exactly those to detach (promise 4). The mask of a stack
|
|
708
|
+
reads its parts' masks, some perhaps through stand-ins: those give way
|
|
709
|
+
first, as the stack's own slots would let them. */
|
|
710
|
+
static void
|
|
711
|
+
ki_attach_stack_parents (ca_iter_state *st, CAStack *stack)
|
|
712
|
+
{
|
|
713
|
+
if ( stack->mask_owner != NULL ) {
|
|
714
|
+
CAStack *o = (CAStack *) stack->mask_owner;
|
|
715
|
+
ca_multi_parent_mask_refresh((CArray *) stack, o->parents,
|
|
716
|
+
o->mask_standins, o->n_parents);
|
|
717
|
+
}
|
|
718
|
+
st->stack_n_parents = 0;
|
|
719
|
+
st->stack_parent_ptrs =
|
|
720
|
+
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
721
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
722
|
+
ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
|
|
723
|
+
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
724
|
+
st->stack_n_parents = kk + 1;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
/* Promise 2: the mask cursor a walk hands out is the walk's own copy of
|
|
729
|
+
the source's mask, taken when the walk opens -- one flag per cell, in
|
|
730
|
+
the view's row-major order -- never the live mask, so a write through
|
|
731
|
+
it reaches nothing. A source without a mask leaves alias_mask NULL.
|
|
732
|
+
finish frees the copy. */
|
|
733
|
+
static void
|
|
734
|
+
ki_take_mask_copy (ca_iter_state *st, CArray *src)
|
|
735
|
+
{
|
|
736
|
+
if ( ! ca_has_mask(src) ) return;
|
|
737
|
+
st->scratch_mask = (boolean8_t *) xmalloc(src->elements > 0 ? src->elements : 1);
|
|
738
|
+
if ( src->elements > 0 ) {
|
|
739
|
+
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
740
|
+
}
|
|
741
|
+
st->alias_mask = st->scratch_mask;
|
|
742
|
+
}
|
|
743
|
+
|
|
727
744
|
/* Whole-view scratch. The view is gathered into the walk's scratch,
|
|
728
745
|
by the view's own transfer (by_copy = 0) or by ca_copy_data (by_copy =
|
|
729
746
|
1, the CAStride compose-fold). A WRITE walk sends back only the cells
|
|
@@ -764,7 +781,7 @@ ki_scratch_push_changed (ca_iter_state *st)
|
|
|
764
781
|
ca_size_t view_bytes = (ca_size_t) src->elements * bytes;
|
|
765
782
|
char *now = st->scratch_ptr;
|
|
766
783
|
char *was = st->scratch_ptr + view_bytes;
|
|
767
|
-
int whole =
|
|
784
|
+
int whole = ( st->level == 1 ); /* L1 is the whole array */
|
|
768
785
|
ca_size_t n = whole ? src->elements : st->slab_elements;
|
|
769
786
|
ca_size_t idx[CA_RANK_MAX];
|
|
770
787
|
ca_size_t *addrs;
|
|
@@ -800,7 +817,16 @@ ki_scratch_push_changed (ca_iter_state *st)
|
|
|
800
817
|
}
|
|
801
818
|
}
|
|
802
819
|
|
|
803
|
-
|
|
820
|
+
/* Shortcut PUSH_ALL_CHANGED.
|
|
821
|
+
Layer write-back.
|
|
822
|
+
Speeds up a whole-view write walk that changed every cell: one
|
|
823
|
+
ca_xfer_all PUT instead of a list of all the addresses.
|
|
824
|
+
Why about 5% on a whole write of a fake view of 1M cells.
|
|
825
|
+
Enters whole && n_changed == n.
|
|
826
|
+
Promises 3 (the same cells go back; the list would hold them all).
|
|
827
|
+
Switch ca_fast_paths_enabled(): off, the address list.
|
|
828
|
+
Checked by the write rows of the delivery matrix (L1 / L2 whole). */
|
|
829
|
+
if ( whole && n_changed == n && ca_fast_paths_enabled() ) {
|
|
804
830
|
ca_xfer_all(src, now, CA_XFER_PUT);
|
|
805
831
|
}
|
|
806
832
|
else if ( n_changed > 0 ) {
|
|
@@ -810,6 +836,36 @@ ki_scratch_push_changed (ca_iter_state *st)
|
|
|
810
836
|
ALLOCV_END(h_addrs);
|
|
811
837
|
}
|
|
812
838
|
|
|
839
|
+
/* The transfer walk: the whole view gathered into scratch through the
|
|
840
|
+
view's own transfer, its mask likewise, and yielded as one contiguous
|
|
841
|
+
slab; on WRITE, sync_slab sends the changed cells back through the same
|
|
842
|
+
transfer (ki_scratch_push_changed).
|
|
843
|
+
|
|
844
|
+
Every promise the iterator makes is kept here by the view itself --
|
|
845
|
+
what a cell is, where its mask flag lives, which root cell a write
|
|
846
|
+
lands on -- so this walk is the reference the other routes are checked
|
|
847
|
+
against (CARRAY_GENERAL_PATH=2 sends every source through it). It is
|
|
848
|
+
also the route for a view the classifier knows nothing more about.
|
|
849
|
+
|
|
850
|
+
The caller has filled src / src_kind / level / policy / ndim / flags /
|
|
851
|
+
bytes / naxes. */
|
|
852
|
+
static void
|
|
853
|
+
ki_open_by_transfer (ca_iter_state *st, CArray *src, uint32_t flags)
|
|
854
|
+
{
|
|
855
|
+
ki_gather_view(st, src, flags, 0);
|
|
856
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
857
|
+
st->alias_ptr = st->scratch_ptr;
|
|
858
|
+
st->composed_strides[0] = src->bytes;
|
|
859
|
+
st->composed_base = 0;
|
|
860
|
+
st->slab_n = src->elements;
|
|
861
|
+
st->total_slabs = 1;
|
|
862
|
+
st->slabs_emitted = 0;
|
|
863
|
+
st->chunk_size = st->slab_n;
|
|
864
|
+
st->outer_idx = NULL;
|
|
865
|
+
|
|
866
|
+
ki_take_mask_copy(st, src);
|
|
867
|
+
}
|
|
868
|
+
|
|
813
869
|
static int
|
|
814
870
|
ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
815
871
|
struct _CArray *src,
|
|
@@ -826,9 +882,9 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
826
882
|
memset(st, 0, sizeof(*st));
|
|
827
883
|
if ( src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
828
884
|
|
|
829
|
-
/*
|
|
830
|
-
|
|
831
|
-
|
|
885
|
+
/* A Face or a lazy marker wraps storage identical to its parent, so it
|
|
886
|
+
is stripped at entry (the same reason as in init_l2 below), before
|
|
887
|
+
validate_inputs.
|
|
832
888
|
|
|
833
889
|
The WRITE rejection is taken against the wrapper, not what it wraps: a
|
|
834
890
|
CALazyMarker is read-only while its parent is not, and stripping first
|
|
@@ -844,12 +900,32 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
844
900
|
int rc = ca_iter_validate_inputs(st, src, policy, flags);
|
|
845
901
|
if ( rc != CA_ITER_OK ) return rc;
|
|
846
902
|
|
|
847
|
-
/*
|
|
848
|
-
|
|
849
|
-
misdispatch. */
|
|
903
|
+
/* CA_SLAB_AXES is L2 only (a kernel that wants a K-D slab uses
|
|
904
|
+
init_l2); init_l1 refuses it rather than walking the whole array. */
|
|
850
905
|
if ( policy == CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
|
|
851
906
|
|
|
852
|
-
|
|
907
|
+
/* The same routing as init_l2: a descriptor view whose parent lends no
|
|
908
|
+
memory, or a grid with holes, takes the transfer walk. An L1 walk
|
|
909
|
+
yields the whole view as one slab, so a descriptor view the L2 walk
|
|
910
|
+
could borrow is gathered here like any other descriptor view. */
|
|
911
|
+
ca_axis_desc_t route_descs[CA_RANK_MAX];
|
|
912
|
+
ca_size_t route_pdims[CA_RANK_MAX];
|
|
913
|
+
int8_t route_ndim = 0;
|
|
914
|
+
uint8_t src_kind = ca_iter_route_source(src, route_descs, route_pdims,
|
|
915
|
+
&route_ndim);
|
|
916
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
917
|
+
src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
918
|
+
}
|
|
919
|
+
/* A write over a descriptor view takes the transfer walk, as an L2
|
|
920
|
+
write over an INDEX or SHIFT slab axis does. Gathering with the
|
|
921
|
+
descriptor engine would leave nothing to send back but the whole
|
|
922
|
+
view, and a view that shows a root cell twice (a grid with a
|
|
923
|
+
repeated index) would then let the copy the kernel did not write
|
|
924
|
+
overwrite the one it did (promise 3: only the changed cells go
|
|
925
|
+
back). */
|
|
926
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR && (flags & CA_KERNEL_WRITE) ) {
|
|
927
|
+
src_kind = CA_ITER_SRC_ATTACH;
|
|
928
|
+
}
|
|
853
929
|
|
|
854
930
|
st->src = src;
|
|
855
931
|
st->src_kind = src_kind;
|
|
@@ -866,41 +942,18 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
866
942
|
st->chunk_size = st->slab_n;
|
|
867
943
|
|
|
868
944
|
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
869
|
-
|
|
870
|
-
CAByteSwap / CABitfield / CABitarray / CAReduce =================
|
|
871
|
-
View's xfer_all delivers materialised data through the unified
|
|
872
|
-
dispatch surface. Previously used `ca_attach(src) + src->ptr`
|
|
873
|
-
(= view's attach slot) and `ca_sync(src)` for WRITE; now uses
|
|
874
|
-
`ca_xfer_all(src, scratch, GET/PUT)` with iterator-owned scratch.
|
|
875
|
-
Decouples kernel_iterator from per-view attach/sync lifecycle
|
|
876
|
-
and inherits xfer reform improvements (transform-fused, etc.)
|
|
877
|
-
automatically. */
|
|
878
|
-
ki_gather_view(st, src, flags, 0);
|
|
879
|
-
st->alias_mode = CA_ITER_ALIAS_NONE; /* scratch-owned, no src.detach */
|
|
880
|
-
st->alias_ptr = st->scratch_ptr;
|
|
881
|
-
st->composed_strides[0] = src->bytes;
|
|
882
|
-
st->composed_base = 0;
|
|
883
|
-
st->outer_idx = NULL;
|
|
884
|
-
/* Step 6: mask gather (xfer_all on mask -- ca_copy_data is a thin
|
|
885
|
-
wrapper, this just removes the extra hop). */
|
|
886
|
-
if ( ca_has_mask(src) ) {
|
|
887
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
888
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
889
|
-
if ( src->elements > 0 ) {
|
|
890
|
-
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
891
|
-
}
|
|
892
|
-
st->alias_mask = st->scratch_mask;
|
|
893
|
-
}
|
|
945
|
+
ki_open_by_transfer(st, src, flags);
|
|
894
946
|
return CA_ITER_OK;
|
|
895
947
|
}
|
|
896
948
|
|
|
897
949
|
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
898
|
-
/* === descriptor path (
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
yield one contig slab
|
|
902
|
-
|
|
903
|
-
per-axis
|
|
950
|
+
/* === descriptor path (CASelectAxis / CAGrid / CASelect / CAWindow /
|
|
951
|
+
CAShift) ==========================================================
|
|
952
|
+
describe_axes → prepare → layout, then gather the view into
|
|
953
|
+
scratch with ca_axis_dispatch_gather and yield one contig slab
|
|
954
|
+
(alias_mode = NONE). The kernel sees a flat slab whatever the
|
|
955
|
+
per-axis kinds are; an INDEX / SHIFT mix is handled inside the
|
|
956
|
+
engine. */
|
|
904
957
|
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
905
958
|
int8_t raw_ndim = 0;
|
|
906
959
|
ca_iter_describe_axes(src, raw_descs, st->parent_axis_dims, &raw_ndim);
|
|
@@ -918,20 +971,17 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
918
971
|
}
|
|
919
972
|
st->total_elements = src->elements;
|
|
920
973
|
|
|
921
|
-
/* Materialise
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
974
|
+
/* Materialise: attach the parent here, allocate the scratch, and
|
|
975
|
+
gather into it with ca_axis_dispatch_gather (INDEX axes included);
|
|
976
|
+
the iterator owns the buffer. Descriptor views inherit from CAView
|
|
977
|
+
so parent is at the CAVIEW prefix slot. */
|
|
925
978
|
CArray *parent = CAVIEW(src)->parent;
|
|
926
979
|
ca_attach(parent); /* engine reads parent->ptr */
|
|
927
980
|
st->root = parent; /* finish() detaches */
|
|
928
981
|
st->scratch_cap = src->elements * src->bytes;
|
|
929
|
-
/*
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
bound_fill = CAWindow/CAShift fill value (Tier 2.B SHIFT-axis
|
|
933
|
-
OOB cell), NULL for non-window views (CSA / CAGrid / CASelect /
|
|
934
|
-
CAMapping don't have OOB semantics — engine sees no SHIFT axes). */
|
|
982
|
+
/* bound_fill = the CAWindow / CAShift fill value for an out-of-bounds
|
|
983
|
+
SHIFT cell; NULL for the other descriptor views, which have no
|
|
984
|
+
SHIFT axes. */
|
|
935
985
|
const void *bound_fill = NULL;
|
|
936
986
|
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
937
987
|
bound_fill = ((CAWindow *) src)->fill;
|
|
@@ -944,22 +994,11 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
944
994
|
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
945
995
|
st->alias_ptr = st->scratch_ptr;
|
|
946
996
|
|
|
947
|
-
|
|
948
|
-
all descriptor views — mask propagates through their
|
|
949
|
-
func_copy_data via the descriptor framework's own mask
|
|
950
|
-
handling). */
|
|
951
|
-
if ( ca_has_mask(src) ) {
|
|
952
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
953
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
954
|
-
if ( src->elements > 0 ) {
|
|
955
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
956
|
-
}
|
|
957
|
-
st->alias_mask = st->scratch_mask;
|
|
958
|
-
}
|
|
997
|
+
ki_take_mask_copy(st, src);
|
|
959
998
|
return CA_ITER_OK;
|
|
960
999
|
}
|
|
961
1000
|
|
|
962
|
-
/* === CAStride path
|
|
1001
|
+
/* === CAStride path ========================================= */
|
|
963
1002
|
if ( ca_iter_can_alias(src, 1) ) {
|
|
964
1003
|
/* alias path: entity or CAStride contig */
|
|
965
1004
|
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
@@ -975,1440 +1014,687 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
975
1014
|
st->alias_ptr = st->scratch_ptr;
|
|
976
1015
|
}
|
|
977
1016
|
|
|
978
|
-
|
|
979
|
-
We always gather into a contig boolean8_t buffer for uniformity
|
|
980
|
-
across alias / scratch / descriptor paths — the kernel sees a
|
|
981
|
-
simple `boolean8_t *` aligned with the value slab. Future
|
|
982
|
-
optimisation: alias mask directly for CAStride contig. Mask is
|
|
983
|
-
informational; kernel uses CA_FOR_EACH_UNMASKED macros to skip
|
|
984
|
-
masked cells. */
|
|
985
|
-
if ( ca_has_mask(src) ) {
|
|
986
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
987
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
988
|
-
if ( src->elements > 0 ) {
|
|
989
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
990
|
-
}
|
|
991
|
-
st->alias_mask = st->scratch_mask;
|
|
992
|
-
}
|
|
1017
|
+
ki_take_mask_copy(st, src);
|
|
993
1018
|
return CA_ITER_OK;
|
|
994
1019
|
}
|
|
995
1020
|
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1021
|
+
/* ---- L2 walks: choosing a route, then building the state for it --------
|
|
1022
|
+
|
|
1023
|
+
init_l2 checks the request, classifies the source and chooses one of the
|
|
1024
|
+
routes below (ki_choose_l2_route, which only decides). ki_open_l2 then
|
|
1025
|
+
builds the state for that route: the part only that route needs, the
|
|
1026
|
+
axis layout every route shares (ki_lay_out_slabs) and the mask. So the
|
|
1027
|
+
state is assembled in one place and init_l2 has one way out. */
|
|
1028
|
+
|
|
1029
|
+
typedef enum {
|
|
1030
|
+
KI_L2_FIBER_FUSED, /* PER_FIBER_FUSED: a CAWindow / CAShift fiber, or a
|
|
1031
|
+
read of a conversion view's innermost fiber */
|
|
1032
|
+
KI_L2_PER_SLAB, /* descriptor view, read, a slab axis INDEX or SHIFT:
|
|
1033
|
+
one slab gathered at a time (PER_SLAB / _HOIST) */
|
|
1034
|
+
KI_L2_DESC_WHOLE, /* descriptor view, read: the whole view gathered by
|
|
1035
|
+
the descriptor engine (outer SHIFT, SHIFT_WHOLE_VIEW) */
|
|
1036
|
+
KI_L2_DESC_STRIDED, /* descriptor view whose slab axes are all STRIDE:
|
|
1037
|
+
the parent's memory is borrowed */
|
|
1038
|
+
KI_L2_STACK, /* CAStack read along its stack axis */
|
|
1039
|
+
KI_L2_STACK_OUTER_K, /* CAStack read with the stack axis outside the slab */
|
|
1040
|
+
KI_L2_TRANSFER, /* the whole view gathered by its own transfer */
|
|
1041
|
+
KI_L2_ROOT_STRIDED, /* CAStride: the root walked through composed strides */
|
|
1042
|
+
KI_L2_VIEW_SCRATCH, /* CAStride over a root that lends no memory */
|
|
1043
|
+
KI_L2_CONTIG /* entity, or a contiguous CAStride */
|
|
1044
|
+
} ki_l2_route;
|
|
1020
1045
|
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1046
|
+
typedef struct {
|
|
1047
|
+
ki_l2_route route;
|
|
1048
|
+
uint8_t src_kind; /* what st->src_kind records */
|
|
1049
|
+
int8_t in_slab[CA_RANK_MAX]; /* 1 for each slab axis */
|
|
1050
|
+
ca_axis_desc_t descs[CA_RANK_MAX]; /* descriptor routes: the view's */
|
|
1051
|
+
ca_size_t pdims[CA_RANK_MAX]; /* axes and the parent's dims */
|
|
1052
|
+
int8_t ndesc;
|
|
1053
|
+
CArray *root; /* KI_L2_ROOT_STRIDED */
|
|
1054
|
+
ca_size_t composed[CA_RANK_MAX];
|
|
1055
|
+
ca_size_t base;
|
|
1056
|
+
} ki_l2_plan;
|
|
1057
|
+
|
|
1058
|
+
/* Per view axis: its extent, its byte stride when it is a slab axis, its
|
|
1059
|
+
byte stride when it is an outer axis, and its mask element stride. */
|
|
1060
|
+
typedef struct {
|
|
1061
|
+
ca_size_t dims[CA_RANK_MAX];
|
|
1062
|
+
ca_size_t slab_strides[CA_RANK_MAX];
|
|
1063
|
+
ca_size_t outer_strides[CA_RANK_MAX];
|
|
1064
|
+
ca_size_t mask_strides[CA_RANK_MAX];
|
|
1065
|
+
} ki_axis_layout;
|
|
1024
1066
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1067
|
+
/* Row-major byte and element strides over the view's own shape. */
|
|
1068
|
+
static void
|
|
1069
|
+
ki_view_row_major (CArray *src, ca_size_t *bytes, ca_size_t *elems)
|
|
1070
|
+
{
|
|
1071
|
+
ca_size_t b = src->bytes, e = 1;
|
|
1072
|
+
for ( int8_t k = src->ndim - 1; k >= 0; k-- ) {
|
|
1073
|
+
if ( bytes ) bytes[k] = b;
|
|
1074
|
+
if ( elems ) elems[k] = e;
|
|
1075
|
+
b *= src->dim[k];
|
|
1076
|
+
e *= src->dim[k];
|
|
1033
1077
|
}
|
|
1078
|
+
}
|
|
1034
1079
|
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
int8_t in_slab[CA_RANK_MAX];
|
|
1063
|
-
int8_t k;
|
|
1064
|
-
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
1065
|
-
for ( k = 0; k < naxes; k++ ) {
|
|
1066
|
-
int8_t ax = axes[k];
|
|
1067
|
-
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1068
|
-
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
1069
|
-
in_slab[ax] = 1;
|
|
1070
|
-
}
|
|
1071
|
-
|
|
1072
|
-
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.3 hook (= parallel to
|
|
1073
|
-
F.6.2 SRC_ATTACH hook): for descriptor views CAWindow /
|
|
1074
|
-
CAShift (predicate returns 1 unconditionally per F.5 always-
|
|
1075
|
-
win bench), skip materialise (alias / C.1 PER_SLAB / B.1.5)
|
|
1076
|
-
and use per-fiber ca_xfer_stride which routes to the view's
|
|
1077
|
-
X.1 OOB-fused per-region fast path. */
|
|
1078
|
-
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
1079
|
-
&& raw_ndim == src->ndim && ca_fast_paths_enabled() ) {
|
|
1080
|
-
int8_t fiber_ax = axes[0];
|
|
1081
|
-
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
1082
|
-
{
|
|
1083
|
-
ca_size_t b = src->bytes;
|
|
1084
|
-
for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
|
|
1085
|
-
row_byte_strides[kk] = b;
|
|
1086
|
-
b *= src->dim[kk];
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
|
|
1090
|
-
row_byte_strides[fiber_ax],
|
|
1091
|
-
flags) ) {
|
|
1092
|
-
st->src = src;
|
|
1093
|
-
st->src_kind = src_kind;
|
|
1094
|
-
st->level = 2;
|
|
1095
|
-
st->policy = policy;
|
|
1096
|
-
st->ndim = src->ndim;
|
|
1097
|
-
st->flags = flags;
|
|
1098
|
-
st->bytes = src->bytes;
|
|
1099
|
-
st->naxes = naxes;
|
|
1100
|
-
|
|
1101
|
-
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
1102
|
-
st->alias_ptr = NULL;
|
|
1103
|
-
st->fiber_axis = fiber_ax;
|
|
1104
|
-
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
1105
|
-
st->fiber_native_strides[kk] = row_byte_strides[kk];
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
/* Row-major element strides for mask (= identity element
|
|
1109
|
-
index strides over src->dim). */
|
|
1110
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1111
|
-
{
|
|
1112
|
-
ca_size_t e = 1;
|
|
1113
|
-
for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
|
|
1114
|
-
row_elem_strides[kk] = e;
|
|
1115
|
-
e *= src->dim[kk];
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
int8_t sp = 0, op = 0;
|
|
1120
|
-
st->slab_elements = 1;
|
|
1121
|
-
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
1122
|
-
if ( in_slab[kk] ) {
|
|
1123
|
-
st->slab_axes_buf[sp] = kk;
|
|
1124
|
-
st->slab_dims[sp] = src->dim[kk];
|
|
1125
|
-
st->slab_strides[sp] = row_byte_strides[kk];
|
|
1126
|
-
st->slab_mask_strides[sp] = row_elem_strides[kk];
|
|
1127
|
-
st->slab_elements *= src->dim[kk];
|
|
1128
|
-
sp++;
|
|
1129
|
-
} else {
|
|
1130
|
-
st->outer_axes[op] = kk;
|
|
1131
|
-
st->outer_dims[op] = src->dim[kk];
|
|
1132
|
-
st->outer_strides[op] = row_byte_strides[kk];
|
|
1133
|
-
st->outer_mask_strides[op] = row_elem_strides[kk];
|
|
1134
|
-
op++;
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
st->slab_ndim = sp;
|
|
1138
|
-
st->outer_ndim = op;
|
|
1139
|
-
st->desc_ndim = 0;
|
|
1140
|
-
|
|
1141
|
-
ca_size_t total = 1;
|
|
1142
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1143
|
-
st->total_slabs = total;
|
|
1144
|
-
st->slab_n = st->slab_elements;
|
|
1145
|
-
st->slabs_emitted = 0;
|
|
1146
|
-
st->chunk_size = st->slab_n;
|
|
1147
|
-
|
|
1148
|
-
if ( st->outer_ndim > 0 ) {
|
|
1149
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1150
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1151
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1152
|
-
} else {
|
|
1153
|
-
st->outer_idx = NULL;
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
/* Mask: per-fiber gather via next_slab_axes; no whole-view
|
|
1157
|
-
materialise (rev2 §3.3, (data,mask) pair travels together). */
|
|
1158
|
-
st->alias_mask = NULL;
|
|
1159
|
-
return CA_ITER_OK;
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
|
|
1163
|
-
/* Phase C T3 (C.1): slab axis has INDEX or SHIFT kind → per-slab
|
|
1164
|
-
materialise fallback path (D1.1 (B) + D1.2 (A)). CAMapping-style
|
|
1165
|
-
views (raw_ndim != src->ndim) are out of C.1 scope — descriptor
|
|
1166
|
-
exposes a flat 1-D INDEX gather that doesn't expose the view-
|
|
1167
|
-
axis partition the user gave via slab_axes. Reject explicitly. */
|
|
1168
|
-
if ( raw_ndim != src->ndim ) {
|
|
1169
|
-
return CA_ITER_ERR_POLICY;
|
|
1170
|
-
}
|
|
1171
|
-
int slab_has_non_stride = 0;
|
|
1172
|
-
int slab_has_index = 0;
|
|
1173
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1174
|
-
if ( in_slab[k] && raw_descs[k].kind != CA_AXIS_KIND_STRIDE ) {
|
|
1175
|
-
slab_has_non_stride = 1;
|
|
1176
|
-
if ( raw_descs[k].kind == CA_AXIS_KIND_INDEX ) {
|
|
1177
|
-
slab_has_index = 1;
|
|
1178
|
-
}
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
/* A slab with a SHIFT axis that leaves out the innermost view axis
|
|
1182
|
-
would be gathered one strided cell at a time, once per slab. Read
|
|
1183
|
-
walks take the whole-view path below instead: the view is gathered
|
|
1184
|
-
once, row by row, and the slabs are walked in the copy. */
|
|
1185
|
-
int whole_view = slab_has_non_stride && ! slab_has_index
|
|
1186
|
-
&& ! in_slab[src->ndim - 1]
|
|
1187
|
-
&& ! ( flags & CA_KERNEL_WRITE )
|
|
1188
|
-
&& ca_fast_paths_enabled();
|
|
1189
|
-
if ( slab_has_non_stride && ! whole_view ) {
|
|
1190
|
-
/* C.1 scope: READ-only. WRITE = future sub-step (C.1c). */
|
|
1191
|
-
if ( flags & CA_KERNEL_WRITE ) {
|
|
1192
|
-
return CA_ITER_ERR_FLAGS;
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
st->src = src;
|
|
1196
|
-
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1197
|
-
st->level = 2;
|
|
1198
|
-
st->policy = policy;
|
|
1199
|
-
st->ndim = src->ndim;
|
|
1200
|
-
st->flags = flags;
|
|
1201
|
-
st->bytes = src->bytes;
|
|
1202
|
-
st->naxes = naxes;
|
|
1203
|
-
|
|
1204
|
-
CArray *parent = CAVIEW(src)->parent;
|
|
1205
|
-
ca_attach(parent);
|
|
1206
|
-
st->root = parent;
|
|
1207
|
-
|
|
1208
|
-
/* Persist raw_descs / pdims / pstrides for per-slab subset
|
|
1209
|
-
construction (PER_SLAB fallback) or for the hoisted manual
|
|
1210
|
-
gather (PER_SLAB_HOIST specialised path). desc_ndim =
|
|
1211
|
-
src->ndim so has_descs branch in next_slab_axes can read
|
|
1212
|
-
st->descs (though T3 path takes its own branch before that
|
|
1213
|
-
check). */
|
|
1214
|
-
memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1215
|
-
memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
|
|
1216
|
-
{
|
|
1217
|
-
ca_size_t s = src->bytes;
|
|
1218
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1219
|
-
st->pstrides[k] = s;
|
|
1220
|
-
s *= raw_pdims[k];
|
|
1221
|
-
}
|
|
1222
|
-
}
|
|
1223
|
-
st->desc_ndim = src->ndim;
|
|
1224
|
-
|
|
1225
|
-
/* View-row-major element strides (= mask scratch layout, since
|
|
1226
|
-
mask is gathered whole-view once at init). */
|
|
1227
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1228
|
-
{
|
|
1229
|
-
ca_size_t e = 1;
|
|
1230
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1231
|
-
row_elem_strides[k] = e;
|
|
1232
|
-
e *= src->dim[k];
|
|
1233
|
-
}
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
/* Row-major byte strides over SLAB axes only (= scratch_ptr
|
|
1237
|
-
layout, since scratch_ptr is sized slab_elements × bytes
|
|
1238
|
-
and refilled per outer iter as a contig row-major slab). */
|
|
1239
|
-
ca_size_t slab_data_strides[CA_RANK_MAX];
|
|
1240
|
-
{
|
|
1241
|
-
ca_size_t b = src->bytes;
|
|
1242
|
-
ca_size_t e_slab = 1;
|
|
1243
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1244
|
-
if ( in_slab[k] ) {
|
|
1245
|
-
slab_data_strides[k] = b;
|
|
1246
|
-
b *= raw_descs[k].count;
|
|
1247
|
-
e_slab *= raw_descs[k].count;
|
|
1248
|
-
} else {
|
|
1249
|
-
slab_data_strides[k] = 0; /* unused for outer axes */
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
st->slab_elements = e_slab;
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
int8_t sp = 0, op = 0;
|
|
1256
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1257
|
-
if ( in_slab[k] ) {
|
|
1258
|
-
st->slab_axes_buf[sp] = k;
|
|
1259
|
-
st->slab_dims[sp] = raw_descs[k].count;
|
|
1260
|
-
st->slab_strides[sp] = slab_data_strides[k];
|
|
1261
|
-
/* Slab mask stride is view-row-major elem stride along
|
|
1262
|
-
this view axis (mask scratch is whole-view layout). */
|
|
1263
|
-
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1264
|
-
sp++;
|
|
1265
|
-
} else {
|
|
1266
|
-
st->outer_axes[op] = k;
|
|
1267
|
-
st->outer_dims[op] = raw_descs[k].count;
|
|
1268
|
-
/* T3 path: outer data offset is always 0 (scratch refilled
|
|
1269
|
-
per slab, ptr = scratch_ptr). */
|
|
1270
|
-
st->outer_strides[op] = 0;
|
|
1271
|
-
/* Outer mask offset uses view-row-major elem stride into
|
|
1272
|
-
whole-view mask scratch. */
|
|
1273
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1274
|
-
op++;
|
|
1275
|
-
}
|
|
1276
|
-
}
|
|
1277
|
-
st->slab_ndim = sp;
|
|
1278
|
-
st->outer_ndim = op;
|
|
1279
|
-
st->composed_base = 0; /* unused for T3 fallback */
|
|
1280
|
-
|
|
1281
|
-
ca_size_t total = 1;
|
|
1282
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1283
|
-
st->total_slabs = total;
|
|
1284
|
-
st->slab_n = st->slab_elements;
|
|
1285
|
-
st->slabs_emitted = 0;
|
|
1286
|
-
st->chunk_size = st->slab_n;
|
|
1287
|
-
|
|
1288
|
-
if ( st->outer_ndim > 0 ) {
|
|
1289
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1290
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1291
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1292
|
-
} else {
|
|
1293
|
-
st->outer_idx = NULL;
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
/* Alloc per-slab data scratch (D1.1 (B): one max-slab buffer
|
|
1297
|
-
refilled per outer iter, instead of per-iter alloc/free or
|
|
1298
|
-
full-view materialise). */
|
|
1299
|
-
ca_size_t slab_bytes = st->slab_elements * src->bytes;
|
|
1300
|
-
st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
|
|
1301
|
-
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1302
|
-
st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
|
|
1303
|
-
|
|
1304
|
-
/* Specialisation (B-1b, C.1b): if innermost slab axis is STRIDE
|
|
1305
|
-
and no SHIFT axes anywhere in the view, use the hoisted
|
|
1306
|
-
manual gather path (= outer + non-innermost-slab hoist,
|
|
1307
|
-
inner pure STRIDE linear memcpy). Else (innermost INDEX /
|
|
1308
|
-
SHIFT-anywhere): use the engine fallback (A). */
|
|
1309
|
-
int specialised_eligible = (sp > 0) && ca_fast_paths_enabled();
|
|
1310
|
-
if ( specialised_eligible ) {
|
|
1311
|
-
int8_t inner_view_ax = st->slab_axes_buf[sp - 1];
|
|
1312
|
-
if ( raw_descs[inner_view_ax].kind != CA_AXIS_KIND_STRIDE ) {
|
|
1313
|
-
specialised_eligible = 0;
|
|
1314
|
-
}
|
|
1315
|
-
}
|
|
1316
|
-
if ( specialised_eligible ) {
|
|
1317
|
-
/* Any SHIFT axis (slab or outer) → fallback to engine for
|
|
1318
|
-
clean bound_fill semantics in C.1b scope. */
|
|
1319
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1320
|
-
if ( raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1321
|
-
specialised_eligible = 0;
|
|
1322
|
-
break;
|
|
1323
|
-
}
|
|
1324
|
-
}
|
|
1325
|
-
}
|
|
1326
|
-
st->alias_mode = specialised_eligible
|
|
1327
|
-
? CA_ITER_ALIAS_PER_SLAB_HOIST
|
|
1328
|
-
: CA_ITER_ALIAS_PER_SLAB;
|
|
1329
|
-
|
|
1330
|
-
/* Mask: gather whole view mask once into scratch_mask (= same
|
|
1331
|
-
strategy as B.1.5). Per-slab mask offset is computed via
|
|
1332
|
-
outer_idx × outer_mask_strides in next_slab_axes. */
|
|
1333
|
-
if ( ca_has_mask(src) ) {
|
|
1334
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1335
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1336
|
-
if ( src->elements > 0 ) {
|
|
1337
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1338
|
-
}
|
|
1339
|
-
st->alias_mask = st->scratch_mask;
|
|
1340
|
-
}
|
|
1341
|
-
|
|
1342
|
-
return CA_ITER_OK;
|
|
1343
|
-
}
|
|
1344
|
-
/* slab is all-STRIDE: fall through to B.1.5 / Phase B paths. */
|
|
1345
|
-
/* Phase B.1.5: outer SHIFT axis → materialise downgrade. Alias
|
|
1346
|
-
can't deliver OOB cells (= need fill_value), and the view-family
|
|
1347
|
-
surface prioritises delivering the cells over avoiding a copy, so
|
|
1348
|
-
we materialise the entire view into a row-major scratch
|
|
1349
|
-
buffer via ca_axis_dispatch_attach (= same engine as the
|
|
1350
|
-
existing SRC_DESCRIPTOR L2 NONE path), then walk it with
|
|
1351
|
-
Phase A-style row-major K-D strides. Slab / outer partition
|
|
1352
|
-
applies to view-axes; the scratch IS the flat row-major view. */
|
|
1353
|
-
int outer_has_shift = 0;
|
|
1354
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1355
|
-
if ( !in_slab[k] && raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1356
|
-
outer_has_shift = 1;
|
|
1357
|
-
break;
|
|
1358
|
-
}
|
|
1359
|
-
}
|
|
1360
|
-
|
|
1361
|
-
if ( outer_has_shift || whole_view ) {
|
|
1362
|
-
st->src = src;
|
|
1363
|
-
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1364
|
-
st->level = 2;
|
|
1365
|
-
st->policy = policy;
|
|
1366
|
-
st->ndim = src->ndim;
|
|
1367
|
-
st->flags = flags;
|
|
1368
|
-
st->bytes = src->bytes;
|
|
1369
|
-
st->naxes = naxes;
|
|
1370
|
-
|
|
1371
|
-
CArray *parent = CAVIEW(src)->parent;
|
|
1372
|
-
ca_attach(parent);
|
|
1373
|
-
st->root = parent;
|
|
1374
|
-
|
|
1375
|
-
ca_axis_desc_t local_descs[CA_RANK_MAX];
|
|
1376
|
-
memcpy(local_descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1377
|
-
const void *bound_fill = NULL;
|
|
1378
|
-
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
1379
|
-
bound_fill = ((CAWindow *) src)->fill;
|
|
1380
|
-
}
|
|
1381
|
-
st->scratch_cap = src->elements * src->bytes;
|
|
1382
|
-
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1383
|
-
ca_axis_dispatch_gather(parent, raw_pdims, local_descs, src->ndim,
|
|
1384
|
-
src->bytes, src->elements, bound_fill,
|
|
1385
|
-
st->scratch_ptr);
|
|
1386
|
-
/* sync_slab scatters the whole scratch back by this count. */
|
|
1387
|
-
st->total_elements = src->elements;
|
|
1388
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1389
|
-
st->alias_ptr = st->scratch_ptr;
|
|
1390
|
-
|
|
1391
|
-
/* Row-major byte / element strides on the scratch (= view layout). */
|
|
1392
|
-
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
1393
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1394
|
-
{
|
|
1395
|
-
ca_size_t b = src->bytes, e = 1;
|
|
1396
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1397
|
-
row_byte_strides[k] = b;
|
|
1398
|
-
row_elem_strides[k] = e;
|
|
1399
|
-
b *= src->dim[k];
|
|
1400
|
-
e *= src->dim[k];
|
|
1401
|
-
}
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
int8_t sp = 0, op = 0;
|
|
1405
|
-
st->slab_elements = 1;
|
|
1406
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1407
|
-
if ( in_slab[k] ) {
|
|
1408
|
-
st->slab_axes_buf[sp] = k;
|
|
1409
|
-
st->slab_dims[sp] = src->dim[k];
|
|
1410
|
-
st->slab_strides[sp] = row_byte_strides[k];
|
|
1411
|
-
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1412
|
-
st->slab_elements *= src->dim[k];
|
|
1413
|
-
sp++;
|
|
1414
|
-
} else {
|
|
1415
|
-
st->outer_axes[op] = k;
|
|
1416
|
-
st->outer_dims[op] = src->dim[k];
|
|
1417
|
-
st->outer_strides[op] = row_byte_strides[k];
|
|
1418
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1419
|
-
op++;
|
|
1420
|
-
}
|
|
1421
|
-
}
|
|
1422
|
-
st->slab_ndim = sp;
|
|
1423
|
-
st->outer_ndim = op;
|
|
1424
|
-
st->composed_base = 0;
|
|
1425
|
-
/* desc_ndim = 0: next_slab_axes treats outer like Phase A
|
|
1426
|
-
(no INDEX lookup), correct because scratch is row-major
|
|
1427
|
-
contig (= STRIDE everywhere). */
|
|
1428
|
-
st->desc_ndim = 0;
|
|
1429
|
-
|
|
1430
|
-
ca_size_t total = 1;
|
|
1431
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1432
|
-
st->total_slabs = total;
|
|
1433
|
-
st->slab_n = st->slab_elements;
|
|
1434
|
-
st->slabs_emitted = 0;
|
|
1435
|
-
st->chunk_size = st->slab_n;
|
|
1436
|
-
|
|
1437
|
-
if ( st->outer_ndim > 0 ) {
|
|
1438
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1439
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1440
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1441
|
-
} else {
|
|
1442
|
-
st->outer_idx = NULL;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
if ( ca_has_mask(src) ) {
|
|
1446
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1447
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1448
|
-
if ( src->elements > 0 ) {
|
|
1449
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1450
|
-
}
|
|
1451
|
-
st->alias_mask = st->scratch_mask;
|
|
1452
|
-
}
|
|
1453
|
-
return CA_ITER_OK;
|
|
1454
|
-
}
|
|
1080
|
+
/* Splits the view's axes into slab and outer axes (both in ascending axis
|
|
1081
|
+
order, so the user's order does not matter), counts the slabs and opens
|
|
1082
|
+
the outer cursor. */
|
|
1083
|
+
static void
|
|
1084
|
+
ki_lay_out_slabs (ca_iter_state *st, CArray *src, const int8_t *in_slab,
|
|
1085
|
+
const ki_axis_layout *lay)
|
|
1086
|
+
{
|
|
1087
|
+
int8_t sp = 0, op = 0;
|
|
1088
|
+
st->slab_elements = 1;
|
|
1089
|
+
for ( int8_t k = 0; k < src->ndim; k++ ) {
|
|
1090
|
+
if ( in_slab[k] ) {
|
|
1091
|
+
st->slab_axes_buf[sp] = k;
|
|
1092
|
+
st->slab_dims[sp] = lay->dims[k];
|
|
1093
|
+
st->slab_strides[sp] = lay->slab_strides[k];
|
|
1094
|
+
st->slab_mask_strides[sp] = lay->mask_strides[k];
|
|
1095
|
+
st->slab_elements *= lay->dims[k];
|
|
1096
|
+
sp++;
|
|
1097
|
+
} else {
|
|
1098
|
+
st->outer_axes[op] = k;
|
|
1099
|
+
st->outer_dims[op] = lay->dims[k];
|
|
1100
|
+
st->outer_strides[op] = lay->outer_strides[k];
|
|
1101
|
+
st->outer_mask_strides[op] = lay->mask_strides[k];
|
|
1102
|
+
op++;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
st->slab_ndim = sp;
|
|
1106
|
+
st->outer_ndim = op;
|
|
1455
1107
|
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
st->bytes = src->bytes;
|
|
1463
|
-
st->naxes = naxes;
|
|
1464
|
-
|
|
1465
|
-
/* Parent row-major byte strides for offset computation. */
|
|
1466
|
-
ca_size_t pstrides[CA_RANK_MAX];
|
|
1467
|
-
{
|
|
1468
|
-
ca_size_t s = src->bytes;
|
|
1469
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1470
|
-
pstrides[k] = s;
|
|
1471
|
-
s *= raw_pdims[k];
|
|
1472
|
-
}
|
|
1473
|
-
}
|
|
1474
|
-
/* View row-major element strides (= mask scratch layout). */
|
|
1475
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1476
|
-
{
|
|
1477
|
-
ca_size_t s = 1;
|
|
1478
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1479
|
-
row_elem_strides[k] = s;
|
|
1480
|
-
s *= src->dim[k];
|
|
1481
|
-
}
|
|
1482
|
-
}
|
|
1108
|
+
ca_size_t total = 1;
|
|
1109
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1110
|
+
st->total_slabs = total;
|
|
1111
|
+
st->slab_n = st->slab_elements;
|
|
1112
|
+
st->slabs_emitted = 0;
|
|
1113
|
+
st->chunk_size = st->slab_n;
|
|
1483
1114
|
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
ca_size_t slab_base = 0;
|
|
1493
|
-
st->slab_elements = 1;
|
|
1494
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1495
|
-
if ( in_slab[k] ) {
|
|
1496
|
-
/* STRIDE-kind slab axis: start*pstride goes to slab_base,
|
|
1497
|
-
step*pstride is the per-cell byte stride. */
|
|
1498
|
-
slab_base += raw_descs[k].start * pstrides[k];
|
|
1499
|
-
st->slab_axes_buf[sp] = k;
|
|
1500
|
-
st->slab_dims[sp] = raw_descs[k].count;
|
|
1501
|
-
st->slab_strides[sp] = raw_descs[k].step * pstrides[k];
|
|
1502
|
-
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1503
|
-
st->slab_elements *= raw_descs[k].count;
|
|
1504
|
-
sp++;
|
|
1505
|
-
} else {
|
|
1506
|
-
/* Outer axis: STRIDE or INDEX (SHIFT was rejected above).
|
|
1507
|
-
Store axis index + view's outer dim/stride for descriptor
|
|
1508
|
-
walk in next_slab_axes. We reuse outer_strides/outer_mask
|
|
1509
|
-
_strides as the OUTER walk metadata; outer-axis kind is
|
|
1510
|
-
carried implicitly via raw_descs[outer_axes[m]] in the
|
|
1511
|
-
state struct's descs[] field (populated below). */
|
|
1512
|
-
st->outer_axes[op] = k;
|
|
1513
|
-
st->outer_dims[op] = raw_descs[k].count;
|
|
1514
|
-
/* For STRIDE: per-axis stride = step * pstride; INDEX axes
|
|
1515
|
-
use indices[] from raw_descs and need a 0 stride here
|
|
1516
|
-
(next_slab_axes branches on descs[].kind to compute the
|
|
1517
|
-
real offset). We persist the full raw_descs in st->descs
|
|
1518
|
-
so the next_slab_axes implementation can dispatch. */
|
|
1519
|
-
if ( raw_descs[k].kind == CA_AXIS_KIND_STRIDE ) {
|
|
1520
|
-
st->outer_strides[op] = raw_descs[k].step * pstrides[k];
|
|
1521
|
-
/* Bake in start*pstride into slab_base for STRIDE outer too
|
|
1522
|
-
so per-cell offset is purely step*idx. */
|
|
1523
|
-
slab_base += raw_descs[k].start * pstrides[k];
|
|
1524
|
-
} else {
|
|
1525
|
-
/* INDEX kind: outer_strides[m] = pstrides[k] (element-unit
|
|
1526
|
-
from indices[]). We DON'T pre-bake into slab_base for
|
|
1527
|
-
INDEX axes; per-iter offset = indices[idx[m]] * pstrides[k]
|
|
1528
|
-
is computed in next_slab_axes. */
|
|
1529
|
-
st->outer_strides[op] = pstrides[k];
|
|
1530
|
-
}
|
|
1531
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1532
|
-
op++;
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
st->slab_ndim = sp;
|
|
1536
|
-
st->outer_ndim = op;
|
|
1537
|
-
st->composed_base = slab_base;
|
|
1538
|
-
|
|
1539
|
-
/* Persist raw_descs / pstrides in state so next_slab_axes can
|
|
1540
|
-
distinguish STRIDE vs INDEX outer per axis. */
|
|
1541
|
-
memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1542
|
-
memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
|
|
1543
|
-
memcpy(st->pstrides, pstrides, src->ndim * sizeof(ca_size_t));
|
|
1544
|
-
st->desc_ndim = src->ndim;
|
|
1545
|
-
|
|
1546
|
-
ca_size_t total = 1;
|
|
1547
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1548
|
-
st->total_slabs = total;
|
|
1549
|
-
st->slab_n = st->slab_elements;
|
|
1550
|
-
st->slabs_emitted = 0;
|
|
1551
|
-
st->chunk_size = st->slab_n;
|
|
1552
|
-
|
|
1553
|
-
if ( st->outer_ndim > 0 ) {
|
|
1554
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1555
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1556
|
-
} else {
|
|
1557
|
-
st->outer_idx = NULL;
|
|
1558
|
-
}
|
|
1115
|
+
if ( st->outer_ndim > 0 ) {
|
|
1116
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1117
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1118
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1119
|
+
} else {
|
|
1120
|
+
st->outer_idx = NULL;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1559
1123
|
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1124
|
+
/* Chooses the route. Decides only: nothing is attached or allocated, so
|
|
1125
|
+
a refusal leaves nothing to close. */
|
|
1126
|
+
static int
|
|
1127
|
+
ki_choose_l2_route (CArray *src, int8_t *axes, int8_t naxes, uint32_t flags,
|
|
1128
|
+
ki_l2_plan *plan)
|
|
1129
|
+
{
|
|
1130
|
+
const int8_t *in_slab = plan->in_slab;
|
|
1131
|
+
ca_size_t row_bytes[CA_RANK_MAX];
|
|
1132
|
+
int8_t k;
|
|
1133
|
+
|
|
1134
|
+
ki_view_row_major(src, row_bytes, NULL);
|
|
1135
|
+
|
|
1136
|
+
if ( plan->src_kind == CA_ITER_SRC_DESCRIPTOR
|
|
1137
|
+
|| plan->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
1138
|
+
/* Shortcut PER_FIBER_FUSED (CAWindow / CAShift).
|
|
1139
|
+
Layer all three, per fiber: each fiber is fetched with the
|
|
1140
|
+
view's ca_xfer_stride (which fills out-of-bounds
|
|
1141
|
+
cells itself) and put back the same way.
|
|
1142
|
+
Speeds up a fiber walk of a CAWindow / CAShift whose innermost
|
|
1143
|
+
axis is SHIFT.
|
|
1144
|
+
Why none measured: about 8% slower than the descriptor
|
|
1145
|
+
route on a sort of a 2000x2000 shift view.
|
|
1146
|
+
Enters CA_KERNEL_FIBER_CONTIG, one slab axis, the view's
|
|
1147
|
+
own rank, and ca_iter_should_per_fiber_fused. An
|
|
1148
|
+
outer-only SHIFT arrives as L2_ALIASABLE and never
|
|
1149
|
+
reaches this hook.
|
|
1150
|
+
Promises 1 (the fiber scratch is contiguous), 2 (the mask is
|
|
1151
|
+
fetched into the fiber mask scratch), 3 (ca_xfer_stride
|
|
1152
|
+
PUT of a fiber that neither converts nor repeats a
|
|
1153
|
+
cell), 4 (CA_ITER_FLAG_WALK_MAY_RAISE).
|
|
1154
|
+
Switch ca_fast_paths_enabled(): off, the descriptor route.
|
|
1155
|
+
Checked by the fiber rows of the "shift" view in the delivery
|
|
1156
|
+
matrix. */
|
|
1157
|
+
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
1158
|
+
&& plan->ndesc == src->ndim && ca_fast_paths_enabled()
|
|
1159
|
+
&& ca_iter_should_per_fiber_fused(src, plan->src_kind, axes[0],
|
|
1160
|
+
row_bytes[axes[0]], flags) ) {
|
|
1161
|
+
plan->route = KI_L2_FIBER_FUSED;
|
|
1576
1162
|
return CA_ITER_OK;
|
|
1577
1163
|
}
|
|
1578
|
-
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
1579
|
-
/* Phase B.5: CA_SLAB_AXES + SRC_ATTACH (= CAFake / CAByteSwap /
|
|
1580
|
-
CABitfield / CABitarray / CAReduce). These overlay views
|
|
1581
|
-
materialise via their own ca_attach (= no descriptor framework
|
|
1582
|
-
path), so we let ca_attach(src) populate src->ptr in view
|
|
1583
|
-
row-major layout and walk it with Phase A-style K-D strides.
|
|
1584
|
-
|
|
1585
|
-
The most common entry path here is wrap_readonly(int_src,
|
|
1586
|
-
CA_FLOAT64) → CAFake, enabling sum_ki etc. to accept any
|
|
1587
|
-
numeric data_type source via auto-cast (= D2.2 confirmed in
|
|
1588
|
-
rev2 sparring). */
|
|
1589
|
-
if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
|
|
1590
|
-
return CA_ITER_ERR_POLICY;
|
|
1591
|
-
}
|
|
1592
|
-
int8_t in_slab[CA_RANK_MAX];
|
|
1593
|
-
int8_t k;
|
|
1594
|
-
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
1595
|
-
for ( k = 0; k < naxes; k++ ) {
|
|
1596
|
-
int8_t ax = axes[k];
|
|
1597
|
-
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1598
|
-
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
1599
|
-
in_slab[ax] = 1;
|
|
1600
|
-
}
|
|
1601
1164
|
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1616
|
-
{
|
|
1617
|
-
ca_size_t b = src->bytes, e = 1;
|
|
1618
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1619
|
-
row_byte_strides[k] = b;
|
|
1620
|
-
row_elem_strides[k] = e;
|
|
1621
|
-
b *= src->dim[k];
|
|
1622
|
-
e *= src->dim[k];
|
|
1623
|
-
}
|
|
1165
|
+
/* The rank check guards a descriptor whose rank differs from the
|
|
1166
|
+
view's (it would not expose the axis partition slab_axes names).
|
|
1167
|
+
Every describe_axes in the engine reports the view's rank, so the
|
|
1168
|
+
check does not fire today; it stays as the contract's guard. */
|
|
1169
|
+
if ( plan->ndesc != src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1170
|
+
|
|
1171
|
+
int slab_has_non_stride = 0;
|
|
1172
|
+
int slab_has_index = 0;
|
|
1173
|
+
int outer_has_shift = 0;
|
|
1174
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1175
|
+
if ( in_slab[k] && plan->descs[k].kind != CA_AXIS_KIND_STRIDE ) {
|
|
1176
|
+
slab_has_non_stride = 1;
|
|
1177
|
+
if ( plan->descs[k].kind == CA_AXIS_KIND_INDEX ) slab_has_index = 1;
|
|
1624
1178
|
}
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
parent ptr access path + tile cache). Engages when:
|
|
1628
|
-
- source.attach is ca_stack_func.attach (= CAStack identity)
|
|
1629
|
-
- no mask (= horizontal mask propagation per-slab gather is
|
|
1630
|
-
out of scope; falls back to SRC_ATTACH whole-view path)
|
|
1631
|
-
- naxes == 1 && axes[0] == k_axis (= K-axis-only slab, the
|
|
1632
|
-
demand-driving case = reduce along the stacked axis like
|
|
1633
|
-
view.mean(axis: k_axis))
|
|
1634
|
-
init attaches K parents up front (O(1) per entity parent),
|
|
1635
|
-
caches parent->ptr aliases + uniform parent-native byte strides,
|
|
1636
|
-
allocates a slab-sized scratch (= K * bytes) plus a tile cache
|
|
1637
|
-
sized to fit the L1d budget. next_slab_axes refills TILE fibers
|
|
1638
|
-
per K contig parent reads (rev4 2026-06-19) and serves the next
|
|
1639
|
-
TILE-1 calls from the L1d-resident cache, beating SRC_ATTACH
|
|
1640
|
-
across all measured sizes. No size-threshold gate: rev3's gate
|
|
1641
|
-
was a perf trade-off justification that the tile cache erased.
|
|
1642
|
-
|
|
1643
|
-
K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): engage predicate
|
|
1644
|
-
generalised from `axes[0] == 0` to `axes[0] == k_axis`. The
|
|
1645
|
-
tile cache mechanism + outer_idx -> parent axis mapping are
|
|
1646
|
-
already k_axis-agnostic: outer_idx[m] maps to parent axis m
|
|
1647
|
-
regardless of k_axis position (= for k_axis = 0 outer is stack
|
|
1648
|
-
axes 1..N-1 = parent axes 0..N-2; for k_axis > 0 outer is stack
|
|
1649
|
-
axes [0..k_axis-1, k_axis+1..N-1] = parent axes [0..k_axis-1,
|
|
1650
|
-
k_axis..N-2]; either way outer_idx[m] = parent axis m). */
|
|
1651
|
-
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1652
|
-
&& !ca_has_mask(src)
|
|
1653
|
-
&& !(flags & CA_KERNEL_WRITE)
|
|
1654
|
-
&& naxes == 1 && axes[0] == ((CAStack *) src)->k_axis
|
|
1655
|
-
&& ca_fast_paths_enabled() ) {
|
|
1656
|
-
CAStack *stack = (CAStack *) src;
|
|
1657
|
-
int8_t parent_ndim = src->ndim - 1;
|
|
1658
|
-
int8_t sp = 0, op = 0;
|
|
1659
|
-
|
|
1660
|
-
st->slab_elements = 1;
|
|
1661
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1662
|
-
if ( in_slab[k] ) {
|
|
1663
|
-
st->slab_axes_buf[sp] = k;
|
|
1664
|
-
st->slab_dims[sp] = src->dim[k];
|
|
1665
|
-
st->slab_elements *= src->dim[k];
|
|
1666
|
-
sp++;
|
|
1667
|
-
} else {
|
|
1668
|
-
st->outer_axes[op] = k;
|
|
1669
|
-
st->outer_dims[op] = src->dim[k];
|
|
1670
|
-
st->outer_strides[op] = row_byte_strides[k]; /* unused on STACK path */
|
|
1671
|
-
st->outer_mask_strides[op] = row_elem_strides[k]; /* unused on STACK path */
|
|
1672
|
-
op++;
|
|
1673
|
-
}
|
|
1674
|
-
}
|
|
1675
|
-
st->slab_ndim = sp;
|
|
1676
|
-
st->outer_ndim = op;
|
|
1677
|
-
st->composed_base = 0;
|
|
1678
|
-
st->desc_ndim = 0;
|
|
1679
|
-
/* slab_strides on STACK path: slab_axes == [0], scratch is
|
|
1680
|
-
packed contig over K elements (= K * bytes), so single-axis
|
|
1681
|
-
slab walk uses bytes stride. */
|
|
1682
|
-
st->slab_strides[0] = src->bytes;
|
|
1683
|
-
st->slab_mask_strides[0] = 1; /* harmless on no-mask path */
|
|
1684
|
-
|
|
1685
|
-
ca_size_t total = 1;
|
|
1686
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1687
|
-
st->total_slabs = total;
|
|
1688
|
-
st->slab_n = st->slab_elements;
|
|
1689
|
-
st->slabs_emitted = 0;
|
|
1690
|
-
st->chunk_size = st->slab_n;
|
|
1691
|
-
|
|
1692
|
-
if ( st->outer_ndim > 0 ) {
|
|
1693
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1694
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1695
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1696
|
-
} else {
|
|
1697
|
-
st->outer_idx = NULL;
|
|
1698
|
-
}
|
|
1699
|
-
|
|
1700
|
-
/* Attach K parents and cache their ptr aliases. CAStack
|
|
1701
|
-
guarantees uniform shape (MEMO §3.2) so one set of parent
|
|
1702
|
-
native byte strides covers all K (= parents[0]->dim is
|
|
1703
|
-
canonical). */
|
|
1704
|
-
/* stack_n_parents counts the parents attached so far: an attach
|
|
1705
|
-
that raises leaves finish with exactly those to detach. */
|
|
1706
|
-
st->stack_n_parents = 0;
|
|
1707
|
-
st->stack_parent_ptrs =
|
|
1708
|
-
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
1709
|
-
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1710
|
-
ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
|
|
1711
|
-
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
1712
|
-
st->stack_n_parents = kk + 1;
|
|
1713
|
-
}
|
|
1714
|
-
{
|
|
1715
|
-
ca_size_t s = src->bytes;
|
|
1716
|
-
for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
|
|
1717
|
-
st->stack_parent_strides[kk] = s;
|
|
1718
|
-
s *= stack->parents[0]->dim[kk];
|
|
1719
|
-
}
|
|
1720
|
-
}
|
|
1721
|
-
|
|
1722
|
-
/* Slab-sized scratch (= K * bytes for slab_axes == [0]). */
|
|
1723
|
-
st->scratch_cap = (ca_size_t) st->slab_elements * src->bytes;
|
|
1724
|
-
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1725
|
-
|
|
1726
|
-
/* Tile cache (pilot/castack-axis0-loop-interchange): refill TILE
|
|
1727
|
-
fibers per K parent reads, serve next TILE-1 fibers from L1d.
|
|
1728
|
-
TILE budget = ~32 KB (half of M2 L1d). Empirically tested
|
|
1729
|
-
values; clamped to [8, 64] to keep the refill loop tight. */
|
|
1730
|
-
{
|
|
1731
|
-
ca_size_t K = (ca_size_t) stack->n_parents;
|
|
1732
|
-
ca_size_t bytes = (ca_size_t) src->bytes;
|
|
1733
|
-
ca_size_t tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
|
|
1734
|
-
if ( tile < 8 ) tile = 8;
|
|
1735
|
-
if ( tile > 64 ) tile = 64;
|
|
1736
|
-
st->stack_tile_cap = tile;
|
|
1737
|
-
st->stack_tile_pos = 0;
|
|
1738
|
-
st->stack_tile_have = 0; /* force refill on first call */
|
|
1739
|
-
st->stack_tile_cache = (char *) ki_data_alloc(src->data_type, K * tile * bytes);
|
|
1740
|
-
}
|
|
1741
|
-
|
|
1742
|
-
st->alias_mode = CA_ITER_ALIAS_STACK;
|
|
1743
|
-
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1744
|
-
st->alias_mask = NULL;
|
|
1745
|
-
return CA_ITER_OK;
|
|
1746
|
-
}
|
|
1747
|
-
|
|
1748
|
-
/* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18):
|
|
1749
|
-
CAStack source + slab_axes excludes axis 0 (= K-axis stays in
|
|
1750
|
-
outer iter, e.g. view.mean(axis: 1), view.mean(axis: 2),
|
|
1751
|
-
view.mean(axis: 1, 2)). Each slab corresponds to a region
|
|
1752
|
-
inside ONE parent selected by outer_idx[K_outer_pos]. Engine
|
|
1753
|
-
pre-attaches K parents (+ K parent masks if mask present),
|
|
1754
|
-
caches their ptrs, and next_slab_axes aliases parents[k]->ptr +
|
|
1755
|
-
parent_off directly -- no scratch, no materialise, parent
|
|
1756
|
-
entity case = eager-equivalent memory bandwidth.
|
|
1757
|
-
|
|
1758
|
-
Layering: engine detects CAStack identity by function-pointer
|
|
1759
|
-
comparison and accesses parents[] / mask via ca_func[STACK]
|
|
1760
|
-
operation table where possible. CAStack downcast occurs only
|
|
1761
|
-
for the n_parents / parents[] read in init_l2 (= same scope as
|
|
1762
|
-
rev3 STACK; AC3 forbids parents[] walk in next_slab_axes only).
|
|
1763
|
-
Q1 disposition: option (i) minimal-diff inline in SLAB_AXES
|
|
1764
|
-
branch -- evaluated as smallest diff with acceptable layering
|
|
1765
|
-
(proposal §3.1). */
|
|
1766
|
-
/* STACK_OUTER_K: K-axis stays in outer iter (slab carved out of
|
|
1767
|
-
one parent at a time). K.3 (2026-06-20): engage predicate +
|
|
1768
|
-
stack-axis -> parent-axis mapping generalised for arbitrary
|
|
1769
|
-
k_axis. stack axis s maps to parent axis (s if s < k_axis else
|
|
1770
|
-
s - 1); only the K stack axis (s == k_axis) has no parent
|
|
1771
|
-
counterpart. */
|
|
1772
|
-
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1773
|
-
&& !(flags & CA_KERNEL_WRITE)
|
|
1774
|
-
&& naxes >= 1 && !in_slab[((CAStack *) src)->k_axis]
|
|
1775
|
-
&& ca_fast_paths_enabled() ) {
|
|
1776
|
-
CAStack *stack = (CAStack *) src;
|
|
1777
|
-
int8_t k_axis = stack->k_axis;
|
|
1778
|
-
int8_t parent_ndim = src->ndim - 1;
|
|
1779
|
-
int8_t sp = 0, op = 0;
|
|
1780
|
-
|
|
1781
|
-
st->slab_elements = 1;
|
|
1782
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1783
|
-
if ( in_slab[k] ) {
|
|
1784
|
-
st->slab_axes_buf[sp] = k;
|
|
1785
|
-
st->slab_dims[sp] = src->dim[k];
|
|
1786
|
-
st->slab_elements *= src->dim[k];
|
|
1787
|
-
/* slab_strides[sp] = parent native byte stride at parent
|
|
1788
|
-
axis (k if k < k_axis else k - 1). k == k_axis cannot
|
|
1789
|
-
appear here -- the engage predicate above excludes it. */
|
|
1790
|
-
sp++;
|
|
1791
|
-
} else {
|
|
1792
|
-
st->outer_axes[op] = k;
|
|
1793
|
-
st->outer_dims[op] = src->dim[k];
|
|
1794
|
-
st->outer_strides[op] = row_byte_strides[k];
|
|
1795
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1796
|
-
op++;
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
st->slab_ndim = sp;
|
|
1800
|
-
st->outer_ndim = op;
|
|
1801
|
-
st->composed_base = 0;
|
|
1802
|
-
st->desc_ndim = 0;
|
|
1803
|
-
|
|
1804
|
-
/* Locate the K-axis (= stack axis k_axis) within outer_axes. */
|
|
1805
|
-
st->stack_k_outer_pos = -1;
|
|
1806
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
1807
|
-
if ( st->outer_axes[m] == k_axis ) {
|
|
1808
|
-
st->stack_k_outer_pos = m;
|
|
1809
|
-
break;
|
|
1810
|
-
}
|
|
1811
|
-
}
|
|
1812
|
-
|
|
1813
|
-
/* Uniform parent native byte strides + parent element strides
|
|
1814
|
-
for mask (= MEMO §3.2 uniform shape across parents,
|
|
1815
|
-
parents[0]->dim is canonical). Mask strides cached so
|
|
1816
|
-
next_slab_axes can compute mask_off without downcasting
|
|
1817
|
-
(= AC3 layering goal). */
|
|
1818
|
-
{
|
|
1819
|
-
ca_size_t sb = src->bytes;
|
|
1820
|
-
ca_size_t se = 1;
|
|
1821
|
-
for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
|
|
1822
|
-
st->stack_parent_strides[kk] = sb;
|
|
1823
|
-
st->stack_parent_mask_strides[kk] = se;
|
|
1824
|
-
sb *= stack->parents[0]->dim[kk];
|
|
1825
|
-
se *= stack->parents[0]->dim[kk];
|
|
1826
|
-
}
|
|
1827
|
-
}
|
|
1828
|
-
|
|
1829
|
-
/* Fill slab_strides + slab_mask_strides for slab axes (all of
|
|
1830
|
-
which are parent inner axes since K is in outer). stack ax
|
|
1831
|
-
-> parent ax: s if s < k_axis else s - 1. */
|
|
1832
|
-
for ( int8_t s_i = 0; s_i < st->slab_ndim; s_i++ ) {
|
|
1833
|
-
int8_t stack_ax = st->slab_axes_buf[s_i];
|
|
1834
|
-
int8_t parent_ax = (stack_ax < k_axis) ? stack_ax : (stack_ax - 1);
|
|
1835
|
-
st->slab_strides[s_i] = st->stack_parent_strides[parent_ax];
|
|
1836
|
-
st->slab_mask_strides[s_i] = st->stack_parent_mask_strides[parent_ax];
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
ca_size_t total = 1;
|
|
1840
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1841
|
-
st->total_slabs = total;
|
|
1842
|
-
st->slab_n = st->slab_elements;
|
|
1843
|
-
st->slabs_emitted = 0;
|
|
1844
|
-
st->chunk_size = st->slab_n;
|
|
1845
|
-
|
|
1846
|
-
if ( st->outer_ndim > 0 ) {
|
|
1847
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1848
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1849
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1850
|
-
} else {
|
|
1851
|
-
st->outer_idx = NULL;
|
|
1852
|
-
}
|
|
1853
|
-
|
|
1854
|
-
/* Attach K parents + cache ptr aliases. Symmetric with rev3
|
|
1855
|
-
Case B; for entity parents attach is O(1). */
|
|
1856
|
-
/* stack_n_parents counts the parents attached so far: an attach
|
|
1857
|
-
that raises leaves finish with exactly those to detach. */
|
|
1858
|
-
st->stack_n_parents = 0;
|
|
1859
|
-
st->stack_parent_ptrs =
|
|
1860
|
-
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
1861
|
-
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1862
|
-
ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
|
|
1863
|
-
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
1864
|
-
st->stack_n_parents = kk + 1;
|
|
1865
|
-
}
|
|
1866
|
-
|
|
1867
|
-
/* If CAStack carries mask (= horizontal propagation already
|
|
1868
|
-
applied at create_mask), copy the K parent masks as they are
|
|
1869
|
-
at the start of the walk; next_slab_axes points into the copy
|
|
1870
|
-
of parent k at mask_off. A copy, as on every other path: the
|
|
1871
|
-
cursor is the walk's own, and a write to it reaches nothing. */
|
|
1872
|
-
if ( ca_has_mask(src) ) {
|
|
1873
|
-
/* The stack's mask may still read stand-ins for parents that
|
|
1874
|
-
have gained a mask since; read through to those. */
|
|
1875
|
-
ca_multi_parent_mask_refresh(src->mask, stack->parents,
|
|
1876
|
-
stack->mask_standins,
|
|
1877
|
-
stack->n_parents, 0);
|
|
1878
|
-
/* Zero-filled: a slot is set once its copy is made, and finish
|
|
1879
|
-
frees the copies whose slot is set. */
|
|
1880
|
-
st->stack_parent_mask_ptrs =
|
|
1881
|
-
(boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
|
|
1882
|
-
memset(st->stack_parent_mask_ptrs, 0,
|
|
1883
|
-
stack->n_parents * sizeof(boolean8_t *));
|
|
1884
|
-
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1885
|
-
CArray *pm = ((CAStack *) src->mask)->parents[kk];
|
|
1886
|
-
ca_size_t n = pm->elements;
|
|
1887
|
-
boolean8_t *copy = (boolean8_t *) xmalloc(n > 0 ? n : 1);
|
|
1888
|
-
st->stack_parent_mask_ptrs[kk] = copy;
|
|
1889
|
-
ca_attach(pm); /* window: only a memcpy before the detach */
|
|
1890
|
-
memcpy(copy, pm->ptr, n);
|
|
1891
|
-
ca_detach(pm);
|
|
1892
|
-
}
|
|
1893
|
-
} else {
|
|
1894
|
-
st->stack_parent_mask_ptrs = NULL;
|
|
1895
|
-
}
|
|
1896
|
-
|
|
1897
|
-
st->scratch_ptr = NULL; /* no scratch: aliasing parent->ptr */
|
|
1898
|
-
st->scratch_cap = 0;
|
|
1899
|
-
st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
|
|
1900
|
-
st->alias_ptr = NULL;
|
|
1901
|
-
st->alias_mask = NULL;
|
|
1902
|
-
return CA_ITER_OK;
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.1 hook: try per-fiber fused
|
|
1906
|
-
dispatch before whole-view materialise. When predicate fires,
|
|
1907
|
-
skip scratch alloc + xfer_all GET (= per-fiber ca_xfer_stride
|
|
1908
|
-
is called on demand in next_slab_axes). Predicate is stubbed
|
|
1909
|
-
in F.6.1 (returns 0); F.6.2+ enables specific source kinds. */
|
|
1910
|
-
int per_fiber_fused = 0;
|
|
1911
|
-
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
1912
|
-
&& ca_fast_paths_enabled() ) {
|
|
1913
|
-
int8_t fiber_ax = axes[0];
|
|
1914
|
-
ca_size_t fiber_ax_stride = row_byte_strides[fiber_ax];
|
|
1915
|
-
if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
|
|
1916
|
-
fiber_ax_stride, flags) ) {
|
|
1917
|
-
per_fiber_fused = 1;
|
|
1918
|
-
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
1919
|
-
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1920
|
-
st->fiber_axis = fiber_ax;
|
|
1921
|
-
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
1922
|
-
st->fiber_native_strides[kk] = row_byte_strides[kk];
|
|
1923
|
-
}
|
|
1924
|
-
}
|
|
1925
|
-
}
|
|
1926
|
-
|
|
1927
|
-
if ( !per_fiber_fused ) {
|
|
1928
|
-
/* 2026-05-31 refactor: iter-owns scratch via ca_xfer_all (was
|
|
1929
|
-
ca_attach(src) + alias src->ptr). */
|
|
1930
|
-
ki_gather_view(st, src, flags, 0);
|
|
1931
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1932
|
-
st->alias_ptr = st->scratch_ptr;
|
|
1179
|
+
if ( !in_slab[k] && plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1180
|
+
outer_has_shift = 1;
|
|
1933
1181
|
}
|
|
1182
|
+
}
|
|
1934
1183
|
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1967
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1968
|
-
} else {
|
|
1969
|
-
st->outer_idx = NULL;
|
|
1184
|
+
/* A slab with a SHIFT axis that leaves out the innermost view axis
|
|
1185
|
+
would be gathered one strided cell at a time, once per slab. Read
|
|
1186
|
+
walks take the whole-view route instead: the view is gathered
|
|
1187
|
+
once, row by row, and the slabs are walked in the copy.
|
|
1188
|
+
|
|
1189
|
+
Shortcut SHIFT_WHOLE_VIEW.
|
|
1190
|
+
Layer data; the fiber layer (ki_fiber_contig_*) sits on top.
|
|
1191
|
+
Speeds up read walks of a CAWindow / CAShift whose slab has a
|
|
1192
|
+
SHIFT axis and not the innermost axis.
|
|
1193
|
+
Why about 1.5x on a median along axis 0 of a 2000x2000
|
|
1194
|
+
shift view.
|
|
1195
|
+
Enters a non-stride slab axis, no INDEX slab axis, the
|
|
1196
|
+
innermost axis outside the slab, a read walk.
|
|
1197
|
+
Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
|
|
1198
|
+
only when the view has one).
|
|
1199
|
+
Switch ca_fast_paths_enabled(): off, the per-slab gather.
|
|
1200
|
+
Checked by the slab-axis rows of the "shift" and "window" views
|
|
1201
|
+
in the delivery matrix. */
|
|
1202
|
+
int whole_view = slab_has_non_stride && ! slab_has_index
|
|
1203
|
+
&& ! in_slab[src->ndim - 1]
|
|
1204
|
+
&& ! ( flags & CA_KERNEL_WRITE )
|
|
1205
|
+
&& ca_fast_paths_enabled();
|
|
1206
|
+
|
|
1207
|
+
/* The descriptor engine's gathers (per slab and whole view) read
|
|
1208
|
+
only. A write takes the transfer walk instead: the view is
|
|
1209
|
+
gathered whole and the changed cells go back through its own
|
|
1210
|
+
transfer (promise 3). */
|
|
1211
|
+
if ( (slab_has_non_stride && ! whole_view) || outer_has_shift ) {
|
|
1212
|
+
if ( flags & CA_KERNEL_WRITE ) {
|
|
1213
|
+
plan->src_kind = CA_ITER_SRC_ATTACH;
|
|
1214
|
+
goto by_transfer;
|
|
1970
1215
|
}
|
|
1216
|
+
}
|
|
1217
|
+
plan->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1218
|
+
if ( slab_has_non_stride && ! whole_view ) {
|
|
1219
|
+
plan->route = KI_L2_PER_SLAB;
|
|
1220
|
+
}
|
|
1221
|
+
else if ( outer_has_shift || whole_view ) {
|
|
1222
|
+
plan->route = KI_L2_DESC_WHOLE;
|
|
1223
|
+
}
|
|
1224
|
+
else {
|
|
1225
|
+
plan->route = KI_L2_DESC_STRIDED;
|
|
1226
|
+
}
|
|
1227
|
+
return CA_ITER_OK;
|
|
1228
|
+
}
|
|
1971
1229
|
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1230
|
+
if ( plan->src_kind == CA_ITER_SRC_ATTACH ) {
|
|
1231
|
+
by_transfer:
|
|
1232
|
+
/* Shortcut STACK.
|
|
1233
|
+
Layer data and fiber: the slab is cut contiguous from the K
|
|
1234
|
+
parents through a tile cache.
|
|
1235
|
+
Speeds up read walks of an unmasked CAStack whose one slab axis
|
|
1236
|
+
is the stack axis.
|
|
1237
|
+
Why about 2x on a mean along the stack axis of 4 parents
|
|
1238
|
+
of 500x2000.
|
|
1239
|
+
Enters a CAStack, no mask, a read walk, one slab axis that is
|
|
1240
|
+
k_axis.
|
|
1241
|
+
Promises 1 (the slab is contiguous), 6 (no mask exists to touch).
|
|
1242
|
+
Switch ca_fast_paths_enabled(): off, the transfer walk.
|
|
1243
|
+
Checked by the slab-axis and fiber rows of the stack parent along
|
|
1244
|
+
axis 0 in the delivery matrix. */
|
|
1245
|
+
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1246
|
+
&& !ca_has_mask(src)
|
|
1247
|
+
&& !(flags & CA_KERNEL_WRITE)
|
|
1248
|
+
&& naxes == 1 && axes[0] == ((CAStack *) src)->k_axis
|
|
1249
|
+
&& ca_fast_paths_enabled() ) {
|
|
1250
|
+
plan->route = KI_L2_STACK;
|
|
1251
|
+
return CA_ITER_OK;
|
|
1252
|
+
}
|
|
1253
|
+
/* Shortcut STACK_OUTER_K.
|
|
1254
|
+
Layer data; the fiber layer sits on top.
|
|
1255
|
+
Speeds up read walks of a CAStack whose stack axis is an outer
|
|
1256
|
+
axis: each slab is cut from the one parent the outer
|
|
1257
|
+
index picks, without gathering the stack.
|
|
1258
|
+
Why about 11x on a mean along axis 1 of 4 parents of
|
|
1259
|
+
500x2000.
|
|
1260
|
+
Enters a CAStack, a read walk, the stack axis outside the slab.
|
|
1261
|
+
Promises 2 (the parents' masks are copied), 6 (masks are read
|
|
1262
|
+
only when a parent has one), 8 (alias_ptr stays NULL,
|
|
1263
|
+
so no slab is lent as a window).
|
|
1264
|
+
Switch ca_fast_paths_enabled(): off, the transfer walk.
|
|
1265
|
+
Checked by the slab-axis, fiber and two-slab-axes rows of the
|
|
1266
|
+
stack parent along axes 1 and 2 in the delivery matrix. */
|
|
1267
|
+
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1268
|
+
&& !(flags & CA_KERNEL_WRITE)
|
|
1269
|
+
&& naxes >= 1 && !in_slab[((CAStack *) src)->k_axis]
|
|
1270
|
+
&& ca_fast_paths_enabled() ) {
|
|
1271
|
+
plan->route = KI_L2_STACK_OUTER_K;
|
|
1987
1272
|
return CA_ITER_OK;
|
|
1988
1273
|
}
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1274
|
+
/* Shortcut PER_FIBER_FUSED (conversion views).
|
|
1275
|
+
Layer all three, per fiber: each fiber is fetched with the
|
|
1276
|
+
view's ca_xfer_stride into the fiber scratch, without
|
|
1277
|
+
gathering the whole view.
|
|
1278
|
+
Speeds up a read of a fiber of a fake or byte-swap view, or of a
|
|
1279
|
+
lazy monop that only casts, along its innermost axis.
|
|
1280
|
+
Why no difference measured on a sort along the last axis
|
|
1281
|
+
of a 2000x2000 fake view.
|
|
1282
|
+
Enters CA_KERNEL_FIBER_CONTIG, one slab axis, a read walk, and
|
|
1283
|
+
ca_iter_should_per_fiber_fused.
|
|
1284
|
+
Promises 1, 2 and 4, as the CAWindow / CAShift hook above. A
|
|
1285
|
+
write takes the transfer walk (promise 3).
|
|
1286
|
+
Switch ca_fast_paths_enabled(): off, the transfer walk.
|
|
1287
|
+
Checked by the fiber rows of the fake parent in the delivery
|
|
1288
|
+
matrix. */
|
|
1289
|
+
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
1290
|
+
&& ca_fast_paths_enabled()
|
|
1291
|
+
&& ca_iter_should_per_fiber_fused(src, CA_ITER_SRC_ATTACH, axes[0],
|
|
1292
|
+
row_bytes[axes[0]], flags) ) {
|
|
1293
|
+
plan->route = KI_L2_FIBER_FUSED;
|
|
1294
|
+
return CA_ITER_OK;
|
|
1992
1295
|
}
|
|
1296
|
+
plan->route = KI_L2_TRANSFER;
|
|
1297
|
+
return CA_ITER_OK;
|
|
1993
1298
|
}
|
|
1994
1299
|
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
&& ca_axis_dispatch_outer_has_shift(raw_descs, raw_ndim) ) {
|
|
2001
|
-
src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1300
|
+
if ( plan->src_kind != CA_ITER_SRC_CASTRIDE ) return CA_ITER_ERR_POLICY;
|
|
1301
|
+
|
|
1302
|
+
if ( ca_iter_can_alias(src, 1) ) {
|
|
1303
|
+
plan->route = KI_L2_CONTIG;
|
|
1304
|
+
return CA_ITER_OK;
|
|
2002
1305
|
}
|
|
1306
|
+
/* A non-contiguous CAStride: compose the strides up to the root. A
|
|
1307
|
+
root that holds nothing to read through would cost one whole-root
|
|
1308
|
+
materialise no matter how few cells the view touches -- the cliff
|
|
1309
|
+
the xfer path avoids by asking for a region -- so that view is
|
|
1310
|
+
gathered by its own region protocol instead: ca_copy_data walks the
|
|
1311
|
+
leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
|
|
1312
|
+
CAObject asks for exactly that block. */
|
|
1313
|
+
ca_stride_compose_to_root((CAStride *) src, &plan->root, plan->composed,
|
|
1314
|
+
&plan->base);
|
|
1315
|
+
plan->route = ca_root_lends_no_memory(plan->root) ? KI_L2_VIEW_SCRATCH
|
|
1316
|
+
: KI_L2_ROOT_STRIDED;
|
|
1317
|
+
return CA_ITER_OK;
|
|
1318
|
+
}
|
|
2003
1319
|
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
st->level = 2;
|
|
2012
|
-
st->policy = policy;
|
|
2013
|
-
st->ndim = 1; /* logical 1-D L2 layout */
|
|
2014
|
-
st->flags = flags;
|
|
2015
|
-
st->bytes = src->bytes;
|
|
2016
|
-
st->naxes = naxes;
|
|
1320
|
+
/* Builds the state for the chosen route. */
|
|
1321
|
+
static void
|
|
1322
|
+
ki_open_l2 (ca_iter_state *st, CArray *src, ca_slab_policy_t policy,
|
|
1323
|
+
int8_t *axes, int8_t naxes, uint32_t flags, ki_l2_plan *plan)
|
|
1324
|
+
{
|
|
1325
|
+
ki_axis_layout lay;
|
|
1326
|
+
int8_t k;
|
|
2017
1327
|
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
st->chunk_size = st->slab_n;
|
|
2027
|
-
st->outer_idx = NULL;
|
|
1328
|
+
st->src = src;
|
|
1329
|
+
st->src_kind = plan->src_kind;
|
|
1330
|
+
st->level = 2;
|
|
1331
|
+
st->policy = policy;
|
|
1332
|
+
st->ndim = src->ndim;
|
|
1333
|
+
st->flags = flags;
|
|
1334
|
+
st->bytes = src->bytes;
|
|
1335
|
+
st->naxes = naxes;
|
|
2028
1336
|
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
1337
|
+
/* The default layout: the view's own row-major strides, which is how
|
|
1338
|
+
a scratch the walk gathered is laid out and how the mask copy is
|
|
1339
|
+
laid out on every route that takes one. */
|
|
1340
|
+
for ( k = 0; k < src->ndim; k++ ) lay.dims[k] = src->dim[k];
|
|
1341
|
+
ki_view_row_major(src, lay.slab_strides, lay.mask_strides);
|
|
1342
|
+
memcpy(lay.outer_strides, lay.slab_strides, src->ndim * sizeof(ca_size_t));
|
|
1343
|
+
|
|
1344
|
+
switch ( plan->route ) {
|
|
1345
|
+
|
|
1346
|
+
case KI_L2_FIBER_FUSED:
|
|
1347
|
+
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
1348
|
+
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1349
|
+
st->fiber_axis = axes[0];
|
|
1350
|
+
memcpy(st->fiber_native_strides, lay.slab_strides,
|
|
1351
|
+
src->ndim * sizeof(ca_size_t));
|
|
1352
|
+
break;
|
|
2039
1353
|
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
2052
|
-
int8_t nd = raw_ndim;
|
|
2053
|
-
int8_t k;
|
|
2054
|
-
|
|
2055
|
-
st->src = src;
|
|
2056
|
-
st->src_kind = CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE;
|
|
2057
|
-
st->level = 2;
|
|
2058
|
-
st->policy = policy;
|
|
2059
|
-
st->ndim = nd;
|
|
2060
|
-
st->flags = flags;
|
|
2061
|
-
st->bytes = src->bytes;
|
|
2062
|
-
st->naxes = naxes;
|
|
2063
|
-
|
|
2064
|
-
/* Persist descs / parent_axis_dims in state; build parent row-major
|
|
2065
|
-
pstrides so prefix_offset / inner offset share the same byte space. */
|
|
2066
|
-
memcpy(st->descs, raw_descs, nd * sizeof(ca_axis_desc_t));
|
|
2067
|
-
memcpy(st->parent_axis_dims, raw_pdims, nd * sizeof(ca_size_t));
|
|
2068
|
-
st->desc_ndim = nd;
|
|
1354
|
+
case KI_L2_PER_SLAB: {
|
|
1355
|
+
/* One slab is gathered into a slab-sized scratch per outer index, so
|
|
1356
|
+
a slab axis steps through that scratch (packed row-major over the
|
|
1357
|
+
slab axes) and an outer axis moves nothing in it. The descriptors
|
|
1358
|
+
are kept for next_slab_axes to build each slab's subset. */
|
|
1359
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1360
|
+
ca_size_t b = src->bytes;
|
|
1361
|
+
ca_attach(parent); /* window: finish detaches st->root */
|
|
1362
|
+
st->root = parent;
|
|
1363
|
+
memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1364
|
+
memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
|
|
2069
1365
|
{
|
|
2070
1366
|
ca_size_t s = src->bytes;
|
|
2071
|
-
for ( k =
|
|
1367
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
2072
1368
|
st->pstrides[k] = s;
|
|
2073
|
-
s *=
|
|
1369
|
+
s *= plan->pdims[k];
|
|
2074
1370
|
}
|
|
2075
1371
|
}
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
if ( nd > 1 ) {
|
|
2084
|
-
ca_axis_dispatch_classify_prefix(st->descs, st->pstrides, nd - 1, st->prefix);
|
|
2085
|
-
}
|
|
2086
|
-
|
|
2087
|
-
/* Total slabs = Π descs[0..nd-2].count (= src->elements / inner_n). */
|
|
2088
|
-
ca_size_t outer_total = 1;
|
|
2089
|
-
for ( k = 0; k < nd - 1; k++ ) outer_total *= raw_descs[k].count;
|
|
2090
|
-
st->total_slabs = outer_total;
|
|
2091
|
-
st->slab_n = inner_n;
|
|
2092
|
-
st->slabs_emitted = 0;
|
|
2093
|
-
st->chunk_size = inner_n;
|
|
2094
|
-
st->total_elements = src->elements;
|
|
2095
|
-
|
|
2096
|
-
/* composed_base holds inner_byte_start; composed_strides[nd-1] holds
|
|
2097
|
-
inner stride so next_slab_strided's existing inner_st extraction
|
|
2098
|
-
(= composed_strides[nd-1]) works without per-iter recomputation.
|
|
2099
|
-
Outer slots of composed_strides are unused by the descriptor
|
|
2100
|
-
branch in next_slab_strided (= prefix[] drives the offset). */
|
|
2101
|
-
st->composed_base = inner_start;
|
|
2102
|
-
st->composed_strides[nd - 1] = inner_stride;
|
|
2103
|
-
|
|
2104
|
-
if ( nd > 1 ) {
|
|
2105
|
-
CA_ASSUME(nd <= CA_RANK_MAX); /* with nd > 1: nd-1 in [1, CA_RANK_MAX-1] */
|
|
2106
|
-
st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
|
|
2107
|
-
for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
|
|
2108
|
-
} else {
|
|
2109
|
-
st->outer_idx = NULL;
|
|
2110
|
-
}
|
|
2111
|
-
|
|
2112
|
-
CArray *parent = CAVIEW(src)->parent;
|
|
2113
|
-
ca_attach(parent);
|
|
2114
|
-
st->root = parent;
|
|
2115
|
-
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
2116
|
-
st->alias_ptr = (char *) parent->ptr;
|
|
2117
|
-
|
|
2118
|
-
/* Mask: gather into scratch_mask (same as SRC_DESCRIPTOR path).
|
|
2119
|
-
Data alias + mask materialise is per rev6 §4.4 deferred to F-4.c. */
|
|
2120
|
-
if ( ca_has_mask(src) ) {
|
|
2121
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
2122
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
2123
|
-
if ( src->elements > 0 ) {
|
|
2124
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1372
|
+
st->desc_ndim = src->ndim;
|
|
1373
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1374
|
+
lay.dims[k] = plan->descs[k].count;
|
|
1375
|
+
lay.outer_strides[k] = 0;
|
|
1376
|
+
if ( plan->in_slab[k] ) {
|
|
1377
|
+
lay.slab_strides[k] = b;
|
|
1378
|
+
b *= plan->descs[k].count;
|
|
2125
1379
|
}
|
|
2126
|
-
st->alias_mask = st->scratch_mask;
|
|
2127
1380
|
}
|
|
2128
|
-
|
|
1381
|
+
break;
|
|
2129
1382
|
}
|
|
2130
1383
|
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
/* raw_descs / raw_pdims / raw_ndim were populated by route_source
|
|
2142
|
-
above so we skip the local describe_axes call. Kept locals
|
|
2143
|
-
named raw_* to match the original code. */
|
|
2144
|
-
st->src = src;
|
|
2145
|
-
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
2146
|
-
st->level = 2;
|
|
2147
|
-
st->policy = policy;
|
|
2148
|
-
st->ndim = src->ndim;
|
|
2149
|
-
st->flags = flags;
|
|
2150
|
-
st->bytes = src->bytes;
|
|
2151
|
-
st->naxes = naxes;
|
|
2152
|
-
|
|
2153
|
-
memcpy(st->parent_axis_dims, raw_pdims, raw_ndim * sizeof(ca_size_t));
|
|
2154
|
-
ca_axis_dispatch_prepare(st->parent_axis_dims, raw_descs, raw_ndim,
|
|
2155
|
-
st->bytes, st->descs, st->pstrides,
|
|
2156
|
-
st->mdim, &st->desc_ndim);
|
|
2157
|
-
ca_axis_dispatch_layout(st->descs, st->pstrides, st->mdim,
|
|
2158
|
-
st->desc_ndim, st->bytes,
|
|
2159
|
-
&st->slab_start, &st->slab_bytes_desc,
|
|
2160
|
-
&st->slab_base);
|
|
2161
|
-
if ( st->slab_start > 0 ) {
|
|
2162
|
-
ca_axis_dispatch_classify_prefix(st->descs, st->pstrides,
|
|
2163
|
-
st->slab_start, st->prefix);
|
|
2164
|
-
}
|
|
2165
|
-
st->total_elements = src->elements;
|
|
2166
|
-
|
|
2167
|
-
/* Materialise via the engine into a scratch buffer (contig
|
|
2168
|
-
layout = single strided run, stride = bytes). */
|
|
2169
|
-
CArray *parent = CAVIEW(src)->parent;
|
|
2170
|
-
ca_attach(parent);
|
|
2171
|
-
st->root = parent;
|
|
2172
|
-
st->scratch_cap = src->elements * src->bytes;
|
|
2173
|
-
const void *bound_fill = NULL;
|
|
1384
|
+
case KI_L2_DESC_WHOLE: {
|
|
1385
|
+
/* The view gathered whole into a row-major scratch by the
|
|
1386
|
+
descriptor engine (which fills out-of-bounds cells), then walked
|
|
1387
|
+
with the default layout. */
|
|
1388
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1389
|
+
ca_axis_desc_t local_descs[CA_RANK_MAX];
|
|
1390
|
+
const void *bound_fill = NULL;
|
|
1391
|
+
ca_attach(parent); /* window: finish detaches st->root */
|
|
1392
|
+
st->root = parent;
|
|
1393
|
+
memcpy(local_descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
2174
1394
|
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
2175
1395
|
bound_fill = ((CAWindow *) src)->fill;
|
|
2176
1396
|
}
|
|
2177
|
-
st->scratch_cap =
|
|
1397
|
+
st->scratch_cap = src->elements * src->bytes;
|
|
2178
1398
|
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
2179
|
-
ca_axis_dispatch_gather(parent,
|
|
2180
|
-
src->bytes,
|
|
1399
|
+
ca_axis_dispatch_gather(parent, plan->pdims, local_descs, src->ndim,
|
|
1400
|
+
src->bytes, src->elements, bound_fill,
|
|
2181
1401
|
st->scratch_ptr);
|
|
1402
|
+
st->total_elements = src->elements;
|
|
1403
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1404
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1405
|
+
break;
|
|
1406
|
+
}
|
|
2182
1407
|
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
st->total_slabs = 1;
|
|
2198
|
-
st->slabs_emitted = 0;
|
|
2199
|
-
st->chunk_size = st->slab_n;
|
|
2200
|
-
st->outer_idx = NULL;
|
|
2201
|
-
|
|
2202
|
-
/* Step 6: mask gather, same as L1 descriptor branch. */
|
|
2203
|
-
if ( ca_has_mask(src) ) {
|
|
2204
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
2205
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
2206
|
-
if ( src->elements > 0 ) {
|
|
2207
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1408
|
+
case KI_L2_DESC_STRIDED: {
|
|
1409
|
+
/* The parent's memory is walked in place. Each slab axis is a
|
|
1410
|
+
STRIDE axis; an outer axis is STRIDE or INDEX (an outer SHIFT went
|
|
1411
|
+
to KI_L2_DESC_WHOLE). The start of every STRIDE axis is folded
|
|
1412
|
+
into composed_base, so a slab or STRIDE outer offset is step *
|
|
1413
|
+
index; an INDEX outer axis is looked up in its indices[] by
|
|
1414
|
+
next_slab_axes, which needs the descriptors kept. */
|
|
1415
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1416
|
+
ca_size_t base = 0;
|
|
1417
|
+
{
|
|
1418
|
+
ca_size_t s = src->bytes;
|
|
1419
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1420
|
+
st->pstrides[k] = s;
|
|
1421
|
+
s *= plan->pdims[k];
|
|
2208
1422
|
}
|
|
2209
|
-
st->alias_mask = st->scratch_mask;
|
|
2210
1423
|
}
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
synthesise row-major byte strides so next_slab_strided's offset
|
|
2226
|
-
math is uniform across alias modes; for CAStride non-contig we
|
|
2227
|
-
compose leaf strides up to the root entity via the substrate. */
|
|
2228
|
-
int8_t nd = src->ndim;
|
|
2229
|
-
int use_strided = 0;
|
|
2230
|
-
int use_view_scratch = 0;
|
|
2231
|
-
CArray *root = NULL;
|
|
2232
|
-
|
|
2233
|
-
if ( ca_iter_can_alias(src, 1) ) {
|
|
2234
|
-
/* Entity or CAStride contig: stride = row-major bytes, base = 0. */
|
|
2235
|
-
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
2236
|
-
src->dim, nd, src->bytes);
|
|
2237
|
-
st->composed_base = 0;
|
|
2238
|
-
} else {
|
|
2239
|
-
/* CAStride non-contig: leaf->root compose. */
|
|
2240
|
-
ca_size_t cs[CA_RANK_MAX];
|
|
2241
|
-
ca_size_t base;
|
|
2242
|
-
ca_stride_compose_to_root((CAStride *) src, &root, cs, &base);
|
|
2243
|
-
if ( !ca_root_lends_no_memory(root) ) {
|
|
2244
|
-
memcpy(st->composed_strides, cs, nd * sizeof(ca_size_t));
|
|
2245
|
-
st->composed_base = base;
|
|
2246
|
-
use_strided = 1;
|
|
2247
|
-
} else {
|
|
2248
|
-
/* Root holds nothing to read through: reaching root->ptr costs one
|
|
2249
|
-
whole-root materialise no matter how few cells the view touches —
|
|
2250
|
-
the same cliff the xfer path avoids by asking for a region. Take
|
|
2251
|
-
the view's own region protocol instead: ca_copy_data walks the
|
|
2252
|
-
leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
|
|
2253
|
-
CAObject asks for exactly that block. The gathered buffer is view
|
|
2254
|
-
row-major, so the slab arithmetic below is the contig-alias case
|
|
2255
|
-
unchanged. */
|
|
2256
|
-
root = NULL;
|
|
2257
|
-
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
2258
|
-
src->dim, nd, src->bytes);
|
|
2259
|
-
st->composed_base = 0;
|
|
2260
|
-
use_view_scratch = 1;
|
|
1424
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1425
|
+
const ca_axis_desc_t *d = &plan->descs[k];
|
|
1426
|
+
lay.dims[k] = d->count;
|
|
1427
|
+
if ( plan->in_slab[k] ) {
|
|
1428
|
+
base += d->start * st->pstrides[k];
|
|
1429
|
+
lay.slab_strides[k] = d->step * st->pstrides[k];
|
|
1430
|
+
}
|
|
1431
|
+
else if ( d->kind == CA_AXIS_KIND_STRIDE ) {
|
|
1432
|
+
base += d->start * st->pstrides[k];
|
|
1433
|
+
lay.outer_strides[k] = d->step * st->pstrides[k];
|
|
1434
|
+
}
|
|
1435
|
+
else {
|
|
1436
|
+
lay.outer_strides[k] = st->pstrides[k];
|
|
1437
|
+
}
|
|
2261
1438
|
}
|
|
1439
|
+
st->composed_base = base;
|
|
1440
|
+
memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1441
|
+
memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
|
|
1442
|
+
st->desc_ndim = src->ndim;
|
|
1443
|
+
ca_attach(parent); /* window: finish detaches st->root */
|
|
1444
|
+
st->root = parent;
|
|
1445
|
+
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
1446
|
+
st->alias_ptr = (char *) parent->ptr;
|
|
1447
|
+
break;
|
|
2262
1448
|
}
|
|
2263
1449
|
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
int8_t in_slab[CA_RANK_MAX];
|
|
2279
|
-
int8_t k;
|
|
2280
|
-
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
2281
|
-
for ( k = 0; k < naxes; k++ ) {
|
|
2282
|
-
int8_t ax = axes[k];
|
|
2283
|
-
if ( ax < 0 || ax >= nd ) return CA_ITER_ERR_POLICY;
|
|
2284
|
-
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY; /* duplicate */
|
|
2285
|
-
in_slab[ax] = 1;
|
|
2286
|
-
}
|
|
2287
|
-
|
|
2288
|
-
/* View row-major element strides (= mask scratch layout strides). */
|
|
2289
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1450
|
+
case KI_L2_STACK: {
|
|
1451
|
+
/* The K parents are attached and read directly; the slab (the K cells
|
|
1452
|
+
along the stack axis) is cut into a slab-sized scratch, refilled
|
|
1453
|
+
from a tile cache of TILE fibers that the parents fill in one pass.
|
|
1454
|
+
CAStack guarantees one shape across its parents, so parents[0]
|
|
1455
|
+
gives the native strides of all of them. */
|
|
1456
|
+
CAStack *stack = (CAStack *) src;
|
|
1457
|
+
int8_t parent_ndim = src->ndim - 1;
|
|
1458
|
+
ca_size_t K = (ca_size_t) stack->n_parents;
|
|
1459
|
+
ca_size_t bytes = (ca_size_t) src->bytes;
|
|
1460
|
+
ca_size_t tile;
|
|
1461
|
+
lay.slab_strides[axes[0]] = src->bytes;
|
|
1462
|
+
lay.mask_strides[axes[0]] = 1;
|
|
1463
|
+
ki_attach_stack_parents(st, stack);
|
|
2290
1464
|
{
|
|
2291
|
-
ca_size_t s =
|
|
2292
|
-
for ( k =
|
|
2293
|
-
|
|
2294
|
-
s *=
|
|
1465
|
+
ca_size_t s = src->bytes;
|
|
1466
|
+
for ( k = parent_ndim - 1; k >= 0; k-- ) {
|
|
1467
|
+
st->stack_parent_strides[k] = s;
|
|
1468
|
+
s *= stack->parents[0]->dim[k];
|
|
2295
1469
|
}
|
|
2296
1470
|
}
|
|
1471
|
+
st->scratch_cap = (ca_size_t) src->dim[axes[0]] * src->bytes;
|
|
1472
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1473
|
+
/* TILE budget = ~32 KB (half of M2 L1d), clamped to [8, 64] to keep
|
|
1474
|
+
the refill loop tight. */
|
|
1475
|
+
tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
|
|
1476
|
+
if ( tile < 8 ) tile = 8;
|
|
1477
|
+
if ( tile > 64 ) tile = 64;
|
|
1478
|
+
st->stack_tile_cap = tile;
|
|
1479
|
+
st->stack_tile_pos = 0;
|
|
1480
|
+
st->stack_tile_have = 0; /* force refill on first call */
|
|
1481
|
+
st->stack_tile_cache = (char *) ki_data_alloc(src->data_type, K * tile * bytes);
|
|
1482
|
+
st->alias_mode = CA_ITER_ALIAS_STACK;
|
|
1483
|
+
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1484
|
+
break;
|
|
1485
|
+
}
|
|
2297
1486
|
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
st->
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
2314
|
-
op++;
|
|
1487
|
+
case KI_L2_STACK_OUTER_K: {
|
|
1488
|
+
/* Each slab is a region inside ONE parent, picked by
|
|
1489
|
+
outer_idx[stack_k_outer_pos]; next_slab_axes points into
|
|
1490
|
+
parents[k]->ptr + parent_off. A slab axis s of the stack is parent
|
|
1491
|
+
axis s, or s - 1 past the stack axis. */
|
|
1492
|
+
CAStack *stack = (CAStack *) src;
|
|
1493
|
+
int8_t k_axis = stack->k_axis;
|
|
1494
|
+
int8_t parent_ndim = src->ndim - 1;
|
|
1495
|
+
{
|
|
1496
|
+
ca_size_t sb = src->bytes, se = 1;
|
|
1497
|
+
for ( k = parent_ndim - 1; k >= 0; k-- ) {
|
|
1498
|
+
st->stack_parent_strides[k] = sb;
|
|
1499
|
+
st->stack_parent_mask_strides[k] = se;
|
|
1500
|
+
sb *= stack->parents[0]->dim[k];
|
|
1501
|
+
se *= stack->parents[0]->dim[k];
|
|
2315
1502
|
}
|
|
2316
1503
|
}
|
|
2317
|
-
st->
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
st->chunk_size = st->slab_n;
|
|
2328
|
-
|
|
2329
|
-
if ( st->outer_ndim > 0 ) {
|
|
2330
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
2331
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
2332
|
-
} else {
|
|
2333
|
-
st->outer_idx = NULL;
|
|
2334
|
-
}
|
|
2335
|
-
|
|
2336
|
-
if ( use_strided ) {
|
|
2337
|
-
ca_attach(root); /* window: finish detaches st->root */
|
|
2338
|
-
st->root = root; /* recorded once attached */
|
|
2339
|
-
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
2340
|
-
st->alias_ptr = (char *) root->ptr;
|
|
2341
|
-
} else if ( use_view_scratch ) {
|
|
2342
|
-
st->root = NULL;
|
|
2343
|
-
ki_gather_view(st, src, flags, 1);
|
|
2344
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
2345
|
-
st->alias_ptr = st->scratch_ptr;
|
|
2346
|
-
} else {
|
|
2347
|
-
st->root = NULL;
|
|
2348
|
-
ca_attach(src);
|
|
2349
|
-
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
2350
|
-
st->alias_ptr = (char *) src->ptr;
|
|
1504
|
+
st->stack_k_outer_pos = 0;
|
|
1505
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1506
|
+
if ( plan->in_slab[k] ) {
|
|
1507
|
+
int8_t parent_ax = (k < k_axis) ? k : (k - 1);
|
|
1508
|
+
lay.slab_strides[k] = st->stack_parent_strides[parent_ax];
|
|
1509
|
+
lay.mask_strides[k] = st->stack_parent_mask_strides[parent_ax];
|
|
1510
|
+
}
|
|
1511
|
+
else if ( k < k_axis ) {
|
|
1512
|
+
st->stack_k_outer_pos++;
|
|
1513
|
+
}
|
|
2351
1514
|
}
|
|
2352
|
-
|
|
2353
|
-
/*
|
|
2354
|
-
|
|
2355
|
-
|
|
1515
|
+
ki_attach_stack_parents(st, stack);
|
|
1516
|
+
/* A masked stack: copy the K parent masks as they are at the start
|
|
1517
|
+
of the walk, and next_slab_axes points into the copy of parent k
|
|
1518
|
+
(promise 2). The stack's mask may still read stand-ins for
|
|
1519
|
+
parents that have gained a mask since; read through to those. */
|
|
2356
1520
|
if ( ca_has_mask(src) ) {
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
1521
|
+
ca_multi_parent_mask_refresh(src->mask, stack->parents,
|
|
1522
|
+
stack->mask_standins,
|
|
1523
|
+
stack->n_parents);
|
|
1524
|
+
/* Zero-filled: a slot is set once its copy is made, and finish
|
|
1525
|
+
frees the copies whose slot is set. */
|
|
1526
|
+
st->stack_parent_mask_ptrs =
|
|
1527
|
+
(boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
|
|
1528
|
+
memset(st->stack_parent_mask_ptrs, 0,
|
|
1529
|
+
stack->n_parents * sizeof(boolean8_t *));
|
|
1530
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1531
|
+
CArray *pm = ((CAStack *) src->mask)->parents[kk];
|
|
1532
|
+
ca_size_t n = pm->elements;
|
|
1533
|
+
boolean8_t *copy = (boolean8_t *) xmalloc(n > 0 ? n : 1);
|
|
1534
|
+
st->stack_parent_mask_ptrs[kk] = copy;
|
|
1535
|
+
ca_attach(pm); /* window: only a memcpy before the detach */
|
|
1536
|
+
memcpy(copy, pm->ptr, n);
|
|
1537
|
+
ca_detach(pm);
|
|
2361
1538
|
}
|
|
2362
|
-
st->alias_mask = st->scratch_mask;
|
|
2363
1539
|
}
|
|
2364
|
-
|
|
1540
|
+
st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
|
|
1541
|
+
st->alias_ptr = NULL;
|
|
1542
|
+
break;
|
|
2365
1543
|
}
|
|
2366
1544
|
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
st->
|
|
2371
|
-
|
|
2372
|
-
st->outer_idx = NULL;
|
|
2373
|
-
} else {
|
|
2374
|
-
ca_size_t total = 1;
|
|
2375
|
-
int8_t k;
|
|
2376
|
-
for ( k = 0; k < nd - 1; k++ ) total *= src->dim[k];
|
|
2377
|
-
st->total_slabs = total;
|
|
2378
|
-
st->slab_n = src->dim[nd - 1];
|
|
2379
|
-
st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
|
|
2380
|
-
for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
|
|
2381
|
-
}
|
|
2382
|
-
st->slabs_emitted = 0;
|
|
2383
|
-
st->chunk_size = st->slab_n;
|
|
1545
|
+
case KI_L2_TRANSFER:
|
|
1546
|
+
ki_gather_view(st, src, flags, 0);
|
|
1547
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1548
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1549
|
+
break;
|
|
2384
1550
|
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
st->
|
|
1551
|
+
case KI_L2_ROOT_STRIDED:
|
|
1552
|
+
memcpy(st->composed_strides, plan->composed, src->ndim * sizeof(ca_size_t));
|
|
1553
|
+
st->composed_base = plan->base;
|
|
1554
|
+
memcpy(lay.slab_strides, plan->composed, src->ndim * sizeof(ca_size_t));
|
|
1555
|
+
memcpy(lay.outer_strides, plan->composed, src->ndim * sizeof(ca_size_t));
|
|
1556
|
+
ca_attach(plan->root); /* window: finish detaches st->root */
|
|
1557
|
+
st->root = plan->root; /* recorded once attached */
|
|
2388
1558
|
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
2389
|
-
st->alias_ptr = (char *) root->ptr;
|
|
2390
|
-
|
|
2391
|
-
|
|
1559
|
+
st->alias_ptr = (char *) plan->root->ptr;
|
|
1560
|
+
break;
|
|
1561
|
+
|
|
1562
|
+
case KI_L2_VIEW_SCRATCH:
|
|
1563
|
+
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
1564
|
+
src->dim, src->ndim, src->bytes);
|
|
2392
1565
|
ki_gather_view(st, src, flags, 1);
|
|
2393
1566
|
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
2394
1567
|
st->alias_ptr = st->scratch_ptr;
|
|
2395
|
-
|
|
2396
|
-
|
|
1568
|
+
break;
|
|
1569
|
+
|
|
1570
|
+
case KI_L2_CONTIG:
|
|
1571
|
+
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
1572
|
+
src->dim, src->ndim, src->bytes);
|
|
2397
1573
|
ca_attach(src);
|
|
2398
1574
|
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
2399
1575
|
st->alias_ptr = (char *) src->ptr;
|
|
1576
|
+
break;
|
|
2400
1577
|
}
|
|
2401
1578
|
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
if (
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
1579
|
+
ki_lay_out_slabs(st, src, plan->in_slab, &lay);
|
|
1580
|
+
|
|
1581
|
+
if ( plan->route == KI_L2_PER_SLAB ) {
|
|
1582
|
+
/* One max-slab buffer refilled per outer index, sized now that the
|
|
1583
|
+
slab is known.
|
|
1584
|
+
|
|
1585
|
+
Shortcut PER_SLAB_HOIST.
|
|
1586
|
+
Layer data; the fiber layer sits on top.
|
|
1587
|
+
Speeds up per-slab gathers whose innermost slab axis is STRIDE
|
|
1588
|
+
and with no SHIFT axis in the view: the outer and
|
|
1589
|
+
non-innermost slab offsets are hoisted and the inner
|
|
1590
|
+
run is copied with memcpy.
|
|
1591
|
+
Why no difference measured on a row-selected 2-D read.
|
|
1592
|
+
Enters at least one slab axis, the innermost slab axis
|
|
1593
|
+
STRIDE, no SHIFT axis.
|
|
1594
|
+
Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
|
|
1595
|
+
only when the view has one).
|
|
1596
|
+
Switch ca_fast_paths_enabled(): off, the PER_SLAB gather.
|
|
1597
|
+
Checked by the two-slab-axes rows of the grid parent in the
|
|
1598
|
+
delivery matrix. */
|
|
1599
|
+
ca_size_t slab_bytes = st->slab_elements * src->bytes;
|
|
1600
|
+
int hoist = (st->slab_ndim > 0) && ca_fast_paths_enabled()
|
|
1601
|
+
&& plan->descs[st->slab_axes_buf[st->slab_ndim - 1]].kind
|
|
1602
|
+
== CA_AXIS_KIND_STRIDE;
|
|
1603
|
+
for ( k = 0; hoist && k < src->ndim; k++ ) {
|
|
1604
|
+
if ( plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) hoist = 0;
|
|
2409
1605
|
}
|
|
2410
|
-
st->
|
|
1606
|
+
st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
|
|
1607
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1608
|
+
st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
|
|
1609
|
+
st->alias_mode = hoist ? CA_ITER_ALIAS_PER_SLAB_HOIST
|
|
1610
|
+
: CA_ITER_ALIAS_PER_SLAB;
|
|
2411
1611
|
}
|
|
1612
|
+
|
|
1613
|
+
/* The mask cursor (promise 2): fiber by fiber through fiber_mask_scratch
|
|
1614
|
+
on PER_FIBER_FUSED, the parents' copies on STACK_OUTER_K, none on
|
|
1615
|
+
STACK (an unmasked stack), and the walk's copy of the whole view's
|
|
1616
|
+
mask everywhere else. */
|
|
1617
|
+
if ( plan->route == KI_L2_FIBER_FUSED || plan->route == KI_L2_STACK
|
|
1618
|
+
|| plan->route == KI_L2_STACK_OUTER_K ) {
|
|
1619
|
+
st->alias_mask = NULL;
|
|
1620
|
+
}
|
|
1621
|
+
else {
|
|
1622
|
+
ki_take_mask_copy(st, src);
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
static int
|
|
1627
|
+
ca_iter_state_init_l2_unprotected (ca_iter_state *st,
|
|
1628
|
+
struct _CArray *src,
|
|
1629
|
+
ca_slab_policy_t policy,
|
|
1630
|
+
int8_t *axes,
|
|
1631
|
+
int8_t naxes,
|
|
1632
|
+
uint32_t flags)
|
|
1633
|
+
{
|
|
1634
|
+
ki_l2_plan plan;
|
|
1635
|
+
int rc;
|
|
1636
|
+
|
|
1637
|
+
/* Leave the state defined before anything can return. Every error
|
|
1638
|
+
return below hands control back to a caller that may not look at the
|
|
1639
|
+
code, so what it is left holding has to be a state that iterates zero
|
|
1640
|
+
times rather than whatever its stack frame happened to contain. */
|
|
1641
|
+
if ( st == NULL ) return CA_ITER_ERR_FLAGS;
|
|
1642
|
+
memset(st, 0, sizeof(*st));
|
|
1643
|
+
if ( src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
1644
|
+
|
|
1645
|
+
/* A Face only layers a semantic identifier and a CALazyMarker only
|
|
1646
|
+
layers "leaf of a lazy chain"; storage is identical to the parent
|
|
1647
|
+
either way, so strip at the kernel-compute entry and descend.
|
|
1648
|
+
Routing and alias decisions belong to the parent. Strip before
|
|
1649
|
+
validate_inputs: left on, both would classify as SRC_ATTACH (they
|
|
1650
|
+
have attach slots) and be materialised whole (see
|
|
1651
|
+
ca_iter_strip_storage_wrapper).
|
|
1652
|
+
|
|
1653
|
+
The Face identifier is re-wrapped onto the result by the caller's lift
|
|
1654
|
+
hook (= primary operators / reductions / etc.). The marker is not:
|
|
1655
|
+
a reduction over a lazy marker yields a plain entity.
|
|
1656
|
+
|
|
1657
|
+
WRITE is rejected against the wrapper, before the strip — a marker is
|
|
1658
|
+
read-only while its parent is not. validate_inputs re-checks the
|
|
1659
|
+
stripped source, which is harmless. */
|
|
1660
|
+
if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
|
|
1661
|
+
return CA_ITER_ERR_READONLY;
|
|
1662
|
+
}
|
|
1663
|
+
if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
|
|
1664
|
+
src = ca_iter_strip_storage_wrapper(src);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
rc = ca_iter_validate_inputs(st, src, policy, flags);
|
|
1668
|
+
if ( rc != CA_ITER_OK ) return rc;
|
|
1669
|
+
/* L2 walks a slab of named axes. The whole-array walk is L1
|
|
1670
|
+
(init_l1 with CA_SLAB_WHOLE); one strided run per innermost row is
|
|
1671
|
+
CA_SLAB_AXES with the last axis. */
|
|
1672
|
+
if ( policy != CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
|
|
1673
|
+
|
|
1674
|
+
/* The slab axes: in range, no repeats. */
|
|
1675
|
+
if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
|
|
1676
|
+
return CA_ITER_ERR_POLICY;
|
|
1677
|
+
}
|
|
1678
|
+
memset(plan.in_slab, 0, sizeof(plan.in_slab));
|
|
1679
|
+
for ( int8_t k = 0; k < naxes; k++ ) {
|
|
1680
|
+
int8_t ax = axes[k];
|
|
1681
|
+
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1682
|
+
if ( plan.in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
1683
|
+
plan.in_slab[ax] = 1;
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
/* route_source returns the refined src_kind and, for descriptor
|
|
1687
|
+
sources, the descriptors, so the choice below does not describe the
|
|
1688
|
+
axes again. */
|
|
1689
|
+
plan.ndesc = 0;
|
|
1690
|
+
plan.root = NULL;
|
|
1691
|
+
plan.base = 0;
|
|
1692
|
+
plan.src_kind = ca_iter_route_source(src, plan.descs, plan.pdims,
|
|
1693
|
+
&plan.ndesc);
|
|
1694
|
+
rc = ki_choose_l2_route(src, axes, naxes, flags, &plan);
|
|
1695
|
+
if ( rc != CA_ITER_OK ) return rc;
|
|
1696
|
+
|
|
1697
|
+
ki_open_l2(st, src, policy, axes, naxes, flags, &plan);
|
|
2412
1698
|
return CA_ITER_OK;
|
|
2413
1699
|
}
|
|
2414
1700
|
|
|
@@ -2435,74 +1721,6 @@ ca_iter_state_next_slab_unprotected (ca_iter_state *st,
|
|
|
2435
1721
|
return 1;
|
|
2436
1722
|
}
|
|
2437
1723
|
|
|
2438
|
-
static int
|
|
2439
|
-
ca_iter_state_next_slab_strided_unprotected (ca_iter_state *st,
|
|
2440
|
-
char **out_ptr,
|
|
2441
|
-
boolean8_t **out_mask,
|
|
2442
|
-
ca_size_t *out_n,
|
|
2443
|
-
ca_size_t *out_stride_bytes)
|
|
2444
|
-
{
|
|
2445
|
-
if ( st == NULL || st->level != 2
|
|
2446
|
-
|| st->slabs_emitted >= st->total_slabs ) {
|
|
2447
|
-
if ( out_ptr ) *out_ptr = NULL;
|
|
2448
|
-
if ( out_mask ) *out_mask = NULL;
|
|
2449
|
-
if ( out_n ) *out_n = 0;
|
|
2450
|
-
if ( out_stride_bytes ) *out_stride_bytes = 0;
|
|
2451
|
-
return 0;
|
|
2452
|
-
}
|
|
2453
|
-
|
|
2454
|
-
int8_t nd = st->ndim;
|
|
2455
|
-
ca_size_t inner_st = (nd > 0)
|
|
2456
|
-
? st->composed_strides[nd - 1]
|
|
2457
|
-
: st->bytes;
|
|
2458
|
-
ca_size_t off = st->composed_base;
|
|
2459
|
-
int8_t k;
|
|
2460
|
-
|
|
2461
|
-
/* Sum outer prefix offset. For F-2 descriptor L2 alias the prefix can
|
|
2462
|
-
contain INDEX axes (and, once OOB fill_slab lands, SHIFT axes), so
|
|
2463
|
-
we delegate to the pre-classified prefix[] engine. Otherwise (=
|
|
2464
|
-
CAStride / entity outer = STRIDE-only by construction) use the
|
|
2465
|
-
direct sum that has been the L2 inner loop since step 3. For
|
|
2466
|
-
0-d / 1-d sources, the loop / call is a no-op and off stays at
|
|
2467
|
-
composed_base. */
|
|
2468
|
-
if ( st->outer_idx != NULL ) {
|
|
2469
|
-
if ( st->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
2470
|
-
int oob = 0;
|
|
2471
|
-
off += ca_axis_dispatch_prefix_offset(st->prefix, st->outer_idx,
|
|
2472
|
-
nd - 1, &oob);
|
|
2473
|
-
/* oob unreachable: outer SHIFT was downgraded in init_l2. */
|
|
2474
|
-
} else {
|
|
2475
|
-
for ( k = 0; k < nd - 1; k++ ) {
|
|
2476
|
-
off += st->outer_idx[k] * st->composed_strides[k];
|
|
2477
|
-
}
|
|
2478
|
-
}
|
|
2479
|
-
}
|
|
2480
|
-
|
|
2481
|
-
if ( out_ptr ) *out_ptr = st->alias_ptr + off;
|
|
2482
|
-
/* mask layout mirrors value layout, but mask is contig boolean8_t
|
|
2483
|
-
when alias_mask is a scratch buffer (step 6 baseline = always
|
|
2484
|
-
gather mask into scratch_mask for uniformity). Per-slab mask
|
|
2485
|
-
offset = i in the outer cursor (= slabs_emitted at this point). */
|
|
2486
|
-
if ( out_mask ) {
|
|
2487
|
-
*out_mask = st->alias_mask
|
|
2488
|
-
? st->alias_mask + st->slabs_emitted * st->slab_n
|
|
2489
|
-
: NULL;
|
|
2490
|
-
}
|
|
2491
|
-
if ( out_n ) *out_n = st->slab_n;
|
|
2492
|
-
if ( out_stride_bytes ) *out_stride_bytes = inner_st;
|
|
2493
|
-
|
|
2494
|
-
st->slabs_emitted += 1;
|
|
2495
|
-
|
|
2496
|
-
/* Advance outer_idx row-major (least-significant axis innermost,
|
|
2497
|
-
so we tick outer_idx[ndim-2] first). */
|
|
2498
|
-
if ( st->outer_idx != NULL ) {
|
|
2499
|
-
for ( k = nd - 2; k >= 0; k-- ) {
|
|
2500
|
-
if ( ++st->outer_idx[k] < st->src->dim[k] ) break;
|
|
2501
|
-
st->outer_idx[k] = 0;
|
|
2502
|
-
}
|
|
2503
|
-
}
|
|
2504
|
-
return 1;
|
|
2505
|
-
}
|
|
2506
1724
|
|
|
2507
1725
|
/* CA_KERNEL_FIBER_CONTIG: a fiber (a walk with one slab axis) reaches the
|
|
2508
1726
|
kernel as a contiguous run of cells and of mask flags. Each path of
|
|
@@ -2565,12 +1783,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2565
1783
|
return 0;
|
|
2566
1784
|
}
|
|
2567
1785
|
|
|
2568
|
-
/*
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
engine per-cell dispatch). Target: 1.5-1.8x win for INDEX slab
|
|
2573
|
-
with innermost STRIDE (= grid / select sparse projection use cases). */
|
|
1786
|
+
/* PER_SLAB_HOIST (see its name tag in ki_open_l2): the innermost slab
|
|
1787
|
+
axis is STRIDE and no axis is SHIFT. The outer and non-innermost
|
|
1788
|
+
slab offsets are computed once per slab row, and the inner run is a
|
|
1789
|
+
plain memcpy instead of the descriptor engine's per-cell walk. */
|
|
2574
1790
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB_HOIST ) {
|
|
2575
1791
|
int8_t inner_view_ax = st->slab_axes_buf[st->slab_ndim - 1];
|
|
2576
1792
|
ca_size_t inner_count = st->descs[inner_view_ax].count;
|
|
@@ -2666,17 +1882,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2666
1882
|
return 1;
|
|
2667
1883
|
}
|
|
2668
1884
|
|
|
2669
|
-
/*
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
k_axis -- stack axis s != k_axis maps to parent axis s if
|
|
2679
|
-
s < k_axis else s - 1. */
|
|
1885
|
+
/* CAStack source with the stack axis k_axis in the outer iteration.
|
|
1886
|
+
Each slab is a region inside parents[k]->ptr where k =
|
|
1887
|
+
outer_idx[K_outer_pos]. Parent inner byte offset = Σ outer_idx[m] *
|
|
1888
|
+
stack_parent_strides[parent_ax] over all outer axes except the K-axis
|
|
1889
|
+
itself. Mask: a pointer into the copy of parent k's mask that init
|
|
1890
|
+
took (stack_parent_mask_ptrs[k] + mask_off); a write through it
|
|
1891
|
+
reaches nothing. The data is yielded in place unless FIBER_CONTIG
|
|
1892
|
+
needs a strided fiber gathered (ki_fiber_contig_data). A stack axis
|
|
1893
|
+
s other than k_axis is parent axis s, or s - 1 past k_axis. */
|
|
2680
1894
|
if ( st->alias_mode == CA_ITER_ALIAS_STACK_OUTER_K ) {
|
|
2681
1895
|
int8_t kpos = st->stack_k_outer_pos;
|
|
2682
1896
|
int32_t k = (int32_t) st->outer_idx[kpos];
|
|
@@ -2711,20 +1925,19 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2711
1925
|
return 1;
|
|
2712
1926
|
}
|
|
2713
1927
|
|
|
2714
|
-
/*
|
|
2715
|
-
|
|
1928
|
+
/* CAStack read along its stack axis, from the parents' memory. For
|
|
1929
|
+
each outer index, compute the
|
|
2716
1930
|
parent inner byte offset (= same across all K parents, uniform
|
|
2717
|
-
shape),
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
stack axes
|
|
2724
|
-
maps
|
|
2725
|
-
stack axis m+1 = parent axis m). */
|
|
1931
|
+
shape), and yield the K cells of that fiber out of stack_tile_cache,
|
|
1932
|
+
which is refilled TILE fibers at a time from the cached parent ptrs
|
|
1933
|
+
(below). No ca_xfer_stride dispatch. The scratch_ptr init
|
|
1934
|
+
allocates is not used by this path.
|
|
1935
|
+
|
|
1936
|
+
init engages this path only for slab_axes == [k_axis], so the outer
|
|
1937
|
+
axes are the stack axes other than k_axis, which are the parent axes
|
|
1938
|
+
in order: outer_idx[m] maps to parent axis m. */
|
|
2726
1939
|
if ( st->alias_mode == CA_ITER_ALIAS_STACK ) {
|
|
2727
|
-
/*
|
|
1940
|
+
/* Tile cache. Outer iter
|
|
2728
1941
|
walks parent storage in row-major order so parent_off increments
|
|
2729
1942
|
by `bytes` per call monotonically. Refill TILE fibers at once
|
|
2730
1943
|
via K contig parent streams (= TILE consecutive cells from each
|
|
@@ -2794,11 +2007,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2794
2007
|
return 1;
|
|
2795
2008
|
}
|
|
2796
2009
|
|
|
2797
|
-
/*
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
for one max slab and reused across iters (D1.1 (B)). */
|
|
2010
|
+
/* PER_SLAB: one slab gathered by ca_axis_dispatch_gather, from a
|
|
2011
|
+
subset of the descriptors with the outer axes pinned at the current
|
|
2012
|
+
outer_idx position. scratch_ptr is sized for one slab and reused for
|
|
2013
|
+
every slab. */
|
|
2802
2014
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB ) {
|
|
2803
2015
|
ca_axis_desc_t subset_descs[CA_RANK_MAX];
|
|
2804
2016
|
memcpy(subset_descs, st->descs, st->ndim * sizeof(ca_axis_desc_t));
|
|
@@ -2833,7 +2045,7 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2833
2045
|
}
|
|
2834
2046
|
}
|
|
2835
2047
|
|
|
2836
|
-
/* CAWindow fill value
|
|
2048
|
+
/* CAWindow fill value (the same lookup as in ki_open_l2). */
|
|
2837
2049
|
const void *bound_fill = NULL;
|
|
2838
2050
|
if ( ca_func[st->src->obj_type].attach == ca_window_func.attach ) {
|
|
2839
2051
|
bound_fill = ((CAWindow *) st->src)->fill;
|
|
@@ -2843,8 +2055,8 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2843
2055
|
st->ndim, st->bytes, st->slab_elements,
|
|
2844
2056
|
bound_fill, st->scratch_ptr);
|
|
2845
2057
|
|
|
2846
|
-
/* Mask offset into whole-view
|
|
2847
|
-
|
|
2058
|
+
/* Mask offset into the whole-view mask copy, which is view
|
|
2059
|
+
row-major to match slab_mask_strides. */
|
|
2848
2060
|
ca_size_t mask_off = 0;
|
|
2849
2061
|
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2850
2062
|
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
@@ -2871,10 +2083,9 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2871
2083
|
(element units, view row-major). Both are zero when outer_ndim
|
|
2872
2084
|
== 0 (= all-axes WHOLE-equivalent case).
|
|
2873
2085
|
|
|
2874
|
-
STRIDE outer axes (
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
|
|
2086
|
+
STRIDE outer axes (SRC_CASTRIDE, and the STRIDE axes of a descriptor
|
|
2087
|
+
view) use outer_idx[m] directly as the multiplier. INDEX outer axes
|
|
2088
|
+
of a descriptor view need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
|
|
2878
2089
|
Mask offset uses outer_idx[m] directly in both cases (= mask
|
|
2879
2090
|
scratch is in view row-major order, outer_idx walks view-axis
|
|
2880
2091
|
positions). */
|
|
@@ -2894,13 +2105,12 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2894
2105
|
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
2895
2106
|
}
|
|
2896
2107
|
|
|
2897
|
-
/*
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
region path.
|
|
2108
|
+
/* PER_FIBER_FUSED: there is no whole-view buffer to alias from. Build
|
|
2109
|
+
the fiber region from outer_idx + fiber_axis and call
|
|
2110
|
+
ca_xfer_stride(src, ..., GET) directly into fiber_data_scratch; the
|
|
2111
|
+
view's region transfer (out-of-bounds filling for CAWindow / CAShift,
|
|
2112
|
+
the conversion for a fake or byte-swap view) delivers it in one
|
|
2113
|
+
pass.
|
|
2904
2114
|
|
|
2905
2115
|
last_data_off carries the fiber region's outer footprint so
|
|
2906
2116
|
sync_slab can reconstruct the same region for WRITE PUT. Encode
|
|
@@ -2924,8 +2134,8 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2924
2134
|
/* Build fiber region: fiber_axis spans the full fiber, all other
|
|
2925
2135
|
axes pinned to outer_idx position (count=1). Cache starts[] in
|
|
2926
2136
|
state so sync_slab can rebuild the same region for WRITE PUT
|
|
2927
|
-
(= captured BEFORE outer_idx advance below, same hazard
|
|
2928
|
-
|
|
2137
|
+
(= captured BEFORE the outer_idx advance below, the same hazard as
|
|
2138
|
+
last_data_off; see sync_slab). */
|
|
2929
2139
|
{
|
|
2930
2140
|
int8_t op = 0;
|
|
2931
2141
|
for ( int8_t k = 0; k < nd; k++ ) {
|
|
@@ -2985,16 +2195,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2985
2195
|
return 1;
|
|
2986
2196
|
}
|
|
2987
2197
|
|
|
2988
|
-
/*
|
|
2989
|
-
For naxes==1 (= single slab axis = fiber) the catalog contract
|
|
2198
|
+
/* Contiguous fiber delivery (promise 1). For naxes==1 (= single slab axis = fiber) the catalog contract
|
|
2990
2199
|
CA_FOR_EACH_FIBER promises contig data delivery. When the fiber
|
|
2991
2200
|
is not innermost-contig (= slab_strides[0] != bytes), the engine
|
|
2992
2201
|
gathers the fiber into fiber_data_scratch before yielding so the
|
|
2993
2202
|
author can write p[i] without stride math.
|
|
2994
2203
|
|
|
2995
2204
|
Capture last_data_off BEFORE the outer_idx advance below; sync_slab
|
|
2996
|
-
consumes it for WRITE scatter. See header field
|
|
2997
|
-
|
|
2205
|
+
consumes it for WRITE scatter. See the header field and the hazard
|
|
2206
|
+
comment in sync_slab. */
|
|
2998
2207
|
/* sync_slab finds the slab here: the fiber it scatters back, and the
|
|
2999
2208
|
cells of a whole-view scratch it compares. */
|
|
3000
2209
|
st->last_data_off = data_off;
|
|
@@ -3041,8 +2250,6 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
3041
2250
|
/* ======================================================================
|
|
3042
2251
|
* !!! CORRECTNESS HAZARD - DO NOT MOVE !!!
|
|
3043
2252
|
*
|
|
3044
|
-
* (PROPOSAL_FIBER_DELIVERY F.1a)
|
|
3045
|
-
*
|
|
3046
2253
|
* Per-fiber scratch reuse + scatter correctness depends on the strict
|
|
3047
2254
|
* evaluation order of the CA_FOR_EACH_FIBER_* macro sandwich:
|
|
3048
2255
|
*
|
|
@@ -3069,14 +2276,13 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
3069
2276
|
* instead of reusing -- separate phase, separate design.
|
|
3070
2277
|
* ====================================================================== */
|
|
3071
2278
|
|
|
3072
|
-
/*
|
|
3073
|
-
When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, the author
|
|
2279
|
+
/* PER_FIBER_FUSED write: the author
|
|
3074
2280
|
wrote into fiber_data_scratch and there is no whole-view buffer
|
|
3075
2281
|
to xfer_all PUT. Rebuild the same fiber region from cached
|
|
3076
2282
|
fiber_region_starts[] (= captured pre-advance in next_slab_axes,
|
|
3077
|
-
same hazard
|
|
3078
|
-
|
|
3079
|
-
|
|
2283
|
+
the same hazard as last_data_off) and call ca_xfer_stride(src, ...,
|
|
2284
|
+
PUT), which puts the fiber back through the view's region transfer
|
|
2285
|
+
in one pass. */
|
|
3080
2286
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
|
|
3081
2287
|
ca_size_t fiber_n = st->slab_dims[0];
|
|
3082
2288
|
int8_t nd = st->ndim;
|
|
@@ -3091,7 +2297,7 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
3091
2297
|
return;
|
|
3092
2298
|
}
|
|
3093
2299
|
|
|
3094
|
-
/* Per-fiber gather path scatter
|
|
2300
|
+
/* Per-fiber gather path scatter.
|
|
3095
2301
|
When next_slab_axes gathered the fiber into fiber_data_scratch
|
|
3096
2302
|
(= naxes==1 + slab_strides[0] != bytes), scatter it back to the
|
|
3097
2303
|
source layout via the strided write. When the fiber was the
|
|
@@ -3116,77 +2322,48 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
3116
2322
|
it fires for any source whose fiber is not innermost-contig -- so
|
|
3117
2323
|
for SRC_ATTACH this has just written the fiber back into the
|
|
3118
2324
|
whole-view scratch, and the push-back below carries it to the
|
|
3119
|
-
source. For the alias paths scratch_ptr is NULL and the
|
|
3120
|
-
below
|
|
3121
|
-
}
|
|
3122
|
-
|
|
3123
|
-
/* SRC_ATTACH path: the kernel wrote into the iterator's whole-view
|
|
3124
|
-
scratch. The changed cells go back through the view's own transfer,
|
|
3125
|
-
which converts, packs or scatters them as the view requires.
|
|
3126
|
-
|
|
3127
|
-
This has to stay below the two per-fiber blocks: consulting
|
|
3128
|
-
src_kind first would push back the whole-view buffer while the
|
|
3129
|
-
fiber the author actually wrote sat unscattered in
|
|
3130
|
-
fiber_data_scratch, and would push back a NULL one for
|
|
3131
|
-
PER_FIBER_FUSED, which owns no whole-view buffer at all. */
|
|
3132
|
-
if ( st->src_kind == CA_ITER_SRC_ATTACH ) {
|
|
3133
|
-
ki_scratch_push_changed(st);
|
|
3134
|
-
st->write_dirty = 0;
|
|
3135
|
-
return;
|
|
2325
|
+
source. For the alias paths scratch_ptr is NULL and the push-back
|
|
2326
|
+
below is skipped. */
|
|
3136
2327
|
}
|
|
3137
2328
|
|
|
3138
|
-
/* alias path: kernel wrote through alias_ptr into
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
stays NULL and we no-op correctly). */
|
|
2329
|
+
/* alias path: the kernel wrote through alias_ptr into the memory the
|
|
2330
|
+
walk borrowed, so nothing is scattered. Applies to all CAStride alias and to
|
|
2331
|
+
STRIDE-only descriptor alias (SRC_DESCRIPTOR_L2_ALIASABLE), whose
|
|
2332
|
+
scratch_ptr is NULL too. An alias walk over a view root is written
|
|
2333
|
+
back once, by finish (ca_sync_detach on the root). */
|
|
3144
2334
|
if ( st->scratch_ptr == NULL ) return;
|
|
3145
2335
|
|
|
3146
|
-
/* scratch path:
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
and are written normally. Iteration order is engine-defined;
|
|
3166
|
-
user kernels must not rely on it.
|
|
3167
|
-
|
|
3168
|
-
scatter wants the pre-merge raw descriptors (the engine re-runs
|
|
3169
|
-
_prepare internally with whatever we give it). init cached
|
|
3170
|
-
the post-merge axes in st->descs for next_slab_strided's
|
|
3171
|
-
offset math; we re-call describe_axes here for the scatter
|
|
3172
|
-
call rather than caching a second copy in the state struct. */
|
|
3173
|
-
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
3174
|
-
ca_size_t raw_pdims[CA_RANK_MAX];
|
|
3175
|
-
int8_t raw_ndim = 0;
|
|
3176
|
-
ca_iter_describe_axes(st->src, raw_descs, raw_pdims, &raw_ndim);
|
|
3177
|
-
ca_axis_dispatch_scatter(st->root /* parent */,
|
|
3178
|
-
raw_pdims,
|
|
3179
|
-
raw_descs, raw_ndim,
|
|
3180
|
-
st->bytes, st->total_elements,
|
|
3181
|
-
st->scratch_ptr);
|
|
3182
|
-
break;
|
|
3183
|
-
}
|
|
3184
|
-
}
|
|
2336
|
+
/* scratch path: the kernel wrote into a whole-view scratch, and the
|
|
2337
|
+
changed cells go back through the view's own transfer, which
|
|
2338
|
+
converts, packs or scatters them as the view requires (promise 3).
|
|
2339
|
+
Every write walk that reads a scratch gathered it with
|
|
2340
|
+
ki_gather_view, which keeps the copy the comparison needs: the
|
|
2341
|
+
transfer walk (SRC_ATTACH) and the CAStride paths that read a
|
|
2342
|
+
gathered copy. The descriptor engine's own gathers serve reads
|
|
2343
|
+
only; a write over a descriptor view is sent to the transfer walk
|
|
2344
|
+
at init.
|
|
2345
|
+
|
|
2346
|
+
This has to stay below the two per-fiber blocks: pushing back first
|
|
2347
|
+
would send the whole-view buffer while the fiber the author actually
|
|
2348
|
+
wrote sat unscattered in fiber_data_scratch. */
|
|
2349
|
+
if ( st->src_kind != CA_ITER_SRC_ATTACH
|
|
2350
|
+
&& st->src_kind != CA_ITER_SRC_CASTRIDE ) {
|
|
2351
|
+
rb_bug("kernel iterator: a write walk over a scratch with nothing to "
|
|
2352
|
+
"compare it against (src_kind %d)", (int) st->src_kind);
|
|
2353
|
+
}
|
|
2354
|
+
ki_scratch_push_changed(st);
|
|
3185
2355
|
st->write_dirty = 0;
|
|
3186
2356
|
}
|
|
3187
2357
|
|
|
3188
2358
|
static VALUE ca_iter_finish_sync (VALUE arg);
|
|
3189
2359
|
|
|
2360
|
+
static VALUE
|
|
2361
|
+
ki_sync_detach_root (VALUE root)
|
|
2362
|
+
{
|
|
2363
|
+
ca_sync_detach((CArray *) root);
|
|
2364
|
+
return Qnil;
|
|
2365
|
+
}
|
|
2366
|
+
|
|
3190
2367
|
void
|
|
3191
2368
|
ca_iter_state_finish (ca_iter_state *st)
|
|
3192
2369
|
{
|
|
@@ -3205,11 +2382,11 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3205
2382
|
/* Lifecycle cleanup — orders matter slightly (free scratch before
|
|
3206
2383
|
detaching parent so the kernel iterator's resources are released
|
|
3207
2384
|
symmetrically with init):
|
|
3208
|
-
- scratch_ptr: owned by
|
|
3209
|
-
descriptor
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
ca_detach.
|
|
2385
|
+
- scratch_ptr: owned by the iterator (every gather into scratch:
|
|
2386
|
+
CAStride non-contig, descriptor, SRC_ATTACH, per-slab, stack).
|
|
2387
|
+
ki_data_free.
|
|
2388
|
+
- root: descriptor parent or CAStride compose-fold root attached
|
|
2389
|
+
at init. ca_detach, or ca_sync_detach on a WRITE walk.
|
|
3213
2390
|
- else (alias paths): src was attached at init, detach. */
|
|
3214
2391
|
if ( st->scratch_ptr ) {
|
|
3215
2392
|
ki_data_free(st->scratch_ptr, st->scratch_cap);
|
|
@@ -3220,7 +2397,7 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3220
2397
|
xfree(st->scratch_mask);
|
|
3221
2398
|
st->scratch_mask = NULL;
|
|
3222
2399
|
}
|
|
3223
|
-
/*
|
|
2400
|
+
/* The per-fiber scratch. */
|
|
3224
2401
|
if ( st->fiber_data_scratch ) {
|
|
3225
2402
|
ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
|
|
3226
2403
|
st->fiber_data_scratch = NULL;
|
|
@@ -3231,9 +2408,10 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3231
2408
|
st->fiber_mask_scratch = NULL;
|
|
3232
2409
|
st->fiber_mask_scratch_cap = 0;
|
|
3233
2410
|
}
|
|
3234
|
-
/*
|
|
3235
|
-
|
|
3236
|
-
|
|
2411
|
+
/* Detach the K parents init attached and free the cached ptr array
|
|
2412
|
+
(symmetric with init_l2's per-parent ca_attach loop). The parent
|
|
2413
|
+
mask copies are freed separately below; they were never attached
|
|
2414
|
+
for the walk. */
|
|
3237
2415
|
if ( st->stack_parent_ptrs ) {
|
|
3238
2416
|
CAStack *stack = (CAStack *) st->src;
|
|
3239
2417
|
for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
|
|
@@ -3242,7 +2420,7 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3242
2420
|
xfree(st->stack_parent_ptrs);
|
|
3243
2421
|
st->stack_parent_ptrs = NULL;
|
|
3244
2422
|
}
|
|
3245
|
-
/*
|
|
2423
|
+
/* The STACK tile cache. */
|
|
3246
2424
|
if ( st->stack_tile_cache ) {
|
|
3247
2425
|
ki_data_free(st->stack_tile_cache, 0);
|
|
3248
2426
|
st->stack_tile_cache = NULL;
|
|
@@ -3260,37 +2438,52 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3260
2438
|
st->stack_parent_mask_ptrs = NULL;
|
|
3261
2439
|
}
|
|
3262
2440
|
st->stack_n_parents = 0;
|
|
3263
|
-
|
|
3264
|
-
/*
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
st->root
|
|
3269
|
-
|
|
3270
|
-
|
|
2441
|
+
{
|
|
2442
|
+
/* The state is emptied before anything here can raise: syncing a
|
|
2443
|
+
write walk's root goes through the root's slots. A finish that
|
|
2444
|
+
raised is then finished already, and a second call returns at once
|
|
2445
|
+
instead of detaching src again. */
|
|
2446
|
+
CArray *root = st->root;
|
|
2447
|
+
CArray *src = st->src;
|
|
2448
|
+
CArray *own = (st->flags & CA_ITER_FLAG_OWNS_SRC) ? src : NULL;
|
|
2449
|
+
int write = (st->flags & CA_KERNEL_WRITE) != 0;
|
|
2450
|
+
int detach_src = root == NULL
|
|
2451
|
+
&& ( st->alias_mode == CA_ITER_ALIAS_CONTIG
|
|
2452
|
+
|| st->alias_mode == CA_ITER_ALIAS_STRIDED );
|
|
2453
|
+
int tag = 0;
|
|
2454
|
+
st->root = NULL;
|
|
2455
|
+
st->src = NULL;
|
|
2456
|
+
st->alias_ptr = NULL;
|
|
2457
|
+
st->flags &= ~CA_ITER_FLAG_OWNS_SRC;
|
|
2458
|
+
if ( root ) {
|
|
2459
|
+
/* A write walk wrote into the root's attach buffer, which is the root
|
|
2460
|
+
itself only for an entity: a view root (a transpose, a non-contig
|
|
2461
|
+
block, a view lending no memory) gave a buffer of its own. */
|
|
2462
|
+
if ( write ) {
|
|
2463
|
+
/* protect: frees the iterator's own copy before the raise passes on */
|
|
2464
|
+
rb_protect(ki_sync_detach_root, (VALUE) root, &tag);
|
|
2465
|
+
}
|
|
2466
|
+
else {
|
|
2467
|
+
ca_detach(root);
|
|
2468
|
+
}
|
|
3271
2469
|
}
|
|
3272
|
-
else {
|
|
3273
|
-
|
|
2470
|
+
else if ( detach_src ) {
|
|
2471
|
+
/* alias path (CONTIG/STRIDED for CAStride family): we attached
|
|
2472
|
+
src directly at init, so detach it here. A SRC_ATTACH walk owns
|
|
2473
|
+
its own scratch (alias_mode = NONE, freed above) and attached
|
|
2474
|
+
nothing to detach. */
|
|
2475
|
+
ca_detach(src);
|
|
3274
2476
|
}
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
}
|
|
3284
|
-
if ( st->flags & CA_ITER_FLAG_OWNS_SRC ) {
|
|
3285
|
-
CArray *own = st->src;
|
|
3286
|
-
st->flags &= ~CA_ITER_FLAG_OWNS_SRC;
|
|
3287
|
-
if ( ca_is_object_type(own) ) {
|
|
3288
|
-
ca_gc_release_buffer(own->ptr);
|
|
2477
|
+
if ( own ) {
|
|
2478
|
+
if ( ca_is_object_type(own) ) {
|
|
2479
|
+
ca_gc_release_buffer(own->ptr);
|
|
2480
|
+
}
|
|
2481
|
+
ca_free(own);
|
|
2482
|
+
}
|
|
2483
|
+
if ( tag ) {
|
|
2484
|
+
rb_jump_tag(tag);
|
|
3289
2485
|
}
|
|
3290
|
-
ca_free(own);
|
|
3291
2486
|
}
|
|
3292
|
-
st->src = NULL;
|
|
3293
|
-
st->alias_ptr = NULL;
|
|
3294
2487
|
}
|
|
3295
2488
|
|
|
3296
2489
|
/* Finish a walk the way the block macros leave it: a slab of a write walk
|
|
@@ -3352,15 +2545,31 @@ typedef struct {
|
|
|
3352
2545
|
char **out_ptr;
|
|
3353
2546
|
boolean8_t **out_mask;
|
|
3354
2547
|
ca_size_t *out_n;
|
|
3355
|
-
ca_size_t *out_stride_bytes;
|
|
3356
2548
|
int rc;
|
|
3357
2549
|
} ca_iter_call_t;
|
|
3358
2550
|
|
|
3359
|
-
/* Whether next_slab / sync_slab on this walk need the protection.
|
|
2551
|
+
/* Whether next_slab / sync_slab on this walk need the protection.
|
|
2552
|
+
|
|
2553
|
+
Shortcut PROTECT_ONLY_MAY_RAISE.
|
|
2554
|
+
Layer the exits (promise 4): which walks run next_slab /
|
|
2555
|
+
sync_slab under rb_protect.
|
|
2556
|
+
Speeds up walks that cannot raise there: they skip rb_protect.
|
|
2557
|
+
Why about 1.5x on a per-axis sum over 200k short fibers.
|
|
2558
|
+
Enters the route is not PER_FIBER_FUSED and the walk does not
|
|
2559
|
+
write back out of scratch.
|
|
2560
|
+
Promises 4 (a walk left out cannot raise between init and finish).
|
|
2561
|
+
Switch ca_fast_paths_enabled(): off, every walk is protected.
|
|
2562
|
+
Checked by test_iter_raise_cleanup.rb; the raise rows of
|
|
2563
|
+
test_iter_delivery_detectors.rb. */
|
|
3360
2564
|
static void
|
|
3361
2565
|
ca_iter_mark_walk_may_raise (ca_iter_state *st)
|
|
3362
2566
|
{
|
|
3363
2567
|
if ( st == NULL ) return;
|
|
2568
|
+
if ( ! ca_fast_paths_enabled() ) {
|
|
2569
|
+
/* the general path protects every walk */
|
|
2570
|
+
st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
|
|
2571
|
+
return;
|
|
2572
|
+
}
|
|
3364
2573
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
|
|
3365
2574
|
/* gathers each fiber through the source's xfer_stride slot */
|
|
3366
2575
|
st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
|
|
@@ -3369,7 +2578,8 @@ ca_iter_mark_walk_may_raise (ca_iter_state *st)
|
|
|
3369
2578
|
&& (st->scratch_ptr != NULL
|
|
3370
2579
|
|| st->src_kind == CA_ITER_SRC_ATTACH) ) {
|
|
3371
2580
|
/* writes back out of iterator-owned scratch, through the source's
|
|
3372
|
-
|
|
2581
|
+
transfer (ki_scratch_push_changed: ca_xfer_addrs / ca_xfer_all PUT)
|
|
2582
|
+
or the descriptor scatter. A write into aliased memory needs no
|
|
3373
2583
|
push-back at all: sync_slab returns at once. */
|
|
3374
2584
|
st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
|
|
3375
2585
|
}
|
|
@@ -3378,6 +2588,8 @@ ca_iter_mark_walk_may_raise (ca_iter_state *st)
|
|
|
3378
2588
|
/* When body raises, finish st -- and the state it is paired with (see
|
|
3379
2589
|
ca_iter_state_init_l2_paired), which is open in the same frame and
|
|
3380
2590
|
would otherwise be jumped over. */
|
|
2591
|
+
static VALUE ca_iter_ensure_finish_one (VALUE arg);
|
|
2592
|
+
|
|
3381
2593
|
static void
|
|
3382
2594
|
ca_iter_protect (ca_iter_state *st, VALUE (*body)(VALUE), VALUE arg)
|
|
3383
2595
|
{
|
|
@@ -3385,10 +2597,16 @@ ca_iter_protect (ca_iter_state *st, VALUE (*body)(VALUE), VALUE arg)
|
|
|
3385
2597
|
rb_protect(body, arg, &tag);
|
|
3386
2598
|
if ( tag ) {
|
|
3387
2599
|
ca_iter_state *pair = (ca_iter_state *) st->pair;
|
|
3388
|
-
|
|
2600
|
+
volatile VALUE error = rb_errinfo();
|
|
2601
|
+
int ignored = 0;
|
|
2602
|
+
/* protect: finishes the pair even when finishing st raises; the
|
|
2603
|
+
body's exception is the one passed on */
|
|
2604
|
+
rb_protect(ca_iter_ensure_finish_one, (VALUE) st, &ignored);
|
|
3389
2605
|
if ( pair != NULL ) {
|
|
3390
|
-
|
|
2606
|
+
/* protect: as above */
|
|
2607
|
+
rb_protect(ca_iter_ensure_finish_one, (VALUE) pair, &ignored);
|
|
3391
2608
|
}
|
|
2609
|
+
rb_set_errinfo(error);
|
|
3392
2610
|
rb_jump_tag(tag);
|
|
3393
2611
|
}
|
|
3394
2612
|
}
|
|
@@ -3420,15 +2638,6 @@ ca_iter_call_next_slab (VALUE arg)
|
|
|
3420
2638
|
return Qnil;
|
|
3421
2639
|
}
|
|
3422
2640
|
|
|
3423
|
-
static VALUE
|
|
3424
|
-
ca_iter_call_next_slab_strided (VALUE arg)
|
|
3425
|
-
{
|
|
3426
|
-
ca_iter_call_t *c = (ca_iter_call_t *) arg;
|
|
3427
|
-
c->rc = ca_iter_state_next_slab_strided_unprotected(c->st, c->out_ptr,
|
|
3428
|
-
c->out_mask, c->out_n,
|
|
3429
|
-
c->out_stride_bytes);
|
|
3430
|
-
return Qnil;
|
|
3431
|
-
}
|
|
3432
2641
|
|
|
3433
2642
|
static VALUE
|
|
3434
2643
|
ca_iter_call_next_slab_axes (VALUE arg)
|
|
@@ -3520,24 +2729,6 @@ ca_iter_state_next_slab (ca_iter_state *st,
|
|
|
3520
2729
|
return ca_iter_mark_dirty(st, c.rc);
|
|
3521
2730
|
}
|
|
3522
2731
|
|
|
3523
|
-
int
|
|
3524
|
-
ca_iter_state_next_slab_strided (ca_iter_state *st,
|
|
3525
|
-
char **out_ptr,
|
|
3526
|
-
boolean8_t **out_mask,
|
|
3527
|
-
ca_size_t *out_n,
|
|
3528
|
-
ca_size_t *out_stride_bytes)
|
|
3529
|
-
{
|
|
3530
|
-
ca_iter_call_t c;
|
|
3531
|
-
if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
|
|
3532
|
-
return ca_iter_mark_dirty(st,
|
|
3533
|
-
ca_iter_state_next_slab_strided_unprotected(st, out_ptr, out_mask,
|
|
3534
|
-
out_n, out_stride_bytes));
|
|
3535
|
-
}
|
|
3536
|
-
c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.out_n = out_n;
|
|
3537
|
-
c.out_stride_bytes = out_stride_bytes; c.rc = 0;
|
|
3538
|
-
ca_iter_protect(st, ca_iter_call_next_slab_strided, (VALUE) &c);
|
|
3539
|
-
return ca_iter_mark_dirty(st, c.rc);
|
|
3540
|
-
}
|
|
3541
2732
|
|
|
3542
2733
|
int
|
|
3543
2734
|
ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
@@ -3673,15 +2864,39 @@ typedef struct {
|
|
|
3673
2864
|
ca_iter_state *states;
|
|
3674
2865
|
} ca_iter_ensure_t;
|
|
3675
2866
|
|
|
2867
|
+
static VALUE
|
|
2868
|
+
ca_iter_ensure_finish_one (VALUE arg)
|
|
2869
|
+
{
|
|
2870
|
+
ca_iter_state_finish((ca_iter_state *) arg);
|
|
2871
|
+
return Qnil;
|
|
2872
|
+
}
|
|
2873
|
+
|
|
3676
2874
|
/* Finishes the walks in reverse order of their slots, as nested windows
|
|
3677
|
-
close.
|
|
3678
|
-
|
|
2875
|
+
close. A finish can raise: on a WRITE walk whose root is a view, it
|
|
2876
|
+
syncs the root (ca_sync_detach) through the view's slots. Each finish
|
|
2877
|
+
is therefore run under rb_protect, so a raise from one does not leave
|
|
2878
|
+
the states before it unfinished; the first raise is re-raised once
|
|
2879
|
+
every state is closed. */
|
|
3679
2880
|
static VALUE
|
|
3680
2881
|
ca_iter_ensure_close (VALUE arg)
|
|
3681
2882
|
{
|
|
3682
2883
|
ca_iter_ensure_t *e = (ca_iter_ensure_t *) arg;
|
|
2884
|
+
int first = 0;
|
|
2885
|
+
volatile VALUE first_err = Qnil;
|
|
3683
2886
|
for ( int32_t i = e->n - 1; i >= 0; i-- ) {
|
|
3684
|
-
|
|
2887
|
+
int status = 0;
|
|
2888
|
+
rb_protect(ca_iter_ensure_finish_one, (VALUE) &e->states[i], &status);
|
|
2889
|
+
if ( status ) {
|
|
2890
|
+
if ( ! first ) {
|
|
2891
|
+
first = status;
|
|
2892
|
+
first_err = rb_errinfo();
|
|
2893
|
+
}
|
|
2894
|
+
rb_set_errinfo(Qnil);
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
if ( first ) {
|
|
2898
|
+
rb_set_errinfo(first_err);
|
|
2899
|
+
rb_jump_tag(first);
|
|
3685
2900
|
}
|
|
3686
2901
|
return Qnil;
|
|
3687
2902
|
}
|
|
@@ -3712,10 +2927,27 @@ ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
3712
2927
|
* or `CARRAY_DEV=1 rake build_ext`). These helpers expose internal
|
|
3713
2928
|
* engine state to Ruby for spec_ai regression pins. Do not consume
|
|
3714
2929
|
* from user code.
|
|
3715
|
-
*
|
|
3716
|
-
* See devel/PROPOSAL_SMOKE_DEV_BUILD_GATE.md
|
|
3717
2930
|
* ============================================================ */
|
|
3718
2931
|
|
|
2932
|
+
/* The innermost-axis walk the L2 smokes run: one strided run per
|
|
2933
|
+
innermost row, handed out as (pointer, cell count, byte stride). */
|
|
2934
|
+
static int
|
|
2935
|
+
t1_smoke_open_rows (ca_iter_state *st, CArray *src, uint32_t flags)
|
|
2936
|
+
{
|
|
2937
|
+
int8_t ax = (int8_t) (src->ndim - 1);
|
|
2938
|
+
return ca_iter_state_init_l2(st, src, CA_SLAB_AXES, &ax, 1, flags);
|
|
2939
|
+
}
|
|
2940
|
+
|
|
2941
|
+
static int
|
|
2942
|
+
t1_smoke_next_row (ca_iter_state *st, char **p, ca_size_t *n,
|
|
2943
|
+
ca_size_t *stride)
|
|
2944
|
+
{
|
|
2945
|
+
if ( ! ca_iter_state_next_slab_axes(st, p, NULL) ) return 0;
|
|
2946
|
+
*n = st->slab_dims[0];
|
|
2947
|
+
*stride = st->slab_strides[0];
|
|
2948
|
+
return 1;
|
|
2949
|
+
}
|
|
2950
|
+
|
|
3719
2951
|
/* ---- Ruby smoke surface --------------------------------------------- */
|
|
3720
2952
|
|
|
3721
2953
|
/* L1 smoke (steps 1+2): CArray.t1_smoke(ca) -> Hash with
|
|
@@ -3760,7 +2992,7 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
|
|
|
3760
2992
|
return result;
|
|
3761
2993
|
}
|
|
3762
2994
|
|
|
3763
|
-
/* L2 smoke
|
|
2995
|
+
/* L2 smoke: CArray.t1_smoke_strided(ca) -> Hash with
|
|
3764
2996
|
rc => Integer
|
|
3765
2997
|
slabs => Integer (= total_slabs)
|
|
3766
2998
|
total_elems => Integer
|
|
@@ -3769,8 +3001,6 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
|
|
|
3769
3001
|
reported (ptr, n, stride_bytes) tuples, in
|
|
3770
3002
|
iteration order; should equal view.to_ca.dump_binary
|
|
3771
3003
|
strides => Array<Integer> — stride_bytes per yielded slab
|
|
3772
|
-
(constant across yields under WHOLE policy in
|
|
3773
|
-
step 3; surfaced for inspection)
|
|
3774
3004
|
On error rc != OK, the walk fields are 0 / empty. */
|
|
3775
3005
|
static VALUE
|
|
3776
3006
|
rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
|
|
@@ -3789,9 +3019,9 @@ rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
|
|
|
3789
3019
|
data = rb_str_new(0, 0);
|
|
3790
3020
|
strides_arr = rb_ary_new();
|
|
3791
3021
|
|
|
3792
|
-
rc =
|
|
3022
|
+
rc = t1_smoke_open_rows(&st, src, 0);
|
|
3793
3023
|
if ( rc == CA_ITER_OK ) {
|
|
3794
|
-
while (
|
|
3024
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
|
|
3795
3025
|
if ( p != NULL && n > 0 ) {
|
|
3796
3026
|
ca_size_t i;
|
|
3797
3027
|
for ( i = 0; i < n; i++ ) {
|
|
@@ -3838,15 +3068,15 @@ rb_t1_smoke_sum_strided_f64 (VALUE klass, VALUE vsrc)
|
|
|
3838
3068
|
"t1_smoke_sum_strided_f64 expects a float64 source");
|
|
3839
3069
|
}
|
|
3840
3070
|
|
|
3841
|
-
rc =
|
|
3071
|
+
rc = t1_smoke_open_rows(&st, src, 0);
|
|
3842
3072
|
if ( rc != CA_ITER_OK ) {
|
|
3843
3073
|
rb_raise(rb_eRuntimeError, "ca_iter_state_init L2 failed (rc=%d)", rc);
|
|
3844
3074
|
}
|
|
3845
3075
|
|
|
3846
|
-
while (
|
|
3847
|
-
/*
|
|
3848
|
-
|
|
3849
|
-
|
|
3076
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
|
|
3077
|
+
/* CA_L2_FOR_EACH takes a contiguous loop when stride ==
|
|
3078
|
+
sizeof(double), which the compiler can vectorise, and a strided
|
|
3079
|
+
loop otherwise. */
|
|
3850
3080
|
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3851
3081
|
acc += *dp;
|
|
3852
3082
|
});
|
|
@@ -3887,9 +3117,8 @@ rb_t1_smoke_write_fill_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
|
3887
3117
|
return INT2NUM(CA_ITER_OK);
|
|
3888
3118
|
}
|
|
3889
3119
|
|
|
3890
|
-
/* WRITE smoke: in-place fill via L2
|
|
3891
|
-
|
|
3892
|
-
already in t1_smoke_write_fill_f64). Kernel walks the strided
|
|
3120
|
+
/* WRITE smoke: in-place fill via an L2 walk (the L1 form is
|
|
3121
|
+
t1_smoke_write_fill_f64). Kernel walks the strided
|
|
3893
3122
|
slab and writes val at every position; sync_slab scatters back. */
|
|
3894
3123
|
static VALUE
|
|
3895
3124
|
rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
@@ -3906,14 +3135,12 @@ rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
|
3906
3135
|
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3907
3136
|
}
|
|
3908
3137
|
|
|
3909
|
-
rc =
|
|
3910
|
-
CA_KERNEL_WRITE);
|
|
3138
|
+
rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
|
|
3911
3139
|
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3912
3140
|
|
|
3913
|
-
while (
|
|
3914
|
-
/*
|
|
3915
|
-
|
|
3916
|
-
hit +18-22% in step 5.5 — macro removes that overhead). */
|
|
3141
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
|
|
3142
|
+
/* Contiguous loop when stride == sizeof(double) (a gathered
|
|
3143
|
+
scratch), strided otherwise. */
|
|
3917
3144
|
CA_L2_FOR_EACH(double, p, n, stride, dp, {
|
|
3918
3145
|
*dp = v;
|
|
3919
3146
|
});
|
|
@@ -3981,13 +3208,11 @@ cmp_double (const void *a, const void *b)
|
|
|
3981
3208
|
|
|
3982
3209
|
/* WRITE smoke: per-row sort via L2 strided dispatch. For a 2D
|
|
3983
3210
|
float64 src of shape [m, n], sort each of the m rows in ascending
|
|
3984
|
-
order.
|
|
3211
|
+
order. Walks the innermost axis so each row is a strided slab; the
|
|
3985
3212
|
kernel materialises into a tight contig scratch, qsorts, then
|
|
3986
3213
|
writes back via the same stride. This exercises L2 WRITE
|
|
3987
3214
|
mechanics (multi-slab walk, per-slab fill of strided cells)
|
|
3988
|
-
without
|
|
3989
|
-
Pattern H specialised op, out of step 4 scope (reviewer advice #3).
|
|
3990
|
-
*/
|
|
3215
|
+
without a strided qsort of its own. */
|
|
3991
3216
|
static VALUE
|
|
3992
3217
|
rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
3993
3218
|
{
|
|
@@ -4003,17 +3228,16 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
|
4003
3228
|
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
4004
3229
|
}
|
|
4005
3230
|
|
|
4006
|
-
rc =
|
|
4007
|
-
CA_KERNEL_WRITE);
|
|
3231
|
+
rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
|
|
4008
3232
|
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
4009
3233
|
|
|
4010
|
-
while (
|
|
3234
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
|
|
4011
3235
|
if ( n > (ca_size_t) (sizeof(scratch) / sizeof(double)) ) {
|
|
4012
3236
|
ca_iter_state_finish(&st);
|
|
4013
3237
|
rb_raise(rb_eRuntimeError, "row too large for smoke scratch");
|
|
4014
3238
|
}
|
|
4015
|
-
/* strided -> contig scratch
|
|
4016
|
-
stride == bytes
|
|
3239
|
+
/* strided -> contig scratch (CA_L2_FOR_EACH takes the contiguous
|
|
3240
|
+
loop when stride == bytes). */
|
|
4017
3241
|
ca_size_t _sc_i = 0;
|
|
4018
3242
|
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
4019
3243
|
scratch[_sc_i++] = *dp;
|
|
@@ -4021,7 +3245,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
|
4021
3245
|
/* in-place qsort on the contig scratch */
|
|
4022
3246
|
qsort(scratch, n, sizeof(double), cmp_double);
|
|
4023
3247
|
/* contig scratch -> strided (writes back to parent via alias_ptr,
|
|
4024
|
-
|
|
3248
|
+
a direct write). */
|
|
4025
3249
|
ca_size_t _wb_i = 0;
|
|
4026
3250
|
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
4027
3251
|
*dp = scratch[_wb_i++];
|
|
@@ -4032,7 +3256,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
|
4032
3256
|
return INT2NUM(CA_ITER_OK);
|
|
4033
3257
|
}
|
|
4034
3258
|
|
|
4035
|
-
/*
|
|
3259
|
+
/* Mask smoke: L1 walk that exposes BOTH the value slab and the
|
|
4036
3260
|
mask slab. Used to pin "mask is delivered to kernel" semantics.
|
|
4037
3261
|
Returns rc + mask_seen (Boolean: was out_mask non-NULL at first
|
|
4038
3262
|
yield) + mask_bytes (String: concatenated mask bytes). */
|
|
@@ -4108,9 +3332,9 @@ rb_t1_smoke_attach (VALUE klass, VALUE vsrc)
|
|
|
4108
3332
|
return SIZE2NUM(total);
|
|
4109
3333
|
}
|
|
4110
3334
|
|
|
4111
|
-
/* L2 bench-grade smoke
|
|
4112
|
-
but
|
|
4113
|
-
exercised through the L2 path. */
|
|
3335
|
+
/* L2 bench-grade smoke. Same shape as t1_smoke_attach
|
|
3336
|
+
but walks the innermost axis through init_l2 so descriptor sources
|
|
3337
|
+
can be exercised through the L2 path. */
|
|
4114
3338
|
static VALUE
|
|
4115
3339
|
rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
4116
3340
|
{
|
|
@@ -4123,10 +3347,10 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
|
4123
3347
|
volatile char sink = 0;
|
|
4124
3348
|
|
|
4125
3349
|
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
4126
|
-
rc =
|
|
3350
|
+
rc = t1_smoke_open_rows(&st, src, 0);
|
|
4127
3351
|
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
4128
3352
|
|
|
4129
|
-
while (
|
|
3353
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
|
|
4130
3354
|
if ( p != NULL && n > 0 ) sink ^= p[0];
|
|
4131
3355
|
total += n;
|
|
4132
3356
|
}
|
|
@@ -4135,7 +3359,7 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
|
4135
3359
|
return SIZE2NUM(total);
|
|
4136
3360
|
}
|
|
4137
3361
|
|
|
4138
|
-
/*
|
|
3362
|
+
/* Flags smoke: minimal smoke that exposes the `flags` argument to Ruby
|
|
4139
3363
|
tests. Just runs init_l1 with the requested flags and returns
|
|
4140
3364
|
the rc — used to verify NO_MASK enforcement (and any future
|
|
4141
3365
|
flag-gated rejection paths) without needing a full kernel walk. */
|
|
@@ -4145,6 +3369,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
|
|
|
4145
3369
|
CArray *src;
|
|
4146
3370
|
ca_iter_state st;
|
|
4147
3371
|
int rc;
|
|
3372
|
+
/* fits: a development smoke argument, not a cell */
|
|
4148
3373
|
uint32_t flags = (uint32_t) NUM2UINT(vflags);
|
|
4149
3374
|
|
|
4150
3375
|
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
@@ -4153,7 +3378,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
|
|
|
4153
3378
|
return INT2NUM(rc);
|
|
4154
3379
|
}
|
|
4155
3380
|
|
|
4156
|
-
/* ----
|
|
3381
|
+
/* ---- CA_SLAB_AXES smoke ----------------------------------------------
|
|
4157
3382
|
sum a float64 source over user-specified slab axes via init_l2 +
|
|
4158
3383
|
next_slab_axes. K-D walk over slab dims using slab_strides for data
|
|
4159
3384
|
and slab_mask_strides for mask (so non-contig CAStride sources work
|
|
@@ -4183,10 +3408,11 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
4183
3408
|
rb_raise(rb_eTypeError, "t1_smoke_sum_axes_f64 expects a float64 source");
|
|
4184
3409
|
}
|
|
4185
3410
|
naxes = (int8_t) (argc - 1);
|
|
4186
|
-
if ( naxes > CA_RANK_MAX ) {
|
|
3411
|
+
if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
|
|
4187
3412
|
rb_raise(rb_eArgError, "too many axes");
|
|
4188
3413
|
}
|
|
4189
3414
|
for ( i = 0; i < naxes; i++ ) {
|
|
3415
|
+
/* fits: an axis number, checked by the engine's init */
|
|
4190
3416
|
slab_axes[i] = (int8_t) NUM2INT(argv[i + 1]);
|
|
4191
3417
|
}
|
|
4192
3418
|
|
|
@@ -4222,7 +3448,7 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
4222
3448
|
return DBL2NUM(acc);
|
|
4223
3449
|
}
|
|
4224
3450
|
|
|
4225
|
-
/* ----
|
|
3451
|
+
/* ---- helper smoke: rb_ca_parse_reduce_axes ---------------------------
|
|
4226
3452
|
Calls the helper and returns the parsed axes as a Ruby Array of
|
|
4227
3453
|
Integer. Lets tests inspect parsing behaviour directly.
|
|
4228
3454
|
Signature: CArray.t1_test_parse_reduce_axes(ca, *args) */
|
|
@@ -4248,7 +3474,7 @@ rb_t1_test_parse_reduce_axes (int argc, VALUE *argv, VALUE klass)
|
|
|
4248
3474
|
return arr;
|
|
4249
3475
|
}
|
|
4250
3476
|
|
|
4251
|
-
/* ----
|
|
3477
|
+
/* ---- helper smoke: rb_ca_parse_reduce_axes_kw ------------------------
|
|
4252
3478
|
Calls the kwarg helper and returns the parsed axes as a Ruby Array.
|
|
4253
3479
|
Lets tests inspect kwarg-form parsing (Qnil / Integer / Array) +
|
|
4254
3480
|
validation parity with the variadic entry.
|
|
@@ -4276,7 +3502,7 @@ rb_test_parse_reduce_axes_kw (int argc, VALUE *argv, VALUE klass)
|
|
|
4276
3502
|
return arr;
|
|
4277
3503
|
}
|
|
4278
3504
|
|
|
4279
|
-
/* ----
|
|
3505
|
+
/* ---- helper smoke: rb_ca_new_reduced ---------------------------------
|
|
4280
3506
|
Exposes rb_ca_new_reduced (carray_core.c) for unit testing. Returns
|
|
4281
3507
|
the newly allocated output CArray (= tests verify shape + data_type).
|
|
4282
3508
|
Signature: CArray.t1_test_new_reduced(ca, data_type, axis1, axis2, ...) */
|
|
@@ -4291,21 +3517,21 @@ rb_t1_test_new_reduced (int argc, VALUE *argv, VALUE klass)
|
|
|
4291
3517
|
rb_raise(rb_eArgError,
|
|
4292
3518
|
"t1_test_new_reduced expects (ca, data_type, axis...) with >= 1 axis");
|
|
4293
3519
|
}
|
|
4294
|
-
/*
|
|
4295
|
-
|
|
4296
|
-
and legacy Integer-code callers both work. */
|
|
3520
|
+
/* Accept Symbol / Integer / Class / String alike through
|
|
3521
|
+
rb_ca_guess_type. */
|
|
4297
3522
|
data_type = (int32_t) rb_ca_guess_type(argv[1]);
|
|
4298
3523
|
naxes = (int8_t) (argc - 2);
|
|
4299
|
-
if ( naxes > CA_RANK_MAX ) {
|
|
3524
|
+
if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
|
|
4300
3525
|
rb_raise(rb_eArgError, "too many axes");
|
|
4301
3526
|
}
|
|
4302
3527
|
for ( int i = 0; i < naxes; i++ ) {
|
|
3528
|
+
/* fits: an axis number, checked by the engine's init */
|
|
4303
3529
|
axes[i] = (int8_t) NUM2INT(argv[2 + i]);
|
|
4304
3530
|
}
|
|
4305
3531
|
return rb_ca_new_reduced(argv[0], axes, naxes, data_type, 0);
|
|
4306
3532
|
}
|
|
4307
3533
|
|
|
4308
|
-
/* ----
|
|
3534
|
+
/* ---- CAReduce smoke --------------------------------------------------
|
|
4309
3535
|
CAReduce has no public Ruby ctor (it's an internal class used by
|
|
4310
3536
|
CARefer mask handling). Tests need to drive kernel_iterator over
|
|
4311
3537
|
a CAReduce, so we expose a thin construction helper plus dedicated
|
|
@@ -4370,8 +3596,9 @@ rb_t1_smoke_reduce_read (VALUE klass, VALUE vparent, VALUE vcount, VALUE voffset
|
|
|
4370
3596
|
}
|
|
4371
3597
|
|
|
4372
3598
|
/* WRITE broadcast smoke for CAReduce: fills the reduce view with
|
|
4373
|
-
`val` (boolean / 0 or 1) and lets sync_slab
|
|
4374
|
-
|
|
3599
|
+
`val` (boolean / 0 or 1) and lets sync_slab send the changed cells
|
|
3600
|
+
back through the view (ki_scratch_push_changed), which broadcasts them
|
|
3601
|
+
to the parent. After the call parent's
|
|
4375
3602
|
[offset..offset+elems*count) bytes should all equal val. Returns
|
|
4376
3603
|
rc; on rc != OK parent is not modified. */
|
|
4377
3604
|
static VALUE
|
|
@@ -4384,6 +3611,7 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
|
|
|
4384
3611
|
ca_size_t n, i;
|
|
4385
3612
|
int rc;
|
|
4386
3613
|
CArray *reduce;
|
|
3614
|
+
/* fits: a development smoke argument, not a cell */
|
|
4387
3615
|
uint8_t v = (uint8_t) NUM2UINT(vval);
|
|
4388
3616
|
|
|
4389
3617
|
TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
|
|
@@ -4399,15 +3627,15 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
|
|
|
4399
3627
|
}
|
|
4400
3628
|
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
4401
3629
|
for ( i = 0; i < n; i++ ) p[i] = v;
|
|
4402
|
-
ca_iter_state_sync_slab(&st); /*
|
|
3630
|
+
ca_iter_state_sync_slab(&st); /* sends the changed cells back */
|
|
4403
3631
|
}
|
|
4404
3632
|
ca_iter_state_finish(&st);
|
|
4405
3633
|
ca_free(reduce);
|
|
4406
3634
|
return INT2NUM(CA_ITER_OK);
|
|
4407
3635
|
}
|
|
4408
3636
|
|
|
4409
|
-
/* ----
|
|
4410
|
-
CARemap has no public Ruby
|
|
3637
|
+
/* ---- CARemap smoke helpers --------------------------------------------
|
|
3638
|
+
CARemap has no public Ruby constructor, so we
|
|
4411
3639
|
construct it from C using the (ref, idx) args. Pattern mirrors
|
|
4412
3640
|
rb_t1_smoke_reduce_*: build the view, run the iterator state
|
|
4413
3641
|
machine, return the standard hash (rc / slabs / total_elems /
|
|
@@ -4474,10 +3702,10 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
|
|
|
4474
3702
|
view = ca_remap_new(ref, idx);
|
|
4475
3703
|
data = rb_str_new(0, 0);
|
|
4476
3704
|
|
|
4477
|
-
rc =
|
|
3705
|
+
rc = t1_smoke_open_rows(&st, view, 0);
|
|
4478
3706
|
if ( rc == CA_ITER_OK ) {
|
|
4479
3707
|
ca_size_t stride;
|
|
4480
|
-
while (
|
|
3708
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
|
|
4481
3709
|
ca_size_t i;
|
|
4482
3710
|
for ( i = 0; i < n; i++ ) {
|
|
4483
3711
|
rb_str_cat(data, p + i * stride, st.bytes);
|
|
@@ -4500,7 +3728,8 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
|
|
|
4500
3728
|
}
|
|
4501
3729
|
|
|
4502
3730
|
/* WRITE smoke: fill view's slab buffer with `val` (float64), let
|
|
4503
|
-
sync_slab
|
|
3731
|
+
sync_slab send the changed cells back to ref through the view
|
|
3732
|
+
(ki_scratch_push_changed: ca_xfer_addrs or ca_xfer_all PUT). */
|
|
4504
3733
|
static VALUE
|
|
4505
3734
|
rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
|
|
4506
3735
|
VALUE vval)
|
|
@@ -4533,17 +3762,17 @@ rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
|
|
|
4533
3762
|
return INT2NUM(CA_ITER_OK);
|
|
4534
3763
|
}
|
|
4535
3764
|
|
|
4536
|
-
/* ----
|
|
3765
|
+
/* ---- CA_FOR_EACH_SLAB macro smoke surfaces ---------------------------
|
|
4537
3766
|
Smoke kernels that exercise the block macros end-to-end. Used by
|
|
4538
3767
|
spec_ai/test_ca_for_each_slab_macros.rb to pin behavioural
|
|
4539
3768
|
correctness of the macro expansion (= same byte-parity result as
|
|
4540
3769
|
the raw API equivalent). */
|
|
4541
3770
|
|
|
4542
|
-
/* sum reduction using CA_FOR_EACH_SLAB
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
3771
|
+
/* sum reduction using CA_FOR_EACH_SLAB over the slab axes given in
|
|
3772
|
+
argv[1..] (K-D slabs). Reads the source as float64 without checking
|
|
3773
|
+
its data_type. Accumulator = scalar (single full-reduction slab when
|
|
3774
|
+
naxes == src->ndim, otherwise per-outer-slab accumulator written to a
|
|
3775
|
+
row-major output buffer). */
|
|
4547
3776
|
static VALUE
|
|
4548
3777
|
rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
4549
3778
|
{
|
|
@@ -4562,6 +3791,7 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
4562
3791
|
rb_raise(rb_eArgError, "bad axes count");
|
|
4563
3792
|
}
|
|
4564
3793
|
for ( int8_t k = 0; k < naxes; k++ ) {
|
|
3794
|
+
/* fits: an axis number, checked by the engine's init */
|
|
4565
3795
|
slab_axes[k] = (int8_t) NUM2INT(argv[1 + k]);
|
|
4566
3796
|
if ( slab_axes[k] < 0 ) slab_axes[k] += ca->ndim;
|
|
4567
3797
|
}
|
|
@@ -4631,8 +3861,8 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
4631
3861
|
|
|
4632
3862
|
/* map kernel using CA_FOR_EACH_SLAB_INOUT. Doubles each element of
|
|
4633
3863
|
a float64 source into a fresh same-shape output. Slab axis is
|
|
4634
|
-
always the innermost (
|
|
4635
|
-
|
|
3864
|
+
always the innermost (ca->ndim - 1); there is no axis argument.
|
|
3865
|
+
Demonstrates the 2-iter pattern. */
|
|
4636
3866
|
static VALUE
|
|
4637
3867
|
rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
|
|
4638
3868
|
{
|
|
@@ -4676,11 +3906,11 @@ rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
|
|
|
4676
3906
|
return vout;
|
|
4677
3907
|
}
|
|
4678
3908
|
|
|
4679
|
-
/* ----
|
|
3909
|
+
/* ---- CA_FOR_EACH_FIBER macro smokes ----------------------------------
|
|
4680
3910
|
Exercise each of the 4 catalog forms end-to-end with float64 fibers,
|
|
4681
3911
|
verifying contig delivery for both innermost (= stride==bytes) and
|
|
4682
3912
|
non-innermost (= stride>bytes, gather path) axis positions. Used by
|
|
4683
|
-
spec_ai/test_fiber_delivery.rb
|
|
3913
|
+
spec_ai/test_fiber_delivery.rb. */
|
|
4684
3914
|
|
|
4685
3915
|
/* form 1 (NO_MASK single): per-axis sum, returns total sum.
|
|
4686
3916
|
Catalog: CA_FOR_EACH_FIBER. */
|
|
@@ -4706,8 +3936,7 @@ rb_caf_fiber_smoke_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4706
3936
|
}
|
|
4707
3937
|
|
|
4708
3938
|
/* form 2 (NO_MASK INOUT): per-axis fiber copy * scalar.
|
|
4709
|
-
Catalog: CA_FOR_EACH_FIBER_INOUT.
|
|
4710
|
-
gather/scatter behavior. */
|
|
3939
|
+
Catalog: CA_FOR_EACH_FIBER_INOUT. Doubles each cell of each fiber. */
|
|
4711
3940
|
static VALUE
|
|
4712
3941
|
rb_caf_fiber_smoke_double_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
4713
3942
|
{
|
|
@@ -4791,9 +4020,9 @@ rb_caf_fiber_smoke_zero_masked_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4791
4020
|
return vout;
|
|
4792
4021
|
}
|
|
4793
4022
|
|
|
4794
|
-
/*
|
|
4795
|
-
Functional equivalent of
|
|
4796
|
-
|
|
4023
|
+
/* Bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
|
|
4024
|
+
Functional equivalent of the per-axis sort_copy (rb_ca_sort_copy in
|
|
4025
|
+
ext/carray_sort.c).
|
|
4797
4026
|
The macro form lets us A/B the author-side overhead of stride math
|
|
4798
4027
|
vs the catalog-macro-driven engine gather/scatter. */
|
|
4799
4028
|
static int
|
|
@@ -4836,7 +4065,7 @@ rb_caf_fiber_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4836
4065
|
return vout;
|
|
4837
4066
|
}
|
|
4838
4067
|
|
|
4839
|
-
/*
|
|
4068
|
+
/* Bench helper variant: sort_copy via SLAB macro + manual
|
|
4840
4069
|
gather/scatter (= what hand-rolled sort_copy effectively does, but
|
|
4841
4070
|
driven through the public CA_FOR_EACH_SLAB_INOUT macro instead of
|
|
4842
4071
|
the raw next_slab_axes API). Lets us isolate macro overhead from
|
|
@@ -4902,7 +4131,7 @@ rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4902
4131
|
return vout;
|
|
4903
4132
|
}
|
|
4904
4133
|
|
|
4905
|
-
/*
|
|
4134
|
+
/* Bench helper: per-fiber fused xfer_stride direct into a contig
|
|
4906
4135
|
scratch, skipping the kernel_iterator SRC_ATTACH whole-view materialise.
|
|
4907
4136
|
Tests user's hypothesis: for transform views (CAFake/CAByteSwap etc.),
|
|
4908
4137
|
does bypassing the whole-view scratch_ptr and calling ca_xfer_stride
|
|
@@ -4981,6 +4210,14 @@ rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4981
4210
|
return rb_float_new(total);
|
|
4982
4211
|
}
|
|
4983
4212
|
|
|
4213
|
+
/* CArray.__iter_buffers__: see ki_buffers_out. */
|
|
4214
|
+
static VALUE
|
|
4215
|
+
rb_ki_buffers_out (VALUE klass)
|
|
4216
|
+
{
|
|
4217
|
+
(void) klass;
|
|
4218
|
+
return LONG2NUM(__atomic_load_n(&ki_buffers_out, __ATOMIC_RELAXED));
|
|
4219
|
+
}
|
|
4220
|
+
|
|
4984
4221
|
#endif /* CARRAY_DEV_BUILD — end smoke surface fence */
|
|
4985
4222
|
|
|
4986
4223
|
void
|
|
@@ -4990,10 +4227,12 @@ Init_ca_kernel_iterator (void)
|
|
|
4990
4227
|
|
|
4991
4228
|
#ifdef CARRAY_DEV_BUILD
|
|
4992
4229
|
/* ==== smoke surface registrations (dev-only, stripped in release) ====
|
|
4993
|
-
*
|
|
4230
|
+
* All `t1_smoke_*`, `caf_smoke_*`,
|
|
4994
4231
|
* `caf_fiber_smoke_*`, `caf_*_bench_*`, helper smokes, and `T1_*` test
|
|
4995
4232
|
* constants are gated here together — none of them are consumed by
|
|
4996
4233
|
* production code (lib/ / other ext/), only by spec_ai regression pins. */
|
|
4234
|
+
rb_define_singleton_method(rb_cCArray, "__iter_buffers__",
|
|
4235
|
+
rb_ki_buffers_out, 0);
|
|
4997
4236
|
rb_define_singleton_method(rb_cCArray, "t1_smoke",
|
|
4998
4237
|
rb_t1_smoke, 1);
|
|
4999
4238
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_strided",
|
|
@@ -5016,20 +4255,20 @@ Init_ca_kernel_iterator (void)
|
|
|
5016
4255
|
rb_t1_smoke_with_mask, 1);
|
|
5017
4256
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_init_rc",
|
|
5018
4257
|
rb_t1_smoke_init_rc, 2);
|
|
5019
|
-
/*
|
|
4258
|
+
/* CA_SLAB_AXES smoke (variadic axes) */
|
|
5020
4259
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_sum_axes_f64",
|
|
5021
4260
|
rb_t1_smoke_sum_axes_f64, -1);
|
|
5022
|
-
/*
|
|
4261
|
+
/* rb_ca_new_reduced helper smoke */
|
|
5023
4262
|
rb_define_singleton_method(rb_cCArray, "t1_test_new_reduced",
|
|
5024
4263
|
rb_t1_test_new_reduced, -1);
|
|
5025
|
-
/*
|
|
4264
|
+
/* rb_ca_parse_reduce_axes helper smoke */
|
|
5026
4265
|
rb_define_singleton_method(rb_cCArray, "t1_test_parse_reduce_axes",
|
|
5027
4266
|
rb_t1_test_parse_reduce_axes, -1);
|
|
5028
4267
|
|
|
5029
|
-
/*
|
|
4268
|
+
/* rb_ca_parse_reduce_axes_kw helper smoke */
|
|
5030
4269
|
rb_define_singleton_method(rb_cCArray, "test_parse_reduce_axes_kw",
|
|
5031
4270
|
rb_test_parse_reduce_axes_kw, -1);
|
|
5032
|
-
/* CAReduce
|
|
4271
|
+
/* CAReduce: no public Ruby API */
|
|
5033
4272
|
rb_define_singleton_method(rb_cCArray, "t1_make_reduce",
|
|
5034
4273
|
rb_t1_make_reduce, 3);
|
|
5035
4274
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_read",
|
|
@@ -5037,19 +4276,19 @@ Init_ca_kernel_iterator (void)
|
|
|
5037
4276
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_write_broadcast",
|
|
5038
4277
|
rb_t1_smoke_reduce_write_broadcast, 4);
|
|
5039
4278
|
|
|
5040
|
-
/*
|
|
4279
|
+
/* CARemap smokes (test-only). */
|
|
5041
4280
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read",
|
|
5042
4281
|
rb_t1_smoke_remap_read, 2);
|
|
5043
4282
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read_strided",
|
|
5044
4283
|
rb_t1_smoke_remap_read_strided, 2);
|
|
5045
4284
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_write_fill_f64",
|
|
5046
4285
|
rb_t1_smoke_remap_write_fill_f64, 3);
|
|
5047
|
-
/*
|
|
4286
|
+
/* block macro smokes */
|
|
5048
4287
|
rb_define_singleton_method(rb_cCArray, "caf_smoke_sum_f64",
|
|
5049
4288
|
rb_caf_smoke_sum_f64, -1);
|
|
5050
4289
|
rb_define_singleton_method(rb_cCArray, "caf_smoke_double_f64",
|
|
5051
4290
|
rb_caf_smoke_double_f64, 1);
|
|
5052
|
-
/*
|
|
4291
|
+
/* fiber macro smokes */
|
|
5053
4292
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_sum_f64",
|
|
5054
4293
|
rb_caf_fiber_smoke_sum_f64, 2);
|
|
5055
4294
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_double_f64",
|
|
@@ -5058,7 +4297,7 @@ Init_ca_kernel_iterator (void)
|
|
|
5058
4297
|
rb_caf_fiber_smoke_unmasked_sum_f64, 2);
|
|
5059
4298
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_zero_masked_f64",
|
|
5060
4299
|
rb_caf_fiber_smoke_zero_masked_f64, 2);
|
|
5061
|
-
/*
|
|
4300
|
+
/* bench helper */
|
|
5062
4301
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_bench_sort_copy_f64",
|
|
5063
4302
|
rb_caf_fiber_bench_sort_copy_f64, 2);
|
|
5064
4303
|
rb_define_singleton_method(rb_cCArray, "caf_slab_bench_sort_copy_f64",
|
|
@@ -5078,5 +4317,6 @@ Init_ca_kernel_iterator (void)
|
|
|
5078
4317
|
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_CONTIG", INT2NUM(CA_ITER_ALIAS_CONTIG));
|
|
5079
4318
|
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_STRIDED", INT2NUM(CA_ITER_ALIAS_STRIDED));
|
|
5080
4319
|
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_ATTACH", INT2NUM(CA_ITER_ALIAS_ATTACH));
|
|
4320
|
+
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_PER_SLAB", INT2NUM(CA_ITER_ALIAS_PER_SLAB));
|
|
5081
4321
|
#endif /* CARRAY_DEV_BUILD */
|
|
5082
4322
|
}
|