carray 3.0.2 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +568 -0
- data/README.md +6 -6
- data/carray.gemspec +1 -1
- data/ext/ca_axis_descriptor.h +27 -24
- data/ext/ca_axis_dispatch.c +220 -0
- data/ext/ca_axis_group.c +201 -123
- data/ext/ca_binop_dispatch.c +46 -52
- data/ext/ca_binop_dispatch.h +1 -1
- data/ext/ca_categorical_iterator.c +249 -183
- data/ext/ca_compare.c +7 -12
- data/ext/ca_for_buffer.h +21 -68
- data/ext/ca_group_iter.c +38 -23
- data/ext/ca_kernel_iterator.c +1765 -2076
- data/ext/ca_kernel_iterator.h +584 -566
- data/ext/ca_monop_dispatch.c +86 -63
- data/ext/ca_monop_dispatch.h +5 -0
- data/ext/ca_obj_array.c +223 -56
- data/ext/ca_obj_bincmp.c +172 -138
- data/ext/ca_obj_binop.c +116 -230
- data/ext/ca_obj_bitarray.c +40 -26
- data/ext/ca_obj_bitfield.c +100 -68
- data/ext/ca_obj_block.c +19 -9
- data/ext/ca_obj_byte_swap.c +45 -46
- data/ext/ca_obj_const_string.c +269 -44
- data/ext/ca_obj_face.c +612 -89
- data/ext/ca_obj_face.h +111 -47
- data/ext/ca_obj_fake.c +36 -79
- data/ext/ca_obj_farray.c +7 -3
- data/ext/ca_obj_field.c +33 -8
- data/ext/ca_obj_fixlen_string.c +23 -24
- data/ext/ca_obj_grid.c +541 -210
- data/ext/ca_obj_meld.c +331 -111
- data/ext/ca_obj_moncmp.c +80 -65
- data/ext/ca_obj_monop.c +177 -149
- data/ext/ca_obj_object.c +664 -248
- data/ext/ca_obj_record.c +63 -23
- data/ext/ca_obj_reduce.c +18 -16
- data/ext/ca_obj_refer.c +140 -58
- data/ext/ca_obj_remap.c +85 -24
- data/ext/ca_obj_repeat.c +24 -9
- data/ext/ca_obj_roll.c +121 -16
- data/ext/ca_obj_select.c +61 -58
- data/ext/ca_obj_select_axis.c +73 -107
- data/ext/ca_obj_shift.c +32 -102
- data/ext/ca_obj_stack.c +372 -122
- data/ext/ca_obj_stride.c +348 -140
- data/ext/ca_obj_string.c +88 -22
- data/ext/ca_obj_tile.c +38 -21
- data/ext/ca_obj_time.c +45 -31
- data/ext/ca_obj_timedelta.c +43 -29
- data/ext/ca_obj_transpose.c +17 -6
- data/ext/ca_obj_triop.c +125 -141
- data/ext/ca_obj_window.c +330 -44
- data/ext/ca_op_ipower.c +24 -11
- data/ext/ca_op_powi.h +57 -30
- data/ext/ca_pow5_table.h +656 -0
- data/ext/ca_sort_kernels.h +70 -0
- data/ext/ca_sweep_engine.c +104 -81
- data/ext/ca_sweep_engine.h +7 -20
- data/ext/ca_triop_dispatch.c +4 -1
- data/ext/ca_triop_dispatch.h +5 -2
- data/ext/caframe_csv_reader.c +542 -0
- data/ext/carray.h +241 -92
- data/ext/carray_access.c +608 -276
- data/ext/carray_address_basis.c +494 -97
- data/ext/carray_attribute.c +129 -4
- data/ext/carray_bincount.c +39 -7
- data/ext/carray_broadcast.c +56 -19
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_call_cfunc.c +108 -28
- data/ext/carray_cast.c +1576 -223
- data/ext/carray_cast_func.rb +77 -24
- data/ext/carray_class.c +4 -3
- data/ext/carray_conversion.c +208 -75
- data/ext/carray_copy.c +82 -24
- data/ext/carray_core.c +646 -162
- data/ext/carray_count.c +171 -88
- data/ext/carray_element.c +94 -60
- data/ext/carray_factorize.c +670 -324
- data/ext/carray_generate.c +260 -83
- data/ext/carray_histogram.c +209 -106
- data/ext/carray_hold.c +14 -16
- data/ext/carray_index_classifier.c +422 -243
- data/ext/carray_internal.h +692 -3
- data/ext/carray_kernels_bincmp.c +7 -0
- data/ext/carray_kernels_binop.c +487 -120
- data/ext/carray_kernels_init.c +653 -50
- data/ext/carray_kernels_moncmp.c +15 -8
- data/ext/carray_kernels_monop.c +99 -43
- data/ext/carray_kernels_reduce_aggregate.c +3925 -6112
- data/ext/carray_kernels_reduce_boolean.c +112 -36
- data/ext/carray_kernels_reduce_cumulative.c +2127 -6903
- data/ext/carray_kernels_reduce_extreme.c +3032 -2892
- data/ext/carray_kernels_reduce_variance.c +439 -500
- data/ext/carray_kernels_scan.c +1097 -488
- data/ext/carray_kernels_search.c +3787 -2176
- data/ext/carray_kernels_sort.c +1200 -920
- data/ext/carray_kernels_triop.c +479 -7
- data/ext/carray_key_windows.c +216 -0
- data/ext/carray_lazy.c +967 -153
- data/ext/carray_loop.c +81 -65
- data/ext/carray_mask.c +910 -324
- data/ext/carray_math_kernel.h +1 -0
- data/ext/carray_median_percentile.c +299 -100
- data/ext/carray_memory_view.c +872 -758
- data/ext/carray_operator.c +1171 -364
- data/ext/carray_order.c +505 -133
- data/ext/carray_partition.c +112 -65
- data/ext/carray_random.c +327 -239
- data/ext/carray_round.c +273 -0
- data/ext/carray_scatter.c +346 -78
- data/ext/carray_segment.c +257 -0
- data/ext/carray_slab.c +249 -86
- data/ext/carray_sort.c +224 -227
- data/ext/carray_sort_kernel.c +8 -1
- data/ext/carray_struct.c +118 -39
- data/ext/carray_test.c +193 -69
- data/ext/carray_time_text.c +1182 -0
- data/ext/carray_undef.c +4 -0
- data/ext/carray_utils.c +579 -113
- data/ext/extconf.rb +27 -2
- data/ext/mk_call_cfunc.rb +19 -4
- data/ext/mk_pow5_table.rb +35 -0
- data/ext/mkkernel.rb +1202 -1014
- data/ext/ruby_carray.c +17 -5
- data/ext/version.h +4 -4
- data/ext/version.rb +4 -2
- data/lib/carray/attribute.rb +160 -17
- data/lib/carray/autoload_carray.rb +23 -27
- data/lib/carray/autoload_method_extension.rb +20 -5
- data/lib/carray/axis_group.rb +360 -63
- data/lib/carray/basics.rb +162 -28
- data/lib/carray/bincount_nd.rb +18 -2
- data/lib/carray/block_iterator.rb +231 -58
- data/lib/carray/boolean_reduce.rb +34 -42
- data/lib/carray/categorical.rb +242 -52
- data/lib/carray/categorical_iterator.rb +437 -827
- data/lib/carray/complex.rb +41 -34
- data/lib/carray/conditional.rb +150 -14
- data/lib/carray/const_string.rb +59 -4
- data/lib/carray/construct.rb +15 -11
- data/lib/carray/data_type_extension.rb +1 -0
- data/lib/carray/distance_limit.rb +77 -0
- data/lib/carray/fixlen_string.rb +7 -1
- data/lib/carray/frame/column_verbs.rb +70 -0
- data/lib/carray/frame/concat.rb +157 -6
- data/lib/carray/frame/convert.rb +82 -7
- data/lib/carray/frame/csv_parser.rb +688 -87
- data/lib/carray/frame/describe.rb +80 -0
- data/lib/carray/frame/frame.rb +369 -45
- data/lib/carray/frame/group.rb +631 -90
- data/lib/carray/frame/io.rb +506 -79
- data/lib/carray/frame/join.rb +411 -62
- data/lib/carray/frame/records.rb +116 -19
- data/lib/carray/frame/reshape.rb +455 -0
- data/lib/carray/frame/sort.rb +30 -18
- data/lib/carray/frame/verbs.rb +868 -61
- data/lib/carray/frame/windows.rb +117 -0
- data/lib/carray/frame.rb +4 -0
- data/lib/carray/fuse_source.rb +118 -2
- data/lib/carray/fusion.rb +257 -42
- data/lib/carray/histogram.rb +6 -1
- data/lib/carray/inspect.rb +54 -13
- data/lib/carray/iterator.rb +99 -9
- data/lib/carray/key_windows.rb +620 -0
- data/lib/carray/lazy.rb +216 -193
- data/lib/carray/mask_gap_fill.rb +112 -92
- data/lib/carray/math.rb +32 -19
- data/lib/carray/meld_reduce.rb +52 -21
- data/lib/carray/methods/align_addr.rb +8 -4
- data/lib/carray/methods/bin.rb +26 -18
- data/lib/carray/methods/bincount.rb +18 -2
- data/lib/carray/methods/bit_string.rb +15 -4
- data/lib/carray/methods/choose.rb +16 -3
- data/lib/carray/methods/composition.rb +18 -10
- data/lib/carray/methods/factorize.rb +2 -2
- data/lib/carray/methods/gather_nd.rb +25 -5
- data/lib/carray/methods/insert_block.rb +9 -14
- data/lib/carray/methods/is_between.rb +59 -0
- data/lib/carray/methods/is_in.rb +140 -10
- data/lib/carray/methods/join.rb +2 -7
- data/lib/carray/methods/lag.rb +49 -0
- data/lib/carray/methods/locate_addr.rb +182 -45
- data/lib/carray/methods/mode.rb +18 -1
- data/lib/carray/methods/nunique.rb +11 -6
- data/lib/carray/methods/repeat.rb +1 -20
- data/lib/carray/methods/resize.rb +21 -23
- data/lib/carray/methods/snap.rb +34 -12
- data/lib/carray/methods/unique.rb +11 -2
- data/lib/carray/methods/value_counts.rb +4 -11
- data/lib/carray/record_order.rb +208 -0
- data/lib/carray/rng.rb +3 -0
- data/lib/carray/runtime.rb +65 -17
- data/lib/carray/segment_iterator.rb +1221 -0
- data/lib/carray/serialize.rb +214 -30
- data/lib/carray/slab_iterator.rb +101 -27
- data/lib/carray/stack.rb +10 -4
- data/lib/carray/string.rb +105 -5
- data/lib/carray/string_operation_extension.rb +12 -4
- data/lib/carray/struct.rb +137 -58
- data/lib/carray/struct_builder.rb +124 -17
- data/lib/carray/time.rb +1222 -186
- data/lib/carray/window_iterator.rb +242 -81
- data/lib/carray.rb +9 -2
- data/yard-stubs/ca_obj_array.rb +4 -4
- data/yard-stubs/ca_obj_bitarray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +23 -12
- data/yard-stubs/ca_obj_fake.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +5 -1
- data/yard-stubs/ca_obj_grid.rb +21 -10
- data/yard-stubs/ca_obj_refer.rb +5 -4
- data/yard-stubs/ca_obj_roll.rb +2 -3
- data/yard-stubs/ca_obj_shift.rb +5 -1
- data/yard-stubs/ca_obj_stride.rb +10 -7
- data/yard-stubs/ca_obj_tile.rb +1 -1
- data/yard-stubs/ca_obj_transpose.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +9 -5
- data/yard-stubs/carray_access.rb +11 -5
- data/yard-stubs/carray_attribute.rb +0 -5
- data/yard-stubs/carray_broadcast.rb +1 -1
- data/yard-stubs/carray_cast.rb +88 -43
- data/yard-stubs/carray_conversion.rb +15 -2
- data/yard-stubs/carray_copy.rb +12 -5
- data/yard-stubs/carray_core.rb +3 -3
- data/yard-stubs/carray_count.rb +9 -0
- data/yard-stubs/carray_element.rb +10 -4
- data/yard-stubs/carray_generate.rb +21 -1
- data/yard-stubs/carray_lazy.rb +5 -1
- data/yard-stubs/carray_loop.rb +4 -1
- data/yard-stubs/carray_mask.rb +60 -12
- data/yard-stubs/carray_math.rb +52 -14
- data/yard-stubs/carray_median_percentile.rb +15 -8
- data/yard-stubs/carray_memory_view.rb +40 -23
- data/yard-stubs/carray_order.rb +61 -29
- data/yard-stubs/carray_random.rb +13 -4
- data/yard-stubs/carray_scatter.rb +21 -9
- data/yard-stubs/carray_segment.rb +72 -0
- data/yard-stubs/carray_slab.rb +10 -2
- data/yard-stubs/carray_sort.rb +49 -8
- data/yard-stubs/carray_test.rb +15 -3
- data/yard-stubs/carray_utils.rb +0 -26
- metadata +19 -5
- data/ext/ca_for_each_element.h +0 -239
- data/ext/carray_kernels_map.c +0 -3467
- data/lib/carray/arrow_tensor.rb +0 -401
- data/lib/carray/table.rb +0 -108
data/ext/ca_kernel_iterator.c
CHANGED
|
@@ -1,61 +1,205 @@
|
|
|
1
1
|
/* ---------------------------------------------------------------------------
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
-
|
|
3
|
+
ca_kernel_iterator.c -- the kernel iterator engine
|
|
4
|
+
|
|
5
|
+
A C kernel asks for the cells of an array, one slab at a time, and gets
|
|
6
|
+
them whatever the array is: an entity, a view of a view, a selection, a
|
|
7
|
+
lazy expression. This file decides, for each source, how to deliver
|
|
8
|
+
them (borrow its memory, gather a copy, walk a stack's parents) and, on
|
|
9
|
+
a write, how the cells the kernel changed get back to the array.
|
|
10
|
+
|
|
11
|
+
Where to read first:
|
|
12
|
+
- writing a kernel: guides/devel/11_kernel_iterator.md and
|
|
13
|
+
docs/authoring/HOW_TO_WRITE_KERNEL.md; the surface itself is
|
|
14
|
+
ca_kernel_iterator.h
|
|
15
|
+
- how the engine works (its routes, the promises every route keeps,
|
|
16
|
+
and which part keeps each): docs/internal/KernelIteratorEngine.md
|
|
17
|
+
|
|
18
|
+
Map of this file:
|
|
19
|
+
- scratch buffers (ki_data_alloc / ki_data_free) and helpers
|
|
20
|
+
- classification: ca_iter_classify_source, ca_iter_route_source
|
|
21
|
+
- the promise parts every route calls: ki_take_mask_copy,
|
|
22
|
+
ki_gather_view, ki_scratch_push_changed, ki_attach_stack_parents,
|
|
23
|
+
ki_fiber_contig_data / _mask
|
|
24
|
+
- opening a walk: init_l1, and for L2 ki_choose_l2_route (decides)
|
|
25
|
+
then ki_open_l2 (builds the state, with ki_lay_out_slabs)
|
|
26
|
+
- handing out slabs: next_slab, next_slab_axes
|
|
27
|
+
- writing back: sync_slab
|
|
28
|
+
- closing: finish, close, and the raise-safe entry points
|
|
29
|
+
(ca_iter_protect, ca_iter_ensure)
|
|
30
|
+
- development-build test surface (CARRAY_DEV_BUILD), at the end
|
|
31
|
+
|
|
32
|
+
A route added for speed is a shortcut, tagged next to its code with
|
|
33
|
+
"Shortcut NAME" and a fixed set of fields; each one is switched off by
|
|
34
|
+
ca_fast_paths_enabled().
|
|
22
35
|
|
|
23
36
|
---------------------------------------------------------------------------- */
|
|
24
37
|
|
|
25
38
|
#include "carray.h"
|
|
26
39
|
#include "carray_internal.h" /* per-obj_type view constructors */
|
|
27
40
|
#include "ca_kernel_iterator.h"
|
|
28
|
-
#include "ca_monop_dispatch.h" /*
|
|
29
|
-
#include "ca_obj_face.h" /*
|
|
41
|
+
#include "ca_monop_dispatch.h" /* ca_monop_view_is_single_cast */
|
|
42
|
+
#include "ca_obj_face.h" /* ca_strip_face, to walk what a Face wraps */
|
|
30
43
|
|
|
31
44
|
#include <assert.h>
|
|
32
45
|
#include <string.h>
|
|
46
|
+
#include <ruby/ractor.h>
|
|
47
|
+
|
|
48
|
+
/* Private flags bit, kept in ca_iter_state.flags like
|
|
49
|
+
CA_ITER_FLAG_WALK_MAY_RAISE below: src is a copy the walk owns (see
|
|
50
|
+
ca_iter_reopen_on_copy), freed when the walk finishes. */
|
|
51
|
+
#define CA_ITER_FLAG_OWNS_SRC 0x40000000u
|
|
52
|
+
|
|
53
|
+
/* Data scratch. A scratch of object cells holds VALUEs no Ruby object
|
|
54
|
+
owns while the kernel calls Ruby for each cell, so it is filled with
|
|
55
|
+
Qnil and held for the GC until finish releases it.
|
|
56
|
+
|
|
57
|
+
One large scratch is kept when a walk frees it and handed to the next
|
|
58
|
+
walk that fits in it: a fresh allocation of that size costs the page
|
|
59
|
+
faults of touching it, which for a reduction is several times the
|
|
60
|
+
reduction itself. The kept buffer is owned by nobody, and a walk that
|
|
61
|
+
takes it owns it alone until it frees it again. Each Ractor keeps its
|
|
62
|
+
own (ractor-local storage), since walks in different Ractors run at
|
|
63
|
+
the same time.
|
|
64
|
+
|
|
65
|
+
Shortcut KEEP_SCRATCH (name tag; docs/internal/KernelIteratorEngine.md
|
|
66
|
+
section 3.4 has the table of all shortcuts).
|
|
67
|
+
Layer data: where a scratch comes from. The walk is unchanged.
|
|
68
|
+
Speeds up walks with a scratch of 1 MiB or more, run one after
|
|
69
|
+
another.
|
|
70
|
+
Why no difference measured on macOS (a per-axis sum of a
|
|
71
|
+
2000x2000 fake view); the page faults it
|
|
72
|
+
avoids are expected to cost more on Linux (unmeasured).
|
|
73
|
+
Enters ki_data_alloc: bytes >= KI_KEEP_MIN and a kept buffer
|
|
74
|
+
that fits; ki_data_free: KI_KEEP_MIN <= bytes <=
|
|
75
|
+
KI_KEEP_MAX.
|
|
76
|
+
Promises 4 (a kept buffer has no owner, so no exit can leave it
|
|
77
|
+
held); 7 (object cells are cleared and held either way).
|
|
78
|
+
Switch ca_fast_paths_enabled(): off, every scratch is fresh.
|
|
79
|
+
Checked by test_iter_ractor_scratch.rb; the buffer count in
|
|
80
|
+
test_iter_delivery_detectors.rb. */
|
|
81
|
+
|
|
82
|
+
#define KI_KEEP_MIN ((ca_size_t) 1 << 20)
|
|
83
|
+
#define KI_KEEP_MAX ((ca_size_t) 64 << 20)
|
|
84
|
+
|
|
85
|
+
typedef struct {
|
|
86
|
+
void *p;
|
|
87
|
+
ca_size_t bytes;
|
|
88
|
+
} ki_kept_t;
|
|
89
|
+
|
|
90
|
+
static rb_ractor_local_key_t ki_kept_key;
|
|
91
|
+
|
|
92
|
+
static void
|
|
93
|
+
ki_kept_free (void *ptr)
|
|
94
|
+
{
|
|
95
|
+
ki_kept_t *k = (ki_kept_t *) ptr;
|
|
96
|
+
if ( k ) {
|
|
97
|
+
if ( k->p ) ruby_xfree(k->p);
|
|
98
|
+
ruby_xfree(k);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
static const struct rb_ractor_local_storage_type ki_kept_type = {
|
|
103
|
+
NULL, ki_kept_free
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
static ki_kept_t *
|
|
107
|
+
ki_kept_slot (void)
|
|
108
|
+
{
|
|
109
|
+
ki_kept_t *k = (ki_kept_t *) rb_ractor_local_storage_ptr(ki_kept_key);
|
|
110
|
+
if ( k == NULL ) {
|
|
111
|
+
k = ALLOC(ki_kept_t);
|
|
112
|
+
k->p = NULL;
|
|
113
|
+
k->bytes = 0;
|
|
114
|
+
rb_ractor_local_storage_ptr_set(ki_kept_key, k);
|
|
115
|
+
}
|
|
116
|
+
return k;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/* Development build: the number of buffers ki_data_alloc has handed out
|
|
120
|
+
that ki_data_free has not taken back (CArray.__iter_buffers__). A walk
|
|
121
|
+
that ends leaves the count where it found it; the spec suite compares it
|
|
122
|
+
around each walk. */
|
|
123
|
+
#ifdef CARRAY_DEV_BUILD
|
|
124
|
+
static long ki_buffers_out = 0;
|
|
125
|
+
/* atomic: walks run in several Ractors at once */
|
|
126
|
+
#define KI_BUFFERS_OUT(d) ( (void) __atomic_add_fetch(&ki_buffers_out, (d), __ATOMIC_RELAXED) )
|
|
127
|
+
#else
|
|
128
|
+
#define KI_BUFFERS_OUT(d) ((void) 0)
|
|
129
|
+
#endif
|
|
130
|
+
|
|
131
|
+
static void *
|
|
132
|
+
ki_data_alloc (int8_t data_type, ca_size_t bytes)
|
|
133
|
+
{
|
|
134
|
+
void *p = NULL;
|
|
135
|
+
KI_BUFFERS_OUT(1);
|
|
136
|
+
if ( bytes >= KI_KEEP_MIN && ca_fast_paths_enabled() ) {
|
|
137
|
+
ki_kept_t *k = ki_kept_slot();
|
|
138
|
+
if ( k->p && bytes <= k->bytes ) {
|
|
139
|
+
p = k->p;
|
|
140
|
+
k->p = NULL;
|
|
141
|
+
k->bytes = 0;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if ( p == NULL ) {
|
|
145
|
+
p = xmalloc(bytes > 0 ? bytes : 1);
|
|
146
|
+
}
|
|
147
|
+
if ( data_type == CA_OBJECT ) {
|
|
148
|
+
ca_size_t n = bytes / (ca_size_t) sizeof(VALUE);
|
|
149
|
+
ca_object_cells_clear(p, n);
|
|
150
|
+
ca_gc_hold_buffer(p, n);
|
|
151
|
+
}
|
|
152
|
+
return p;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/* `bytes` is the size p was allocated with (0 when unknown: never kept). */
|
|
156
|
+
static void
|
|
157
|
+
ki_data_free (void *p, ca_size_t bytes)
|
|
158
|
+
{
|
|
159
|
+
KI_BUFFERS_OUT(-1);
|
|
160
|
+
ca_gc_release_buffer(p);
|
|
161
|
+
if ( bytes >= KI_KEEP_MIN && bytes <= KI_KEEP_MAX && ca_fast_paths_enabled() ) {
|
|
162
|
+
ki_kept_t *k = ki_kept_slot();
|
|
163
|
+
if ( bytes > k->bytes ) {
|
|
164
|
+
void *old = k->p;
|
|
165
|
+
k->p = p;
|
|
166
|
+
k->bytes = bytes;
|
|
167
|
+
if ( old ) {
|
|
168
|
+
xfree(old);
|
|
169
|
+
}
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
xfree(p);
|
|
174
|
+
}
|
|
33
175
|
|
|
34
176
|
/* Defined in carray_core.c. (ca_is_readonly / ca_has_mask /
|
|
35
177
|
ca_sync_data are already declared in carray.h.) */
|
|
36
178
|
extern int ca_attach_is_alias (void *ap);
|
|
37
179
|
extern int ca_root_lends_no_memory (void *ap);
|
|
38
180
|
|
|
39
|
-
/* CAStride family attach-fn marker
|
|
181
|
+
/* CAStride family attach-fn marker. */
|
|
40
182
|
extern ca_operation_function_t ca_stride_func;
|
|
41
183
|
|
|
42
|
-
/* Descriptor framework view ops
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
(
|
|
46
|
-
|
|
47
|
-
|
|
184
|
+
/* Descriptor framework view ops. Each view's describe_axes emits the
|
|
185
|
+
per-axis descriptor + parent dim snapshot consumed by the
|
|
186
|
+
ca_axis_dispatch_* engine. CAShift shares CAWindow's operation table
|
|
187
|
+
(ca_shift_func is a copy with only free / clone / create_mask
|
|
188
|
+
overridden), so it is recognised by its attach pointer matching
|
|
189
|
+
ca_window_func.attach. */
|
|
48
190
|
extern ca_operation_function_t ca_select_axis_func;
|
|
49
191
|
extern ca_operation_function_t ca_grid_func;
|
|
50
192
|
extern ca_operation_function_t ca_select_func;
|
|
51
193
|
extern ca_operation_function_t ca_window_func;
|
|
52
|
-
/* ca_mapping_func retired in R.3 (PROPOSAL_CAMAPPING_REMOVAL). */
|
|
53
194
|
|
|
54
|
-
/*
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
195
|
+
/* Operation tables the classifier compares a source's attach slot
|
|
196
|
+
against. Every view below is delivered as CA_ITER_SRC_ATTACH: the
|
|
197
|
+
engine gathers its cells into a scratch buffer of its own through the
|
|
198
|
+
view's transfer (ki_gather_view = ca_xfer_all GET) and, on a WRITE
|
|
199
|
+
walk, sends back the cells the kernel changed (ki_scratch_push_changed
|
|
200
|
+
= ca_xfer_addrs, or ca_xfer_all PUT when every cell changed). The
|
|
201
|
+
view's attach / sync slots are not called. Cast / byte swap / bit
|
|
202
|
+
unpack / reduction views: */
|
|
59
203
|
extern ca_operation_function_t ca_fake_func;
|
|
60
204
|
extern ca_operation_function_t ca_byte_swap_func;
|
|
61
205
|
extern ca_operation_function_t ca_bitfield_func;
|
|
@@ -63,46 +207,29 @@ extern ca_operation_function_t ca_bitarray_func;
|
|
|
63
207
|
extern ca_operation_function_t ca_reduce_func;
|
|
64
208
|
extern ca_operation_function_t ca_object_func;
|
|
65
209
|
|
|
66
|
-
/*
|
|
67
|
-
|
|
68
|
-
descriptor framework's kind enum {STRIDE, INDEX, SHIFT}, so the
|
|
69
|
-
innermost-STRIDE L2 alias path is not used; but the view-specific
|
|
70
|
-
func_attach (embed-region gather) materialises src->ptr and func_sync
|
|
71
|
-
scatters back — structurally identical to SRC_ATTACH. Fills the gap
|
|
72
|
-
in the "deliver" principle (= these were previously rejected via
|
|
73
|
-
SRC_NONE). */
|
|
210
|
+
/* CATile / CARoll: the per-cell modulo wrap has no STRIDE / INDEX /
|
|
211
|
+
SHIFT descriptor, so they are delivered as SRC_ATTACH (above). */
|
|
74
212
|
extern ca_operation_function_t ca_tile_func;
|
|
75
213
|
extern ca_operation_function_t ca_roll_func;
|
|
76
214
|
|
|
77
|
-
/*
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
func_sync scatters back via xfer_all PUT. kernel sees a flat
|
|
81
|
-
contig slab over the stacked output of shape (K, *parent_shape). */
|
|
215
|
+
/* CAStack / CAMeld: SRC_ATTACH (above). A stack can also be walked
|
|
216
|
+
through its parents directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K,
|
|
217
|
+
see init_l2). */
|
|
82
218
|
extern ca_operation_function_t ca_stack_func;
|
|
83
219
|
extern ca_operation_function_t ca_meld_func;
|
|
84
220
|
|
|
85
|
-
/*
|
|
86
|
-
|
|
87
|
-
SRC_ATTACH source. Per-element gather has no STRIDE structure to
|
|
88
|
-
preserve, so SRC_ATTACH (scratch materialise via ca_remap_func_attach)
|
|
89
|
-
is the natural acceptance path. func_attach allocates a fresh
|
|
90
|
-
buffer and runs xfer_all(GET); kernel sees a flat contig slab.
|
|
91
|
-
On WRITE, ca_sync routes through xfer_all(PUT). */
|
|
221
|
+
/* CARemap (internal, no Ruby class constant): a per-element gather with
|
|
222
|
+
no STRIDE structure, delivered as SRC_ATTACH (above). */
|
|
92
223
|
extern ca_operation_function_t ca_remap_func;
|
|
93
224
|
|
|
94
|
-
/*
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
the lazy tree's materialise into a fresh contig buffer; kernel sees
|
|
98
|
-
a flat slab. Same SRC_ATTACH structural pattern as CAFake et al,
|
|
99
|
-
so a 4-line addition to the classify_source list opens all 22
|
|
100
|
-
mkkernel-generated reduction ops (sum / count / mean / variance /
|
|
101
|
-
argmin / ...) to lazy operands — `(a.lazy + b).sum` now works. */
|
|
225
|
+
/* Lazy element-wise views (CAMonOp / CABinOp / CABinCmp / CAMonCmp /
|
|
226
|
+
CATriOp): SRC_ATTACH (above); gathering evaluates the expression into
|
|
227
|
+
the scratch, so every generated reduction accepts a lazy operand. */
|
|
102
228
|
extern ca_operation_function_t ca_monop_func;
|
|
103
229
|
extern ca_operation_function_t ca_binop_func;
|
|
104
230
|
extern ca_operation_function_t ca_bincmp_func;
|
|
105
231
|
extern ca_operation_function_t ca_moncmp_func;
|
|
232
|
+
extern ca_operation_function_t ca_triop_func;
|
|
106
233
|
extern CArray *ca_remap_new (CArray *ref, CArray *idx);
|
|
107
234
|
|
|
108
235
|
/* ca_reduce_new is declared in carray.h; rb_cCAReduce defined in
|
|
@@ -154,16 +281,27 @@ ca_iter_register_source_kind (int obj_type, uint8_t kind)
|
|
|
154
281
|
ca_iter_registered_source_kind[obj_type] = kind;
|
|
155
282
|
}
|
|
156
283
|
|
|
157
|
-
/* Classify the source by
|
|
158
|
-
|
|
284
|
+
/* Classify the source by how its cells are delivered. Returns
|
|
285
|
+
CA_ITER_SRC_NONE only for a NULL source or one with no attach slot;
|
|
286
|
+
every other view is delivered, as SRC_ATTACH if nothing more specific
|
|
287
|
+
applies. */
|
|
159
288
|
static uint8_t
|
|
160
289
|
ca_iter_classify_source (CArray *src)
|
|
161
290
|
{
|
|
162
291
|
if ( src == NULL ) return CA_ITER_SRC_NONE;
|
|
292
|
+
|
|
293
|
+
/* Development build, CARRAY_GENERAL_PATH=2: every source takes the
|
|
294
|
+
transfer walk (see ki_open_by_transfer), the reference the other
|
|
295
|
+
routes are checked against. */
|
|
296
|
+
if ( ca_iter_by_transfer_only() ) {
|
|
297
|
+
return ca_func[src->obj_type].attach != NULL ? CA_ITER_SRC_ATTACH
|
|
298
|
+
: CA_ITER_SRC_NONE;
|
|
299
|
+
}
|
|
300
|
+
|
|
163
301
|
if ( ca_is_entity(src) ) return CA_ITER_SRC_CASTRIDE;
|
|
164
302
|
if ( ca_iter_is_castride_family(src) ) return CA_ITER_SRC_CASTRIDE;
|
|
165
303
|
|
|
166
|
-
/* Externally installed obj_types declare their routing
|
|
304
|
+
/* Externally installed obj_types declare their routing.
|
|
167
305
|
Placed after the two structural cases and before the built-in list:
|
|
168
306
|
entity and CAStride-family sources are recognised from the struct
|
|
169
307
|
itself and are read directly, so a registration must not be able to
|
|
@@ -175,88 +313,73 @@ ca_iter_classify_source (CArray *src)
|
|
|
175
313
|
if ( kind != CA_ITER_SRC_NONE ) return kind;
|
|
176
314
|
}
|
|
177
315
|
|
|
178
|
-
/* Descriptor
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
check below. */
|
|
316
|
+
/* Descriptor views: CASelectAxis / CAGrid / CASelect / CAWindow.
|
|
317
|
+
CAShift shares ca_window_func's attach, so it matches the CAWindow
|
|
318
|
+
check. */
|
|
182
319
|
void *attach = ca_func[src->obj_type].attach;
|
|
183
320
|
if ( attach == ca_select_axis_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
184
321
|
if ( attach == ca_grid_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
185
322
|
if ( attach == ca_select_func.attach ) return CA_ITER_SRC_DESCRIPTOR;
|
|
186
323
|
if ( attach == ca_window_func.attach ) return CA_ITER_SRC_DESCRIPTOR; /* + CAShift */
|
|
187
324
|
|
|
188
|
-
/*
|
|
189
|
-
|
|
325
|
+
/* The views below are SRC_ATTACH: gathered into scratch through their
|
|
326
|
+
own transfer (ki_gather_view) and, on WRITE, sent back the changed
|
|
327
|
+
cells (ki_scratch_push_changed). Their attach / sync slots are not
|
|
328
|
+
called by the engine. */
|
|
190
329
|
if ( attach == ca_fake_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
191
330
|
if ( attach == ca_byte_swap_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
192
331
|
if ( attach == ca_bitfield_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
193
332
|
if ( attach == ca_bitarray_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
194
333
|
if ( attach == ca_reduce_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
195
|
-
/*
|
|
196
|
-
|
|
197
|
-
copy_data, func_sync scatters back via Ruby sync_data, CA_FLAG_
|
|
198
|
-
READ_ONLY auto-rejects WRITE via ca_is_readonly). Bench gate
|
|
199
|
-
n/a (Ruby callback overhead structurally dominant). */
|
|
334
|
+
/* CAObject: each transfer calls the Ruby callback. A read-only
|
|
335
|
+
CAObject is refused for WRITE by the ca_is_readonly check. */
|
|
200
336
|
if ( attach == ca_object_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
201
337
|
|
|
202
|
-
/* F-2 follow-up: CATile / CARoll via SRC_ATTACH (embed-region gather
|
|
203
|
-
materialises src->ptr; ca_sync scatters back per view-specific
|
|
204
|
-
semantics — CATile tile decomposition, CARoll cyclic permutation). */
|
|
205
338
|
if ( attach == ca_tile_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
206
339
|
if ( attach == ca_roll_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
207
340
|
|
|
208
|
-
/*
|
|
209
|
-
|
|
210
|
-
bytes alloc, caller responsibility per MEMO §3.4); func_sync
|
|
211
|
-
scatters back via xfer_all PUT. kernel sees a flat contig slab
|
|
212
|
-
over the stacked output. Routine ndim mismatch is irrelevant
|
|
213
|
-
here (= attach delivers a self-owned buffer of shape (K, *parent
|
|
214
|
-
shape), kernel reads/writes that buffer). */
|
|
341
|
+
/* CAStack: SRC_ATTACH here; init_l2 may instead walk its parents
|
|
342
|
+
directly (CA_ITER_ALIAS_STACK / STACK_OUTER_K). */
|
|
215
343
|
if ( attach == ca_stack_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
216
344
|
|
|
217
|
-
/* CAMeld
|
|
218
|
-
|
|
219
|
-
seg_offset[k] * tail_bytes offsets; func_sync scatters back. Same
|
|
220
|
-
SRC_ATTACH structural pattern as CAStack; reduce hot paths bypass
|
|
221
|
-
this via the per-parent decompose in lib/carray/meld_reduce.rb. */
|
|
345
|
+
/* CAMeld: reduce hot paths avoid the gather through the per-parent
|
|
346
|
+
decompose in lib/carray/meld_reduce.rb. */
|
|
222
347
|
if ( attach == ca_meld_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
223
348
|
|
|
224
|
-
/* M.6: CARemap — per-element gather, internal-only. Same SRC_ATTACH
|
|
225
|
-
structural pattern (func_attach materialises via xfer_all GET,
|
|
226
|
-
func_sync via xfer_all PUT). */
|
|
227
349
|
if ( attach == ca_remap_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
228
350
|
|
|
229
|
-
/*
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
slab. Read-only (= CA_FLAG_READ_ONLY) so WRITE auto-rejects at
|
|
233
|
-
the ca_is_readonly check. Opens 22 mkkernel-generated reduction
|
|
234
|
-
ops to lazy operands. */
|
|
351
|
+
/* Lazy views (CAMonOp / CABinOp / CABinCmp / CAMonCmp / CATriOp):
|
|
352
|
+
gathering evaluates the expression. They are read-only, so WRITE is
|
|
353
|
+
refused by the ca_is_readonly check. */
|
|
235
354
|
if ( attach == ca_monop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
236
355
|
if ( attach == ca_binop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
237
356
|
if ( attach == ca_bincmp_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
238
357
|
if ( attach == ca_moncmp_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
358
|
+
if ( attach == ca_triop_func.attach ) return CA_ITER_SRC_ATTACH;
|
|
359
|
+
|
|
360
|
+
/* Anything else -- a view class installed from outside that did not
|
|
361
|
+
register, or an in-tree class with no case above (such as the mask
|
|
362
|
+
views CAMaskOfOperands and CAObjectMask). It is classified as a
|
|
363
|
+
registration would declare it, SRC_ATTACH, and delivered by the
|
|
364
|
+
transfer walk through its xfer slots rather than refused. */
|
|
365
|
+
if ( attach != NULL ) return CA_ITER_SRC_ATTACH;
|
|
239
366
|
|
|
240
367
|
return CA_ITER_SRC_NONE;
|
|
241
368
|
}
|
|
242
369
|
|
|
243
|
-
/*
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
Per-walk overhead = 1 describe_axes call (descriptor sources only),
|
|
257
|
-
which init was going to do anyway -- so the routing is net zero cost
|
|
258
|
-
compared to the pre-rev6 path where classify_source + init both
|
|
259
|
-
computed describe_axes redundantly. See prep doc rev2 §4.1.1. */
|
|
370
|
+
/* Routes a source by running classify_source and, for descriptor-routed views, also calling
|
|
371
|
+
describe_axes to inspect the innermost axis kind. A descriptor view
|
|
372
|
+
comes back as SRC_DESCRIPTOR_L2_ALIASABLE (innermost axis STRIDE) or
|
|
373
|
+
SRC_DESCRIPTOR, and then out_descs / out_parent_dims / out_ndim are
|
|
374
|
+
filled so the caller does not re-call describe_axes. A grid with holes
|
|
375
|
+
and a view whose parent lends no memory come back as SRC_ATTACH, and
|
|
376
|
+
the out_* arguments are not touched; nor are they for any other kind,
|
|
377
|
+
whose classify_source verdict is returned as-is.
|
|
378
|
+
|
|
379
|
+
Cost: classify_source is O(1) pointer compares; a descriptor view
|
|
380
|
+
adds one describe_axes call (O(ndim)), which init would make anyway,
|
|
381
|
+
and init reads the descriptors from here instead of describing the
|
|
382
|
+
axes again. */
|
|
260
383
|
static uint8_t ca_iter_classify_source (CArray *src);
|
|
261
384
|
static void ca_iter_describe_axes (CArray *src, ca_axis_desc_t *,
|
|
262
385
|
ca_size_t *, int8_t *);
|
|
@@ -270,6 +393,25 @@ ca_iter_route_source (CArray *src,
|
|
|
270
393
|
uint8_t kind = ca_iter_classify_source(src);
|
|
271
394
|
if ( kind != CA_ITER_SRC_DESCRIPTOR ) return kind;
|
|
272
395
|
|
|
396
|
+
/* A grid with holes (masked index cells) is not described by the engine
|
|
397
|
+
either: its holes are delivered by its own transfers. */
|
|
398
|
+
if ( ca_func[src->obj_type].attach == ca_grid_func.attach
|
|
399
|
+
&& ca_grid_has_holes(src) ) {
|
|
400
|
+
return CA_ITER_SRC_ATTACH;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/* The descriptor walk reads the parent's memory, which such a parent
|
|
404
|
+
only has after a whole materialise -- however few cells the view
|
|
405
|
+
selects. Deliver through the view's own transfer instead: SRC_ATTACH
|
|
406
|
+
gathers the view into scratch with ca_xfer_all, and the view asks its
|
|
407
|
+
parent for the selected cells alone. */
|
|
408
|
+
{
|
|
409
|
+
CArray *parent = ((CAView *) src)->parent;
|
|
410
|
+
if ( ! parent->ptr && ca_parent_lends_no_memory(parent) ) {
|
|
411
|
+
return CA_ITER_SRC_ATTACH;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
273
415
|
/* Descriptor source: describe_axes + inspect innermost. */
|
|
274
416
|
ca_iter_describe_axes(src, out_descs, out_parent_dims, out_ndim);
|
|
275
417
|
if ( ca_axis_dispatch_is_innermost_stride(out_descs, *out_ndim) ) {
|
|
@@ -278,9 +420,8 @@ ca_iter_route_source (CArray *src,
|
|
|
278
420
|
return CA_ITER_SRC_DESCRIPTOR;
|
|
279
421
|
}
|
|
280
422
|
|
|
281
|
-
/* Dispatch to the view's describe_axes
|
|
282
|
-
|
|
283
|
-
typedef). */
|
|
423
|
+
/* Dispatch to the view's describe_axes, keyed on the attach pointer
|
|
424
|
+
(CAShift shares CAWindow's). */
|
|
284
425
|
static void
|
|
285
426
|
ca_iter_describe_axes (CArray *src,
|
|
286
427
|
ca_axis_desc_t *out_descs,
|
|
@@ -315,7 +456,8 @@ ca_iter_describe_axes (CArray *src,
|
|
|
315
456
|
*out_ndim = 0;
|
|
316
457
|
}
|
|
317
458
|
|
|
318
|
-
/* Public alias eligibility predicate
|
|
459
|
+
/* Public alias eligibility predicate: whether a walk at `level` could
|
|
460
|
+
read the source's own memory in place. */
|
|
319
461
|
int
|
|
320
462
|
ca_iter_can_alias (void *ap, int level)
|
|
321
463
|
{
|
|
@@ -331,9 +473,8 @@ ca_iter_can_alias (void *ap, int level)
|
|
|
331
473
|
return ca_iter_is_castride_family(ca);
|
|
332
474
|
|
|
333
475
|
default:
|
|
334
|
-
/* L3 (and any future level)
|
|
335
|
-
|
|
336
|
-
probe ahead. */
|
|
476
|
+
/* L3 (and any future level) is not implemented; answer as L1 so
|
|
477
|
+
the predicate stays well-defined for callers that probe ahead. */
|
|
337
478
|
return ca_attach_is_alias(ca);
|
|
338
479
|
}
|
|
339
480
|
}
|
|
@@ -353,24 +494,24 @@ ca_iter_build_rowmajor_strides (ca_size_t *strides,
|
|
|
353
494
|
}
|
|
354
495
|
}
|
|
355
496
|
|
|
356
|
-
/* ----
|
|
497
|
+
/* ---- PER_FIBER_FUSED: which views take it ----------------------------- */
|
|
357
498
|
|
|
358
|
-
/*
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
499
|
+
/* A src_kind branch, then a check of the view's operation table, then of
|
|
500
|
+
the fiber axis's stride. Returns nonzero for
|
|
501
|
+
CAFake / CAByteSwap / a single-cast CAMonOp under SRC_ATTACH when the
|
|
502
|
+
fiber is contiguous in the parent, and for CAWindow / CAShift under
|
|
503
|
+
SRC_DESCRIPTOR; zero otherwise.
|
|
363
504
|
|
|
364
505
|
The fiber_axis_stride argument is the effective byte stride of the
|
|
365
506
|
fiber axis on the view (= what next_slab_axes would yield as
|
|
366
|
-
slab_strides[0]).
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
Called from
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
state
|
|
507
|
+
slab_strides[0]). fiber_axis_stride == src->bytes means the fiber is
|
|
508
|
+
a contiguous run of the parent's memory, which is what lets the view's
|
|
509
|
+
region transfer deliver it in one pass.
|
|
510
|
+
|
|
511
|
+
Called from two places in init_l2: the CA_SLAB_AXES descriptor branch
|
|
512
|
+
and the SRC_ATTACH branch, each before it would materialise. When it
|
|
513
|
+
returns nonzero the caller skips the materialise and sets alias_mode =
|
|
514
|
+
CA_ITER_ALIAS_PER_FIBER_FUSED with the fiber dispatch state filled. */
|
|
374
515
|
static int
|
|
375
516
|
ca_iter_should_per_fiber_fused (CArray *src,
|
|
376
517
|
int src_kind,
|
|
@@ -379,27 +520,22 @@ ca_iter_should_per_fiber_fused (CArray *src,
|
|
|
379
520
|
uint32_t flags)
|
|
380
521
|
{
|
|
381
522
|
(void) fiber_axis;
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
materialise (CAFake) / 16-18% (CAByteSwap). Non-innermost (=
|
|
392
|
-
fiber_axis_stride != bytes) was a loser, so the stride gate
|
|
393
|
-
keeps current path for that case.
|
|
394
|
-
|
|
395
|
-
Phase 6 P.6.2.d (Q13 α): single-cast CAMonOp (= post-migration
|
|
396
|
-
successor of CAFake numeric path) is recognised via
|
|
397
|
-
ca_monop_view_is_single_cast and routed through the same F.6.2
|
|
398
|
-
fast path. Chain CAMonOp (depth ≥ 2) is excluded — its
|
|
399
|
-
materialise path (= arena-pooled scratches from Phase 3) is
|
|
400
|
-
structurally different from the X.4 inner-contig fast path. */
|
|
523
|
+
|
|
524
|
+
/* CAFake / CAByteSwap: when the fiber is a contiguous run of the
|
|
525
|
+
parent's memory, ca_xfer_stride takes the view's converting fast
|
|
526
|
+
path (ca_xfer_stride_transform_fused) and delivers the fiber in one
|
|
527
|
+
pass. A fiber along another axis was measured slower this way, so
|
|
528
|
+
the stride gate keeps it on the gathered route. A single-cast
|
|
529
|
+
CAMonOp (ca_monop_view_is_single_cast) is the lazy form of the same
|
|
530
|
+
conversion and takes the same route; a chain of lazy operations
|
|
531
|
+
gathers through the lazy arena instead, which this does not fit. */
|
|
401
532
|
void *attach = ca_func[src->obj_type].attach;
|
|
402
|
-
|
|
533
|
+
/* A conversion view is read-only on this path. Its put-back sends the
|
|
534
|
+
whole fiber, and a cell the kernel did not write does not come back
|
|
535
|
+
unchanged through a conversion that loses information (an int64
|
|
536
|
+
above 2^53 seen as float64), so a write takes the transfer walk,
|
|
537
|
+
which sends back only the changed cells (promise 3). */
|
|
538
|
+
if ( src_kind == CA_ITER_SRC_ATTACH && !(flags & CA_KERNEL_WRITE) ) {
|
|
403
539
|
if ( attach == ca_fake_func.attach
|
|
404
540
|
|| attach == ca_byte_swap_func.attach
|
|
405
541
|
|| ca_monop_view_is_single_cast(src) ) {
|
|
@@ -407,64 +543,26 @@ ca_iter_should_per_fiber_fused (CArray *src,
|
|
|
407
543
|
}
|
|
408
544
|
}
|
|
409
545
|
|
|
410
|
-
/*
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
(
|
|
416
|
-
STRIDE promotion), the existing alias path is already optimal
|
|
417
|
-
(parent.ptr + composed offsets, zero materialise). Per-fiber
|
|
418
|
-
dispatch in that case adds ca_xfer_stride per-call overhead with
|
|
419
|
-
no payoff -- F.6.3 bench measured 0.35-0.82x (regression). Gate
|
|
420
|
-
on SRC_DESCRIPTOR only. */
|
|
546
|
+
/* CAShift / CAWindow: the view's region transfer fills out-of-bounds
|
|
547
|
+
cells itself, so a fiber can be fetched in one call instead of a
|
|
548
|
+
per-slab or whole-view gather (SRC_DESCRIPTOR). A view that can
|
|
549
|
+
lend the parent's memory (SRC_DESCRIPTOR_L2_ALIASABLE, an interior
|
|
550
|
+
CAWindow) is already read in place; fetching it fiber by fiber was
|
|
551
|
+
measured slower (0.35-0.82x), so only SRC_DESCRIPTOR takes it. */
|
|
421
552
|
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
422
|
-
/*
|
|
423
|
-
typedef pattern: ca_shift_func = ca_window_func). */
|
|
553
|
+
/* CAWindow and CAShift share ca_window_func.attach. */
|
|
424
554
|
if ( attach == ca_window_func.attach ) {
|
|
425
555
|
return 1;
|
|
426
556
|
}
|
|
427
557
|
}
|
|
428
558
|
|
|
429
|
-
/*
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
batched throughput. Structurally different from
|
|
437
|
-
CAFake/CAByteSwap (transform-fused at byte level, X.4) and
|
|
438
|
-
CAShift/CAWindow OOB (per-region fused with bound check, X.1):
|
|
439
|
-
descriptor framework views do not have a per-region fused fast
|
|
440
|
-
path designed for per-fiber entry.
|
|
441
|
-
|
|
442
|
-
Note (rev4): bench split fiber-axis kind shows STRIDE fiber
|
|
443
|
-
1.25x slower vs INDEX fiber 4-5x slower. STRIDE-fiber case
|
|
444
|
-
is close to parity and could conceivably reach win with finer
|
|
445
|
-
gating (= bypass per-axis kind classification when the outer
|
|
446
|
-
prefix is homogeneous STRIDE). Not pursued at current 1.25x
|
|
447
|
-
deficit; recorded as a future possible refinement should an
|
|
448
|
-
application motivate it. See proposal §3.1 SRC_DESCRIPTOR
|
|
449
|
-
(CSA, CAGrid, ...) inline comment.
|
|
450
|
-
|
|
451
|
-
F.6.6 audit (devel/bench_f6_6_bit_audit.rb): CABitarray /
|
|
452
|
-
CABitfield kept on current path. Bench measured via
|
|
453
|
-
bits.fake(:float64) (= production CAFake-wrap hot path) shows
|
|
454
|
-
per-fiber 1.23-5.86x slower on the F.6.2-gate-off axes.
|
|
455
|
-
|
|
456
|
-
The CAFake wrap overhead applies equally to both per-fiber and
|
|
457
|
-
whole-view paths (same total cell-cast cost), so the 5x ratio
|
|
458
|
-
is essentially the CABitarray whole-view-vs-per-fiber ratio
|
|
459
|
-
itself. Phase A bench note "xfer_stride 0.98x parity" is a
|
|
460
|
-
single-call comparison: N per-fiber calls cumulatively cost
|
|
461
|
-
N x parity > 1 x xfer_all (+ N dispatch overhead + per-call
|
|
462
|
-
bit alignment math). This reasoning generalises to direct
|
|
463
|
-
CABitarray / CABitfield source (= when a non-f64 fiber smoke
|
|
464
|
-
is added in a future phase) -- no separate audit needed for
|
|
465
|
-
that case. Predicate stays unchanged.
|
|
466
|
-
|
|
467
|
-
F.6.7 will audit CAReduce. */
|
|
559
|
+
/* Not taken by the other views, each measured slower fiber by fiber:
|
|
560
|
+
CASelectAxis / CAGrid / CASelect (1.25-4.7x), whose descriptor
|
|
561
|
+
engine already gathers the whole view in one batched pass and has no
|
|
562
|
+
per-region fast path to enter; CABitarray / CABitfield (1.2-5.9x,
|
|
563
|
+
through a fake over them), where N per-fiber calls each pay the bit
|
|
564
|
+
alignment that one whole-view transfer pays once. CAReduce is not
|
|
565
|
+
routed here either. */
|
|
468
566
|
return 0;
|
|
469
567
|
}
|
|
470
568
|
|
|
@@ -480,44 +578,38 @@ ca_iter_validate_inputs (ca_iter_state *st,
|
|
|
480
578
|
uint32_t flags)
|
|
481
579
|
{
|
|
482
580
|
if ( st == NULL || src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
483
|
-
/*
|
|
484
|
-
|
|
581
|
+
/* CA_SLAB_AXES is accepted here and refused by init_l1 itself; init_l2
|
|
582
|
+
refuses CA_SLAB_WHOLE (see each init's policy gate). */
|
|
485
583
|
if ( policy != CA_SLAB_WHOLE && policy != CA_SLAB_AXES )
|
|
486
584
|
return CA_ITER_ERR_POLICY;
|
|
487
585
|
|
|
488
|
-
/*
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
lands in step 7; step 6 accepts the flag but does not enforce.
|
|
492
|
-
CA_KERNEL_FIBER_CONTIG (PROPOSAL_FIBER_DELIVERY F.1a) accepted —
|
|
493
|
-
activates per-fiber contig delivery for naxes==1 in next_slab_axes. */
|
|
586
|
+
/* Accepted: CA_KERNEL_WRITE, CA_KERNEL_NO_MASK (enforced below) and
|
|
587
|
+
CA_KERNEL_FIBER_CONTIG (per-fiber contig delivery for naxes==1 in
|
|
588
|
+
next_slab_axes). CA_KERNEL_CHUNK_HINT is reserved and rejected. */
|
|
494
589
|
const uint32_t accepted = CA_KERNEL_WRITE | CA_KERNEL_NO_MASK
|
|
495
590
|
| CA_KERNEL_FIBER_CONTIG;
|
|
496
591
|
if ( flags & ~accepted ) return CA_ITER_ERR_FLAGS;
|
|
497
592
|
|
|
498
|
-
/* WRITE on a readonly view (CARepeat stride-0 /
|
|
499
|
-
CAWrap
|
|
500
|
-
write. */
|
|
593
|
+
/* WRITE on a readonly view (CARepeat stride-0 / a lazy expression /
|
|
594
|
+
a read-only CAWrap / any view of a frozen or read-only array) --
|
|
595
|
+
explicit reject, would otherwise SEGV on write. */
|
|
501
596
|
if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
|
|
502
597
|
return CA_ITER_ERR_READONLY;
|
|
503
598
|
}
|
|
504
599
|
|
|
505
|
-
/*
|
|
506
|
-
CA_ITER_ERR_MASK
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
so the caller can decide whether to peel via .value /
|
|
511
|
-
.strip_mask(fill) or pick a mask-aware kernel. */
|
|
600
|
+
/* Masked sources are accepted; the mask travels with the slab, so
|
|
601
|
+
nothing returns CA_ITER_ERR_MASK. CA_KERNEL_NO_MASK is the kernel
|
|
602
|
+
declaring it cannot handle a mask: a masked source is then refused
|
|
603
|
+
with a dedicated code, so the caller can decide whether to peel via
|
|
604
|
+
.value / .strip_mask(fill) or pick a mask-aware kernel. */
|
|
512
605
|
if ( (flags & CA_KERNEL_NO_MASK) && ca_has_mask(src) ) {
|
|
513
606
|
return CA_ITER_ERR_MASK_NOT_ALLOWED;
|
|
514
607
|
}
|
|
515
608
|
|
|
516
|
-
/* Gate: classify source.
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
5.2 and Phase 2 respectively. */
|
|
609
|
+
/* Gate: classify source. Entity, CAStride family, descriptor views
|
|
610
|
+
and SRC_ATTACH views are accepted; a source the classifier does not
|
|
611
|
+
recognise (an externally installed view that has not registered a
|
|
612
|
+
kind) is refused. */
|
|
521
613
|
if ( ca_iter_classify_source(src) == CA_ITER_SRC_NONE ) {
|
|
522
614
|
return CA_ITER_ERR_NOT_CHEAP;
|
|
523
615
|
}
|
|
@@ -553,6 +645,29 @@ ca_iter_strip_storage_wrapper (CArray *src)
|
|
|
553
645
|
return src;
|
|
554
646
|
}
|
|
555
647
|
|
|
648
|
+
/* Raise unless a and b have the same shape. The INOUT and PAIR fiber
|
|
649
|
+
macros call it before opening either walk, so a refusal leaves nothing
|
|
650
|
+
to close. */
|
|
651
|
+
int
|
|
652
|
+
ca_iter_check_same_shape (struct _CArray *a, struct _CArray *b)
|
|
653
|
+
{
|
|
654
|
+
int8_t k;
|
|
655
|
+
int same = ( a->ndim == b->ndim );
|
|
656
|
+
for ( k = 0; same && k < a->ndim; k++ ) {
|
|
657
|
+
if ( a->dim[k] != b->dim[k] ) same = 0;
|
|
658
|
+
}
|
|
659
|
+
if ( ! same ) {
|
|
660
|
+
VALUE sa = rb_ary_new(), sb = rb_ary_new();
|
|
661
|
+
for ( k = 0; k < a->ndim; k++ ) rb_ary_push(sa, SIZE2NUM(a->dim[k]));
|
|
662
|
+
for ( k = 0; k < b->ndim; k++ ) rb_ary_push(sb, SIZE2NUM(b->dim[k]));
|
|
663
|
+
rb_raise(rb_eArgError,
|
|
664
|
+
"kernel iterator: the two arrays of a paired walk "
|
|
665
|
+
"differ in shape (%"PRIsVALUE" vs %"PRIsVALUE")",
|
|
666
|
+
rb_inspect(sa), rb_inspect(sb));
|
|
667
|
+
}
|
|
668
|
+
return 0;
|
|
669
|
+
}
|
|
670
|
+
|
|
556
671
|
int
|
|
557
672
|
ca_iter_check_init (int rc)
|
|
558
673
|
{
|
|
@@ -562,7 +677,7 @@ ca_iter_check_init (int rc)
|
|
|
562
677
|
|
|
563
678
|
switch ( rc ) {
|
|
564
679
|
case CA_ITER_ERR_NOT_CHEAP:
|
|
565
|
-
why = "the
|
|
680
|
+
why = "the iterator does not recognise this kind of source";
|
|
566
681
|
break;
|
|
567
682
|
case CA_ITER_ERR_POLICY:
|
|
568
683
|
why = "the slab policy or the axes given do not apply to this source";
|
|
@@ -587,6 +702,170 @@ ca_iter_check_init (int rc)
|
|
|
587
702
|
return rc; /* not reached */
|
|
588
703
|
}
|
|
589
704
|
|
|
705
|
+
/* Attaches a CAStack's parents and keeps their buffers. stack_n_parents
|
|
706
|
+
counts the parents attached so far, so an attach that raises leaves
|
|
707
|
+
finish with exactly those to detach (promise 4). The mask of a stack
|
|
708
|
+
reads its parts' masks, some perhaps through stand-ins: those give way
|
|
709
|
+
first, as the stack's own slots would let them. */
|
|
710
|
+
static void
|
|
711
|
+
ki_attach_stack_parents (ca_iter_state *st, CAStack *stack)
|
|
712
|
+
{
|
|
713
|
+
if ( stack->mask_owner != NULL ) {
|
|
714
|
+
CAStack *o = (CAStack *) stack->mask_owner;
|
|
715
|
+
ca_multi_parent_mask_refresh((CArray *) stack, o->parents,
|
|
716
|
+
o->mask_standins, o->n_parents);
|
|
717
|
+
}
|
|
718
|
+
st->stack_n_parents = 0;
|
|
719
|
+
st->stack_parent_ptrs =
|
|
720
|
+
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
721
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
722
|
+
ca_attach(stack->parents[kk]); /* window: finish detaches the counted parents */
|
|
723
|
+
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
724
|
+
st->stack_n_parents = kk + 1;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
/* Promise 2: the mask cursor a walk hands out is the walk's own copy of
|
|
729
|
+
the source's mask, taken when the walk opens -- one flag per cell, in
|
|
730
|
+
the view's row-major order -- never the live mask, so a write through
|
|
731
|
+
it reaches nothing. A source without a mask leaves alias_mask NULL.
|
|
732
|
+
finish frees the copy. */
|
|
733
|
+
static void
|
|
734
|
+
ki_take_mask_copy (ca_iter_state *st, CArray *src)
|
|
735
|
+
{
|
|
736
|
+
if ( ! ca_has_mask(src) ) return;
|
|
737
|
+
st->scratch_mask = (boolean8_t *) xmalloc(src->elements > 0 ? src->elements : 1);
|
|
738
|
+
if ( src->elements > 0 ) {
|
|
739
|
+
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
740
|
+
}
|
|
741
|
+
st->alias_mask = st->scratch_mask;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
/* Whole-view scratch. The view is gathered into the walk's scratch,
|
|
745
|
+
by the view's own transfer (by_copy = 0) or by ca_copy_data (by_copy =
|
|
746
|
+
1, the CAStride compose-fold). A WRITE walk sends back only the cells
|
|
747
|
+
the kernel changed (ki_scratch_push_changed), so the scratch is twice
|
|
748
|
+
the view and its second half keeps what was gathered: a cell the
|
|
749
|
+
kernel left alone must not be sent back, for it may be another copy
|
|
750
|
+
of a cell the kernel did write (a tile, a stack of one array with
|
|
751
|
+
itself) or a cell a conversion view would not bring back unchanged. */
|
|
752
|
+
static void
|
|
753
|
+
ki_gather_view (ca_iter_state *st, CArray *src, uint32_t flags, int by_copy)
|
|
754
|
+
{
|
|
755
|
+
ca_size_t view_bytes = (ca_size_t) src->elements * src->bytes;
|
|
756
|
+
st->scratch_cap = (flags & CA_KERNEL_WRITE) ? 2 * view_bytes : view_bytes;
|
|
757
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
758
|
+
if ( src->elements > 0 ) {
|
|
759
|
+
if ( by_copy ) {
|
|
760
|
+
ca_copy_data(src, st->scratch_ptr);
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
|
|
764
|
+
}
|
|
765
|
+
if ( flags & CA_KERNEL_WRITE ) {
|
|
766
|
+
memcpy(st->scratch_ptr + view_bytes, st->scratch_ptr, view_bytes);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/* Send back the cells of the current slab (the whole view for a WHOLE
|
|
772
|
+
walk) that differ from what was gathered, and record them as sent.
|
|
773
|
+
The slab of a CA_SLAB_AXES walk starts at last_data_off in the
|
|
774
|
+
scratch, which is laid out row-major over the view, so a cell's
|
|
775
|
+
offset divided by the cell size is its address in the view. */
|
|
776
|
+
static void
|
|
777
|
+
ki_scratch_push_changed (ca_iter_state *st)
|
|
778
|
+
{
|
|
779
|
+
CArray *src = st->src;
|
|
780
|
+
ca_size_t bytes = src->bytes;
|
|
781
|
+
ca_size_t view_bytes = (ca_size_t) src->elements * bytes;
|
|
782
|
+
char *now = st->scratch_ptr;
|
|
783
|
+
char *was = st->scratch_ptr + view_bytes;
|
|
784
|
+
int whole = ( st->level == 1 ); /* L1 is the whole array */
|
|
785
|
+
ca_size_t n = whole ? src->elements : st->slab_elements;
|
|
786
|
+
ca_size_t idx[CA_RANK_MAX];
|
|
787
|
+
ca_size_t *addrs;
|
|
788
|
+
char *vals;
|
|
789
|
+
ca_size_t n_changed = 0;
|
|
790
|
+
ca_size_t i;
|
|
791
|
+
int8_t k;
|
|
792
|
+
volatile VALUE h_addrs = 0, h_vals = 0;
|
|
793
|
+
|
|
794
|
+
if ( n <= 0 ) return;
|
|
795
|
+
|
|
796
|
+
addrs = ALLOCV_N(ca_size_t, h_addrs, n);
|
|
797
|
+
vals = (char *) ALLOCV(h_vals, n * bytes);
|
|
798
|
+
for ( k = 0; k < st->slab_ndim; k++ ) idx[k] = 0;
|
|
799
|
+
|
|
800
|
+
for ( i = 0; i < n; i++ ) {
|
|
801
|
+
ca_size_t off;
|
|
802
|
+
if ( whole ) {
|
|
803
|
+
off = i * bytes;
|
|
804
|
+
}
|
|
805
|
+
else {
|
|
806
|
+
off = st->last_data_off;
|
|
807
|
+
for ( k = 0; k < st->slab_ndim; k++ ) off += idx[k] * st->slab_strides[k];
|
|
808
|
+
for ( k = st->slab_ndim - 1; k >= 0; k-- ) {
|
|
809
|
+
if ( ++idx[k] < st->slab_dims[k] ) break;
|
|
810
|
+
idx[k] = 0;
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
if ( memcmp(now + off, was + off, bytes) != 0 ) {
|
|
814
|
+
memcpy(was + off, now + off, bytes);
|
|
815
|
+
memcpy(vals + n_changed * bytes, now + off, bytes);
|
|
816
|
+
addrs[n_changed++] = off / bytes;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/* Shortcut PUSH_ALL_CHANGED.
|
|
821
|
+
Layer write-back.
|
|
822
|
+
Speeds up a whole-view write walk that changed every cell: one
|
|
823
|
+
ca_xfer_all PUT instead of a list of all the addresses.
|
|
824
|
+
Why about 5% on a whole write of a fake view of 1M cells.
|
|
825
|
+
Enters whole && n_changed == n.
|
|
826
|
+
Promises 3 (the same cells go back; the list would hold them all).
|
|
827
|
+
Switch ca_fast_paths_enabled(): off, the address list.
|
|
828
|
+
Checked by the write rows of the delivery matrix (L1 / L2 whole). */
|
|
829
|
+
if ( whole && n_changed == n && ca_fast_paths_enabled() ) {
|
|
830
|
+
ca_xfer_all(src, now, CA_XFER_PUT);
|
|
831
|
+
}
|
|
832
|
+
else if ( n_changed > 0 ) {
|
|
833
|
+
ca_xfer_addrs(src, n_changed, addrs, vals, CA_XFER_PUT);
|
|
834
|
+
}
|
|
835
|
+
ALLOCV_END(h_vals);
|
|
836
|
+
ALLOCV_END(h_addrs);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/* The transfer walk: the whole view gathered into scratch through the
|
|
840
|
+
view's own transfer, its mask likewise, and yielded as one contiguous
|
|
841
|
+
slab; on WRITE, sync_slab sends the changed cells back through the same
|
|
842
|
+
transfer (ki_scratch_push_changed).
|
|
843
|
+
|
|
844
|
+
Every promise the iterator makes is kept here by the view itself --
|
|
845
|
+
what a cell is, where its mask flag lives, which root cell a write
|
|
846
|
+
lands on -- so this walk is the reference the other routes are checked
|
|
847
|
+
against (CARRAY_GENERAL_PATH=2 sends every source through it). It is
|
|
848
|
+
also the route for a view the classifier knows nothing more about.
|
|
849
|
+
|
|
850
|
+
The caller has filled src / src_kind / level / policy / ndim / flags /
|
|
851
|
+
bytes / naxes. */
|
|
852
|
+
static void
|
|
853
|
+
ki_open_by_transfer (ca_iter_state *st, CArray *src, uint32_t flags)
|
|
854
|
+
{
|
|
855
|
+
ki_gather_view(st, src, flags, 0);
|
|
856
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
857
|
+
st->alias_ptr = st->scratch_ptr;
|
|
858
|
+
st->composed_strides[0] = src->bytes;
|
|
859
|
+
st->composed_base = 0;
|
|
860
|
+
st->slab_n = src->elements;
|
|
861
|
+
st->total_slabs = 1;
|
|
862
|
+
st->slabs_emitted = 0;
|
|
863
|
+
st->chunk_size = st->slab_n;
|
|
864
|
+
st->outer_idx = NULL;
|
|
865
|
+
|
|
866
|
+
ki_take_mask_copy(st, src);
|
|
867
|
+
}
|
|
868
|
+
|
|
590
869
|
static int
|
|
591
870
|
ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
592
871
|
struct _CArray *src,
|
|
@@ -601,10 +880,11 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
601
880
|
times rather than whatever its stack frame happened to contain. */
|
|
602
881
|
if ( st == NULL ) return CA_ITER_ERR_FLAGS;
|
|
603
882
|
memset(st, 0, sizeof(*st));
|
|
883
|
+
if ( src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
604
884
|
|
|
605
|
-
/*
|
|
606
|
-
|
|
607
|
-
|
|
885
|
+
/* A Face or a lazy marker wraps storage identical to its parent, so it
|
|
886
|
+
is stripped at entry (the same reason as in init_l2 below), before
|
|
887
|
+
validate_inputs.
|
|
608
888
|
|
|
609
889
|
The WRITE rejection is taken against the wrapper, not what it wraps: a
|
|
610
890
|
CALazyMarker is read-only while its parent is not, and stripping first
|
|
@@ -620,12 +900,32 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
620
900
|
int rc = ca_iter_validate_inputs(st, src, policy, flags);
|
|
621
901
|
if ( rc != CA_ITER_OK ) return rc;
|
|
622
902
|
|
|
623
|
-
/*
|
|
624
|
-
|
|
625
|
-
misdispatch. */
|
|
903
|
+
/* CA_SLAB_AXES is L2 only (a kernel that wants a K-D slab uses
|
|
904
|
+
init_l2); init_l1 refuses it rather than walking the whole array. */
|
|
626
905
|
if ( policy == CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
|
|
627
906
|
|
|
628
|
-
|
|
907
|
+
/* The same routing as init_l2: a descriptor view whose parent lends no
|
|
908
|
+
memory, or a grid with holes, takes the transfer walk. An L1 walk
|
|
909
|
+
yields the whole view as one slab, so a descriptor view the L2 walk
|
|
910
|
+
could borrow is gathered here like any other descriptor view. */
|
|
911
|
+
ca_axis_desc_t route_descs[CA_RANK_MAX];
|
|
912
|
+
ca_size_t route_pdims[CA_RANK_MAX];
|
|
913
|
+
int8_t route_ndim = 0;
|
|
914
|
+
uint8_t src_kind = ca_iter_route_source(src, route_descs, route_pdims,
|
|
915
|
+
&route_ndim);
|
|
916
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
917
|
+
src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
918
|
+
}
|
|
919
|
+
/* A write over a descriptor view takes the transfer walk, as an L2
|
|
920
|
+
write over an INDEX or SHIFT slab axis does. Gathering with the
|
|
921
|
+
descriptor engine would leave nothing to send back but the whole
|
|
922
|
+
view, and a view that shows a root cell twice (a grid with a
|
|
923
|
+
repeated index) would then let the copy the kernel did not write
|
|
924
|
+
overwrite the one it did (promise 3: only the changed cells go
|
|
925
|
+
back). */
|
|
926
|
+
if ( src_kind == CA_ITER_SRC_DESCRIPTOR && (flags & CA_KERNEL_WRITE) ) {
|
|
927
|
+
src_kind = CA_ITER_SRC_ATTACH;
|
|
928
|
+
}
|
|
629
929
|
|
|
630
930
|
st->src = src;
|
|
631
931
|
st->src_kind = src_kind;
|
|
@@ -634,7 +934,6 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
634
934
|
st->ndim = src->ndim;
|
|
635
935
|
st->flags = flags;
|
|
636
936
|
st->bytes = src->bytes;
|
|
637
|
-
st->axes = axes;
|
|
638
937
|
st->naxes = naxes;
|
|
639
938
|
|
|
640
939
|
st->slab_n = src->elements;
|
|
@@ -643,45 +942,18 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
643
942
|
st->chunk_size = st->slab_n;
|
|
644
943
|
|
|
645
944
|
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
646
|
-
|
|
647
|
-
CAByteSwap / CABitfield / CABitarray / CAReduce =================
|
|
648
|
-
View's xfer_all delivers materialised data through the unified
|
|
649
|
-
dispatch surface. Previously used `ca_attach(src) + src->ptr`
|
|
650
|
-
(= view's attach slot) and `ca_sync(src)` for WRITE; now uses
|
|
651
|
-
`ca_xfer_all(src, scratch, GET/PUT)` with iterator-owned scratch.
|
|
652
|
-
Decouples kernel_iterator from per-view attach/sync lifecycle
|
|
653
|
-
and inherits xfer reform improvements (transform-fused, etc.)
|
|
654
|
-
automatically. */
|
|
655
|
-
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
656
|
-
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
657
|
-
if ( src->elements > 0 ) {
|
|
658
|
-
ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
|
|
659
|
-
}
|
|
660
|
-
st->alias_mode = CA_ITER_ALIAS_NONE; /* scratch-owned, no src.detach */
|
|
661
|
-
st->alias_ptr = st->scratch_ptr;
|
|
662
|
-
st->composed_strides[0] = src->bytes;
|
|
663
|
-
st->composed_base = 0;
|
|
664
|
-
st->outer_idx = NULL;
|
|
665
|
-
/* Step 6: mask gather (xfer_all on mask -- ca_copy_data is a thin
|
|
666
|
-
wrapper, this just removes the extra hop). */
|
|
667
|
-
if ( ca_has_mask(src) ) {
|
|
668
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
669
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
670
|
-
if ( src->elements > 0 ) {
|
|
671
|
-
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
672
|
-
}
|
|
673
|
-
st->alias_mask = st->scratch_mask;
|
|
674
|
-
}
|
|
945
|
+
ki_open_by_transfer(st, src, flags);
|
|
675
946
|
return CA_ITER_OK;
|
|
676
947
|
}
|
|
677
948
|
|
|
678
949
|
if ( src_kind == CA_ITER_SRC_DESCRIPTOR ) {
|
|
679
|
-
/* === descriptor path (
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
yield one contig slab
|
|
683
|
-
|
|
684
|
-
per-axis
|
|
950
|
+
/* === descriptor path (CASelectAxis / CAGrid / CASelect / CAWindow /
|
|
951
|
+
CAShift) ==========================================================
|
|
952
|
+
describe_axes → prepare → layout, then gather the view into
|
|
953
|
+
scratch with ca_axis_dispatch_gather and yield one contig slab
|
|
954
|
+
(alias_mode = NONE). The kernel sees a flat slab whatever the
|
|
955
|
+
per-axis kinds are; an INDEX / SHIFT mix is handled inside the
|
|
956
|
+
engine. */
|
|
685
957
|
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
686
958
|
int8_t raw_ndim = 0;
|
|
687
959
|
ca_iter_describe_axes(src, raw_descs, st->parent_axis_dims, &raw_ndim);
|
|
@@ -699,49 +971,34 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
699
971
|
}
|
|
700
972
|
st->total_elements = src->elements;
|
|
701
973
|
|
|
702
|
-
/* Materialise
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
974
|
+
/* Materialise: attach the parent here, allocate the scratch, and
|
|
975
|
+
gather into it with ca_axis_dispatch_gather (INDEX axes included);
|
|
976
|
+
the iterator owns the buffer. Descriptor views inherit from CAView
|
|
977
|
+
so parent is at the CAVIEW prefix slot. */
|
|
706
978
|
CArray *parent = CAVIEW(src)->parent;
|
|
707
979
|
ca_attach(parent); /* engine reads parent->ptr */
|
|
708
980
|
st->root = parent; /* finish() detaches */
|
|
709
981
|
st->scratch_cap = src->elements * src->bytes;
|
|
710
|
-
/*
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
bound_fill = CAWindow/CAShift fill value (Tier 2.B SHIFT-axis
|
|
714
|
-
OOB cell), NULL for non-window views (CSA / CAGrid / CASelect /
|
|
715
|
-
CAMapping don't have OOB semantics — engine sees no SHIFT axes). */
|
|
982
|
+
/* bound_fill = the CAWindow / CAShift fill value for an out-of-bounds
|
|
983
|
+
SHIFT cell; NULL for the other descriptor views, which have no
|
|
984
|
+
SHIFT axes. */
|
|
716
985
|
const void *bound_fill = NULL;
|
|
717
986
|
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
718
987
|
bound_fill = ((CAWindow *) src)->fill;
|
|
719
988
|
}
|
|
720
|
-
st->
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
bound_fill);
|
|
989
|
+
st->scratch_cap = st->total_elements * src->bytes;
|
|
990
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
991
|
+
ca_axis_dispatch_gather(parent, st->parent_axis_dims, raw_descs, raw_ndim,
|
|
992
|
+
src->bytes, st->total_elements, bound_fill,
|
|
993
|
+
st->scratch_ptr);
|
|
726
994
|
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
727
995
|
st->alias_ptr = st->scratch_ptr;
|
|
728
996
|
|
|
729
|
-
|
|
730
|
-
all descriptor views — mask propagates through their
|
|
731
|
-
func_copy_data via the descriptor framework's own mask
|
|
732
|
-
handling). */
|
|
733
|
-
if ( ca_has_mask(src) ) {
|
|
734
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
735
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
736
|
-
if ( src->elements > 0 ) {
|
|
737
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
738
|
-
}
|
|
739
|
-
st->alias_mask = st->scratch_mask;
|
|
740
|
-
}
|
|
997
|
+
ki_take_mask_copy(st, src);
|
|
741
998
|
return CA_ITER_OK;
|
|
742
999
|
}
|
|
743
1000
|
|
|
744
|
-
/* === CAStride path
|
|
1001
|
+
/* === CAStride path ========================================= */
|
|
745
1002
|
if ( ca_iter_can_alias(src, 1) ) {
|
|
746
1003
|
/* alias path: entity or CAStride contig */
|
|
747
1004
|
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
@@ -752,1430 +1009,692 @@ ca_iter_state_init_l1_unprotected (ca_iter_state *st,
|
|
|
752
1009
|
to ca_stride_func_copy_data which composes leaf strides up to
|
|
753
1010
|
the root entity and gathers in one pass — no per-intermediate
|
|
754
1011
|
view materialise. */
|
|
755
|
-
|
|
756
|
-
st->scratch_cap = cap;
|
|
757
|
-
st->scratch_ptr = xmalloc(cap > 0 ? cap : 1);
|
|
758
|
-
if ( cap > 0 ) {
|
|
759
|
-
ca_copy_data(src, st->scratch_ptr);
|
|
760
|
-
}
|
|
1012
|
+
ki_gather_view(st, src, flags, 1);
|
|
761
1013
|
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
762
1014
|
st->alias_ptr = st->scratch_ptr;
|
|
763
1015
|
}
|
|
764
1016
|
|
|
765
|
-
|
|
766
|
-
We always gather into a contig boolean8_t buffer for uniformity
|
|
767
|
-
across alias / scratch / descriptor paths — the kernel sees a
|
|
768
|
-
simple `boolean8_t *` aligned with the value slab. Future
|
|
769
|
-
optimisation: alias mask directly for CAStride contig. Mask is
|
|
770
|
-
informational; kernel uses CA_FOR_EACH_UNMASKED macros to skip
|
|
771
|
-
masked cells. */
|
|
772
|
-
if ( ca_has_mask(src) ) {
|
|
773
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
774
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
775
|
-
if ( src->elements > 0 ) {
|
|
776
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
777
|
-
}
|
|
778
|
-
st->alias_mask = st->scratch_mask;
|
|
779
|
-
}
|
|
1017
|
+
ki_take_mask_copy(st, src);
|
|
780
1018
|
return CA_ITER_OK;
|
|
781
1019
|
}
|
|
782
1020
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
/*
|
|
799
|
-
|
|
1021
|
+
/* ---- L2 walks: choosing a route, then building the state for it --------
|
|
1022
|
+
|
|
1023
|
+
init_l2 checks the request, classifies the source and chooses one of the
|
|
1024
|
+
routes below (ki_choose_l2_route, which only decides). ki_open_l2 then
|
|
1025
|
+
builds the state for that route: the part only that route needs, the
|
|
1026
|
+
axis layout every route shares (ki_lay_out_slabs) and the mask. So the
|
|
1027
|
+
state is assembled in one place and init_l2 has one way out. */
|
|
1028
|
+
|
|
1029
|
+
typedef enum {
|
|
1030
|
+
KI_L2_FIBER_FUSED, /* PER_FIBER_FUSED: a CAWindow / CAShift fiber, or a
|
|
1031
|
+
read of a conversion view's innermost fiber */
|
|
1032
|
+
KI_L2_PER_SLAB, /* descriptor view, read, a slab axis INDEX or SHIFT:
|
|
1033
|
+
one slab gathered at a time (PER_SLAB / _HOIST) */
|
|
1034
|
+
KI_L2_DESC_WHOLE, /* descriptor view, read: the whole view gathered by
|
|
1035
|
+
the descriptor engine (outer SHIFT, SHIFT_WHOLE_VIEW) */
|
|
1036
|
+
KI_L2_DESC_STRIDED, /* descriptor view whose slab axes are all STRIDE:
|
|
1037
|
+
the parent's memory is borrowed */
|
|
1038
|
+
KI_L2_STACK, /* CAStack read along its stack axis */
|
|
1039
|
+
KI_L2_STACK_OUTER_K, /* CAStack read with the stack axis outside the slab */
|
|
1040
|
+
KI_L2_TRANSFER, /* the whole view gathered by its own transfer */
|
|
1041
|
+
KI_L2_ROOT_STRIDED, /* CAStride: the root walked through composed strides */
|
|
1042
|
+
KI_L2_VIEW_SCRATCH, /* CAStride over a root that lends no memory */
|
|
1043
|
+
KI_L2_CONTIG /* entity, or a contiguous CAStride */
|
|
1044
|
+
} ki_l2_route;
|
|
800
1045
|
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
1046
|
+
typedef struct {
|
|
1047
|
+
ki_l2_route route;
|
|
1048
|
+
uint8_t src_kind; /* what st->src_kind records */
|
|
1049
|
+
int8_t in_slab[CA_RANK_MAX]; /* 1 for each slab axis */
|
|
1050
|
+
ca_axis_desc_t descs[CA_RANK_MAX]; /* descriptor routes: the view's */
|
|
1051
|
+
ca_size_t pdims[CA_RANK_MAX]; /* axes and the parent's dims */
|
|
1052
|
+
int8_t ndesc;
|
|
1053
|
+
CArray *root; /* KI_L2_ROOT_STRIDED */
|
|
1054
|
+
ca_size_t composed[CA_RANK_MAX];
|
|
1055
|
+
ca_size_t base;
|
|
1056
|
+
} ki_l2_plan;
|
|
1057
|
+
|
|
1058
|
+
/* Per view axis: its extent, its byte stride when it is a slab axis, its
|
|
1059
|
+
byte stride when it is an outer axis, and its mask element stride. */
|
|
1060
|
+
typedef struct {
|
|
1061
|
+
ca_size_t dims[CA_RANK_MAX];
|
|
1062
|
+
ca_size_t slab_strides[CA_RANK_MAX];
|
|
1063
|
+
ca_size_t outer_strides[CA_RANK_MAX];
|
|
1064
|
+
ca_size_t mask_strides[CA_RANK_MAX];
|
|
1065
|
+
} ki_axis_layout;
|
|
810
1066
|
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
1067
|
+
/* Row-major byte and element strides over the view's own shape. */
|
|
1068
|
+
static void
|
|
1069
|
+
ki_view_row_major (CArray *src, ca_size_t *bytes, ca_size_t *elems)
|
|
1070
|
+
{
|
|
1071
|
+
ca_size_t b = src->bytes, e = 1;
|
|
1072
|
+
for ( int8_t k = src->ndim - 1; k >= 0; k-- ) {
|
|
1073
|
+
if ( bytes ) bytes[k] = b;
|
|
1074
|
+
if ( elems ) elems[k] = e;
|
|
1075
|
+
b *= src->dim[k];
|
|
1076
|
+
e *= src->dim[k];
|
|
819
1077
|
}
|
|
1078
|
+
}
|
|
820
1079
|
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
int8_t in_slab[CA_RANK_MAX];
|
|
849
|
-
int8_t k;
|
|
850
|
-
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
851
|
-
for ( k = 0; k < naxes; k++ ) {
|
|
852
|
-
int8_t ax = axes[k];
|
|
853
|
-
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
854
|
-
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
855
|
-
in_slab[ax] = 1;
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
/* PROPOSAL_FIBER_PER_SOURCE_PATH F.6.3 hook (= parallel to
|
|
859
|
-
F.6.2 SRC_ATTACH hook): for descriptor views CAWindow /
|
|
860
|
-
CAShift (predicate returns 1 unconditionally per F.5 always-
|
|
861
|
-
win bench), skip materialise (alias / C.1 PER_SLAB / B.1.5)
|
|
862
|
-
and use per-fiber ca_xfer_stride which routes to the view's
|
|
863
|
-
X.1 OOB-fused per-region fast path. */
|
|
864
|
-
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
865
|
-
&& raw_ndim == src->ndim ) {
|
|
866
|
-
int8_t fiber_ax = axes[0];
|
|
867
|
-
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
868
|
-
{
|
|
869
|
-
ca_size_t b = src->bytes;
|
|
870
|
-
for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
|
|
871
|
-
row_byte_strides[kk] = b;
|
|
872
|
-
b *= src->dim[kk];
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
|
|
876
|
-
row_byte_strides[fiber_ax],
|
|
877
|
-
flags) ) {
|
|
878
|
-
st->src = src;
|
|
879
|
-
st->src_kind = src_kind;
|
|
880
|
-
st->level = 2;
|
|
881
|
-
st->policy = policy;
|
|
882
|
-
st->ndim = src->ndim;
|
|
883
|
-
st->flags = flags;
|
|
884
|
-
st->bytes = src->bytes;
|
|
885
|
-
st->axes = axes;
|
|
886
|
-
st->naxes = naxes;
|
|
887
|
-
|
|
888
|
-
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
889
|
-
st->alias_ptr = NULL;
|
|
890
|
-
st->fiber_axis = fiber_ax;
|
|
891
|
-
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
892
|
-
st->fiber_native_strides[kk] = row_byte_strides[kk];
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
/* Row-major element strides for mask (= identity element
|
|
896
|
-
index strides over src->dim). */
|
|
897
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
898
|
-
{
|
|
899
|
-
ca_size_t e = 1;
|
|
900
|
-
for ( int8_t kk = src->ndim - 1; kk >= 0; kk-- ) {
|
|
901
|
-
row_elem_strides[kk] = e;
|
|
902
|
-
e *= src->dim[kk];
|
|
903
|
-
}
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
int8_t sp = 0, op = 0;
|
|
907
|
-
st->slab_elements = 1;
|
|
908
|
-
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
909
|
-
if ( in_slab[kk] ) {
|
|
910
|
-
st->slab_axes_buf[sp] = kk;
|
|
911
|
-
st->slab_dims[sp] = src->dim[kk];
|
|
912
|
-
st->slab_strides[sp] = row_byte_strides[kk];
|
|
913
|
-
st->slab_mask_strides[sp] = row_elem_strides[kk];
|
|
914
|
-
st->slab_elements *= src->dim[kk];
|
|
915
|
-
sp++;
|
|
916
|
-
} else {
|
|
917
|
-
st->outer_axes[op] = kk;
|
|
918
|
-
st->outer_dims[op] = src->dim[kk];
|
|
919
|
-
st->outer_strides[op] = row_byte_strides[kk];
|
|
920
|
-
st->outer_mask_strides[op] = row_elem_strides[kk];
|
|
921
|
-
op++;
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
st->slab_ndim = sp;
|
|
925
|
-
st->outer_ndim = op;
|
|
926
|
-
st->desc_ndim = 0;
|
|
927
|
-
|
|
928
|
-
ca_size_t total = 1;
|
|
929
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
930
|
-
st->total_slabs = total;
|
|
931
|
-
st->slab_n = st->slab_elements;
|
|
932
|
-
st->slabs_emitted = 0;
|
|
933
|
-
st->chunk_size = st->slab_n;
|
|
934
|
-
|
|
935
|
-
if ( st->outer_ndim > 0 ) {
|
|
936
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
937
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
938
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
939
|
-
} else {
|
|
940
|
-
st->outer_idx = NULL;
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
/* Mask: per-fiber gather via next_slab_axes; no whole-view
|
|
944
|
-
materialise (rev2 §3.3, (data,mask) pair travels together). */
|
|
945
|
-
st->alias_mask = NULL;
|
|
946
|
-
return CA_ITER_OK;
|
|
947
|
-
}
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
/* Phase C T3 (C.1): slab axis has INDEX or SHIFT kind → per-slab
|
|
951
|
-
materialise fallback path (D1.1 (B) + D1.2 (A)). CAMapping-style
|
|
952
|
-
views (raw_ndim != src->ndim) are out of C.1 scope — descriptor
|
|
953
|
-
exposes a flat 1-D INDEX gather that doesn't expose the view-
|
|
954
|
-
axis partition the user gave via slab_axes. Reject explicitly. */
|
|
955
|
-
if ( raw_ndim != src->ndim ) {
|
|
956
|
-
return CA_ITER_ERR_POLICY;
|
|
957
|
-
}
|
|
958
|
-
int slab_has_non_stride = 0;
|
|
959
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
960
|
-
if ( in_slab[k] && raw_descs[k].kind != CA_AXIS_KIND_STRIDE ) {
|
|
961
|
-
slab_has_non_stride = 1;
|
|
962
|
-
break;
|
|
963
|
-
}
|
|
964
|
-
}
|
|
965
|
-
if ( slab_has_non_stride ) {
|
|
966
|
-
/* C.1 scope: READ-only. WRITE = future sub-step (C.1c). */
|
|
967
|
-
if ( flags & CA_KERNEL_WRITE ) {
|
|
968
|
-
return CA_ITER_ERR_FLAGS;
|
|
969
|
-
}
|
|
970
|
-
|
|
971
|
-
st->src = src;
|
|
972
|
-
st->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
973
|
-
st->level = 2;
|
|
974
|
-
st->policy = policy;
|
|
975
|
-
st->ndim = src->ndim;
|
|
976
|
-
st->flags = flags;
|
|
977
|
-
st->bytes = src->bytes;
|
|
978
|
-
st->axes = axes;
|
|
979
|
-
st->naxes = naxes;
|
|
980
|
-
|
|
981
|
-
CArray *parent = CAVIEW(src)->parent;
|
|
982
|
-
ca_attach(parent);
|
|
983
|
-
st->root = parent;
|
|
984
|
-
|
|
985
|
-
/* Persist raw_descs / pdims / pstrides for per-slab subset
|
|
986
|
-
construction (PER_SLAB fallback) or for the hoisted manual
|
|
987
|
-
gather (PER_SLAB_HOIST specialised path). desc_ndim =
|
|
988
|
-
src->ndim so has_descs branch in next_slab_axes can read
|
|
989
|
-
st->descs (though T3 path takes its own branch before that
|
|
990
|
-
check). */
|
|
991
|
-
memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
992
|
-
memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
|
|
993
|
-
{
|
|
994
|
-
ca_size_t s = src->bytes;
|
|
995
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
996
|
-
st->pstrides[k] = s;
|
|
997
|
-
s *= raw_pdims[k];
|
|
998
|
-
}
|
|
999
|
-
}
|
|
1000
|
-
st->desc_ndim = src->ndim;
|
|
1001
|
-
|
|
1002
|
-
/* View-row-major element strides (= mask scratch layout, since
|
|
1003
|
-
mask is gathered whole-view once at init). */
|
|
1004
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1005
|
-
{
|
|
1006
|
-
ca_size_t e = 1;
|
|
1007
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1008
|
-
row_elem_strides[k] = e;
|
|
1009
|
-
e *= src->dim[k];
|
|
1010
|
-
}
|
|
1011
|
-
}
|
|
1012
|
-
|
|
1013
|
-
/* Row-major byte strides over SLAB axes only (= scratch_ptr
|
|
1014
|
-
layout, since scratch_ptr is sized slab_elements × bytes
|
|
1015
|
-
and refilled per outer iter as a contig row-major slab). */
|
|
1016
|
-
ca_size_t slab_data_strides[CA_RANK_MAX];
|
|
1017
|
-
{
|
|
1018
|
-
ca_size_t b = src->bytes;
|
|
1019
|
-
ca_size_t e_slab = 1;
|
|
1020
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1021
|
-
if ( in_slab[k] ) {
|
|
1022
|
-
slab_data_strides[k] = b;
|
|
1023
|
-
b *= raw_descs[k].count;
|
|
1024
|
-
e_slab *= raw_descs[k].count;
|
|
1025
|
-
} else {
|
|
1026
|
-
slab_data_strides[k] = 0; /* unused for outer axes */
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
1029
|
-
st->slab_elements = e_slab;
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
int8_t sp = 0, op = 0;
|
|
1033
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1034
|
-
if ( in_slab[k] ) {
|
|
1035
|
-
st->slab_axes_buf[sp] = k;
|
|
1036
|
-
st->slab_dims[sp] = raw_descs[k].count;
|
|
1037
|
-
st->slab_strides[sp] = slab_data_strides[k];
|
|
1038
|
-
/* Slab mask stride is view-row-major elem stride along
|
|
1039
|
-
this view axis (mask scratch is whole-view layout). */
|
|
1040
|
-
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
1041
|
-
sp++;
|
|
1042
|
-
} else {
|
|
1043
|
-
st->outer_axes[op] = k;
|
|
1044
|
-
st->outer_dims[op] = raw_descs[k].count;
|
|
1045
|
-
/* T3 path: outer data offset is always 0 (scratch refilled
|
|
1046
|
-
per slab, ptr = scratch_ptr). */
|
|
1047
|
-
st->outer_strides[op] = 0;
|
|
1048
|
-
/* Outer mask offset uses view-row-major elem stride into
|
|
1049
|
-
whole-view mask scratch. */
|
|
1050
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1051
|
-
op++;
|
|
1052
|
-
}
|
|
1053
|
-
}
|
|
1054
|
-
st->slab_ndim = sp;
|
|
1055
|
-
st->outer_ndim = op;
|
|
1056
|
-
st->composed_base = 0; /* unused for T3 fallback */
|
|
1057
|
-
|
|
1058
|
-
ca_size_t total = 1;
|
|
1059
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1060
|
-
st->total_slabs = total;
|
|
1061
|
-
st->slab_n = st->slab_elements;
|
|
1062
|
-
st->slabs_emitted = 0;
|
|
1063
|
-
st->chunk_size = st->slab_n;
|
|
1064
|
-
|
|
1065
|
-
if ( st->outer_ndim > 0 ) {
|
|
1066
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1067
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1068
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1069
|
-
} else {
|
|
1070
|
-
st->outer_idx = NULL;
|
|
1071
|
-
}
|
|
1072
|
-
|
|
1073
|
-
/* Alloc per-slab data scratch (D1.1 (B): one max-slab buffer
|
|
1074
|
-
refilled per outer iter, instead of per-iter alloc/free or
|
|
1075
|
-
full-view materialise). */
|
|
1076
|
-
ca_size_t slab_bytes = st->slab_elements * src->bytes;
|
|
1077
|
-
st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
|
|
1078
|
-
st->scratch_ptr = (char *) xmalloc(st->scratch_cap);
|
|
1079
|
-
st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
|
|
1080
|
-
|
|
1081
|
-
/* Specialisation (B-1b, C.1b): if innermost slab axis is STRIDE
|
|
1082
|
-
and no SHIFT axes anywhere in the view, use the hoisted
|
|
1083
|
-
manual gather path (= outer + non-innermost-slab hoist,
|
|
1084
|
-
inner pure STRIDE linear memcpy). Else (innermost INDEX /
|
|
1085
|
-
SHIFT-anywhere): use the engine fallback (A). */
|
|
1086
|
-
int specialised_eligible = (sp > 0);
|
|
1087
|
-
if ( specialised_eligible ) {
|
|
1088
|
-
int8_t inner_view_ax = st->slab_axes_buf[sp - 1];
|
|
1089
|
-
if ( raw_descs[inner_view_ax].kind != CA_AXIS_KIND_STRIDE ) {
|
|
1090
|
-
specialised_eligible = 0;
|
|
1091
|
-
}
|
|
1092
|
-
}
|
|
1093
|
-
if ( specialised_eligible ) {
|
|
1094
|
-
/* Any SHIFT axis (slab or outer) → fallback to engine for
|
|
1095
|
-
clean bound_fill semantics in C.1b scope. */
|
|
1096
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1097
|
-
if ( raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1098
|
-
specialised_eligible = 0;
|
|
1099
|
-
break;
|
|
1100
|
-
}
|
|
1101
|
-
}
|
|
1102
|
-
}
|
|
1103
|
-
st->alias_mode = specialised_eligible
|
|
1104
|
-
? CA_ITER_ALIAS_PER_SLAB_HOIST
|
|
1105
|
-
: CA_ITER_ALIAS_PER_SLAB;
|
|
1106
|
-
|
|
1107
|
-
/* Mask: gather whole view mask once into scratch_mask (= same
|
|
1108
|
-
strategy as B.1.5). Per-slab mask offset is computed via
|
|
1109
|
-
outer_idx × outer_mask_strides in next_slab_axes. */
|
|
1110
|
-
if ( ca_has_mask(src) ) {
|
|
1111
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1112
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1113
|
-
if ( src->elements > 0 ) {
|
|
1114
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
1115
|
-
}
|
|
1116
|
-
st->alias_mask = st->scratch_mask;
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
return CA_ITER_OK;
|
|
1120
|
-
}
|
|
1121
|
-
/* slab is all-STRIDE: fall through to B.1.5 / Phase B paths. */
|
|
1122
|
-
/* Phase B.1.5: outer SHIFT axis → materialise downgrade. Alias
|
|
1123
|
-
can't deliver OOB cells (= need fill_value), and the view-family
|
|
1124
|
-
surface prioritises delivering the cells over avoiding a copy, so
|
|
1125
|
-
we materialise the entire view into a row-major scratch
|
|
1126
|
-
buffer via ca_axis_dispatch_attach (= same engine as the
|
|
1127
|
-
existing SRC_DESCRIPTOR L2 NONE path), then walk it with
|
|
1128
|
-
Phase A-style row-major K-D strides. Slab / outer partition
|
|
1129
|
-
applies to view-axes; the scratch IS the flat row-major view. */
|
|
1130
|
-
int outer_has_shift = 0;
|
|
1131
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1132
|
-
if ( !in_slab[k] && raw_descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1133
|
-
outer_has_shift = 1;
|
|
1134
|
-
break;
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1080
|
+
/* Splits the view's axes into slab and outer axes (both in ascending axis
|
|
1081
|
+
order, so the user's order does not matter), counts the slabs and opens
|
|
1082
|
+
the outer cursor. */
|
|
1083
|
+
static void
|
|
1084
|
+
ki_lay_out_slabs (ca_iter_state *st, CArray *src, const int8_t *in_slab,
|
|
1085
|
+
const ki_axis_layout *lay)
|
|
1086
|
+
{
|
|
1087
|
+
int8_t sp = 0, op = 0;
|
|
1088
|
+
st->slab_elements = 1;
|
|
1089
|
+
for ( int8_t k = 0; k < src->ndim; k++ ) {
|
|
1090
|
+
if ( in_slab[k] ) {
|
|
1091
|
+
st->slab_axes_buf[sp] = k;
|
|
1092
|
+
st->slab_dims[sp] = lay->dims[k];
|
|
1093
|
+
st->slab_strides[sp] = lay->slab_strides[k];
|
|
1094
|
+
st->slab_mask_strides[sp] = lay->mask_strides[k];
|
|
1095
|
+
st->slab_elements *= lay->dims[k];
|
|
1096
|
+
sp++;
|
|
1097
|
+
} else {
|
|
1098
|
+
st->outer_axes[op] = k;
|
|
1099
|
+
st->outer_dims[op] = lay->dims[k];
|
|
1100
|
+
st->outer_strides[op] = lay->outer_strides[k];
|
|
1101
|
+
st->outer_mask_strides[op] = lay->mask_strides[k];
|
|
1102
|
+
op++;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
st->slab_ndim = sp;
|
|
1106
|
+
st->outer_ndim = op;
|
|
1137
1107
|
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
st->flags = flags;
|
|
1145
|
-
st->bytes = src->bytes;
|
|
1146
|
-
st->axes = axes;
|
|
1147
|
-
st->naxes = naxes;
|
|
1148
|
-
|
|
1149
|
-
CArray *parent = CAVIEW(src)->parent;
|
|
1150
|
-
ca_attach(parent);
|
|
1151
|
-
st->root = parent;
|
|
1152
|
-
|
|
1153
|
-
ca_axis_desc_t local_descs[CA_RANK_MAX];
|
|
1154
|
-
memcpy(local_descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1155
|
-
const void *bound_fill = NULL;
|
|
1156
|
-
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
1157
|
-
bound_fill = ((CAWindow *) src)->fill;
|
|
1158
|
-
}
|
|
1159
|
-
st->scratch_cap = src->elements * src->bytes;
|
|
1160
|
-
st->scratch_ptr = ca_axis_dispatch_attach(parent, raw_pdims,
|
|
1161
|
-
local_descs, src->ndim,
|
|
1162
|
-
src->bytes, src->elements,
|
|
1163
|
-
bound_fill);
|
|
1164
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1165
|
-
st->alias_ptr = st->scratch_ptr;
|
|
1166
|
-
|
|
1167
|
-
/* Row-major byte / element strides on the scratch (= view layout). */
|
|
1168
|
-
ca_size_t row_byte_strides[CA_RANK_MAX];
|
|
1169
|
-
ca_size_t row_elem_strides[CA_RANK_MAX];
|
|
1170
|
-
{
|
|
1171
|
-
ca_size_t b = src->bytes, e = 1;
|
|
1172
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1173
|
-
row_byte_strides[k] = b;
|
|
1174
|
-
row_elem_strides[k] = e;
|
|
1175
|
-
b *= src->dim[k];
|
|
1176
|
-
e *= src->dim[k];
|
|
1177
|
-
}
|
|
1178
|
-
}
|
|
1108
|
+
ca_size_t total = 1;
|
|
1109
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1110
|
+
st->total_slabs = total;
|
|
1111
|
+
st->slab_n = st->slab_elements;
|
|
1112
|
+
st->slabs_emitted = 0;
|
|
1113
|
+
st->chunk_size = st->slab_n;
|
|
1179
1114
|
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
st->slab_elements *= src->dim[k];
|
|
1189
|
-
sp++;
|
|
1190
|
-
} else {
|
|
1191
|
-
st->outer_axes[op] = k;
|
|
1192
|
-
st->outer_dims[op] = src->dim[k];
|
|
1193
|
-
st->outer_strides[op] = row_byte_strides[k];
|
|
1194
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1195
|
-
op++;
|
|
1196
|
-
}
|
|
1197
|
-
}
|
|
1198
|
-
st->slab_ndim = sp;
|
|
1199
|
-
st->outer_ndim = op;
|
|
1200
|
-
st->composed_base = 0;
|
|
1201
|
-
/* desc_ndim = 0: next_slab_axes treats outer like Phase A
|
|
1202
|
-
(no INDEX lookup), correct because scratch is row-major
|
|
1203
|
-
contig (= STRIDE everywhere). */
|
|
1204
|
-
st->desc_ndim = 0;
|
|
1205
|
-
|
|
1206
|
-
ca_size_t total = 1;
|
|
1207
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1208
|
-
st->total_slabs = total;
|
|
1209
|
-
st->slab_n = st->slab_elements;
|
|
1210
|
-
st->slabs_emitted = 0;
|
|
1211
|
-
st->chunk_size = st->slab_n;
|
|
1212
|
-
|
|
1213
|
-
if ( st->outer_ndim > 0 ) {
|
|
1214
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1215
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1216
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1217
|
-
} else {
|
|
1218
|
-
st->outer_idx = NULL;
|
|
1219
|
-
}
|
|
1115
|
+
if ( st->outer_ndim > 0 ) {
|
|
1116
|
+
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1117
|
+
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1118
|
+
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1119
|
+
} else {
|
|
1120
|
+
st->outer_idx = NULL;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1220
1123
|
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1124
|
+
/* Chooses the route. Decides only: nothing is attached or allocated, so
|
|
1125
|
+
a refusal leaves nothing to close. */
|
|
1126
|
+
static int
|
|
1127
|
+
ki_choose_l2_route (CArray *src, int8_t *axes, int8_t naxes, uint32_t flags,
|
|
1128
|
+
ki_l2_plan *plan)
|
|
1129
|
+
{
|
|
1130
|
+
const int8_t *in_slab = plan->in_slab;
|
|
1131
|
+
ca_size_t row_bytes[CA_RANK_MAX];
|
|
1132
|
+
int8_t k;
|
|
1133
|
+
|
|
1134
|
+
ki_view_row_major(src, row_bytes, NULL);
|
|
1135
|
+
|
|
1136
|
+
if ( plan->src_kind == CA_ITER_SRC_DESCRIPTOR
|
|
1137
|
+
|| plan->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
1138
|
+
/* Shortcut PER_FIBER_FUSED (CAWindow / CAShift).
|
|
1139
|
+
Layer all three, per fiber: each fiber is fetched with the
|
|
1140
|
+
view's ca_xfer_stride (which fills out-of-bounds
|
|
1141
|
+
cells itself) and put back the same way.
|
|
1142
|
+
Speeds up a fiber walk of a CAWindow / CAShift whose innermost
|
|
1143
|
+
axis is SHIFT.
|
|
1144
|
+
Why none measured: about 8% slower than the descriptor
|
|
1145
|
+
route on a sort of a 2000x2000 shift view.
|
|
1146
|
+
Enters CA_KERNEL_FIBER_CONTIG, one slab axis, the view's
|
|
1147
|
+
own rank, and ca_iter_should_per_fiber_fused. An
|
|
1148
|
+
outer-only SHIFT arrives as L2_ALIASABLE and never
|
|
1149
|
+
reaches this hook.
|
|
1150
|
+
Promises 1 (the fiber scratch is contiguous), 2 (the mask is
|
|
1151
|
+
fetched into the fiber mask scratch), 3 (ca_xfer_stride
|
|
1152
|
+
PUT of a fiber that neither converts nor repeats a
|
|
1153
|
+
cell), 4 (CA_ITER_FLAG_WALK_MAY_RAISE).
|
|
1154
|
+
Switch ca_fast_paths_enabled(): off, the descriptor route.
|
|
1155
|
+
Checked by the fiber rows of the "shift" view in the delivery
|
|
1156
|
+
matrix. */
|
|
1157
|
+
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
1158
|
+
&& plan->ndesc == src->ndim && ca_fast_paths_enabled()
|
|
1159
|
+
&& ca_iter_should_per_fiber_fused(src, plan->src_kind, axes[0],
|
|
1160
|
+
row_bytes[axes[0]], flags) ) {
|
|
1161
|
+
plan->route = KI_L2_FIBER_FUSED;
|
|
1162
|
+
return CA_ITER_OK;
|
|
1163
|
+
}
|
|
1231
1164
|
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
ca_size_t s = src->bytes;
|
|
1246
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1247
|
-
pstrides[k] = s;
|
|
1248
|
-
s *= raw_pdims[k];
|
|
1249
|
-
}
|
|
1165
|
+
/* The rank check guards a descriptor whose rank differs from the
|
|
1166
|
+
view's (it would not expose the axis partition slab_axes names).
|
|
1167
|
+
Every describe_axes in the engine reports the view's rank, so the
|
|
1168
|
+
check does not fire today; it stays as the contract's guard. */
|
|
1169
|
+
if ( plan->ndesc != src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1170
|
+
|
|
1171
|
+
int slab_has_non_stride = 0;
|
|
1172
|
+
int slab_has_index = 0;
|
|
1173
|
+
int outer_has_shift = 0;
|
|
1174
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1175
|
+
if ( in_slab[k] && plan->descs[k].kind != CA_AXIS_KIND_STRIDE ) {
|
|
1176
|
+
slab_has_non_stride = 1;
|
|
1177
|
+
if ( plan->descs[k].kind == CA_AXIS_KIND_INDEX ) slab_has_index = 1;
|
|
1250
1178
|
}
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
{
|
|
1254
|
-
ca_size_t s = 1;
|
|
1255
|
-
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1256
|
-
row_elem_strides[k] = s;
|
|
1257
|
-
s *= src->dim[k];
|
|
1258
|
-
}
|
|
1179
|
+
if ( !in_slab[k] && plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) {
|
|
1180
|
+
outer_has_shift = 1;
|
|
1259
1181
|
}
|
|
1182
|
+
}
|
|
1260
1183
|
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
use indices[] from raw_descs and need a 0 stride here
|
|
1293
|
-
(next_slab_axes branches on descs[].kind to compute the
|
|
1294
|
-
real offset). We persist the full raw_descs in st->descs
|
|
1295
|
-
so the next_slab_axes implementation can dispatch. */
|
|
1296
|
-
if ( raw_descs[k].kind == CA_AXIS_KIND_STRIDE ) {
|
|
1297
|
-
st->outer_strides[op] = raw_descs[k].step * pstrides[k];
|
|
1298
|
-
/* Bake in start*pstride into slab_base for STRIDE outer too
|
|
1299
|
-
so per-cell offset is purely step*idx. */
|
|
1300
|
-
slab_base += raw_descs[k].start * pstrides[k];
|
|
1301
|
-
} else {
|
|
1302
|
-
/* INDEX kind: outer_strides[m] = pstrides[k] (element-unit
|
|
1303
|
-
from indices[]). We DON'T pre-bake into slab_base for
|
|
1304
|
-
INDEX axes; per-iter offset = indices[idx[m]] * pstrides[k]
|
|
1305
|
-
is computed in next_slab_axes. */
|
|
1306
|
-
st->outer_strides[op] = pstrides[k];
|
|
1307
|
-
}
|
|
1308
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1309
|
-
op++;
|
|
1310
|
-
}
|
|
1311
|
-
}
|
|
1312
|
-
st->slab_ndim = sp;
|
|
1313
|
-
st->outer_ndim = op;
|
|
1314
|
-
st->composed_base = slab_base;
|
|
1315
|
-
|
|
1316
|
-
/* Persist raw_descs / pstrides in state so next_slab_axes can
|
|
1317
|
-
distinguish STRIDE vs INDEX outer per axis. */
|
|
1318
|
-
memcpy(st->descs, raw_descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1319
|
-
memcpy(st->parent_axis_dims, raw_pdims, src->ndim * sizeof(ca_size_t));
|
|
1320
|
-
memcpy(st->pstrides, pstrides, src->ndim * sizeof(ca_size_t));
|
|
1321
|
-
st->desc_ndim = src->ndim;
|
|
1322
|
-
|
|
1323
|
-
ca_size_t total = 1;
|
|
1324
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1325
|
-
st->total_slabs = total;
|
|
1326
|
-
st->slab_n = st->slab_elements;
|
|
1327
|
-
st->slabs_emitted = 0;
|
|
1328
|
-
st->chunk_size = st->slab_n;
|
|
1329
|
-
|
|
1330
|
-
if ( st->outer_ndim > 0 ) {
|
|
1331
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1332
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1333
|
-
} else {
|
|
1334
|
-
st->outer_idx = NULL;
|
|
1184
|
+
/* A slab with a SHIFT axis that leaves out the innermost view axis
|
|
1185
|
+
would be gathered one strided cell at a time, once per slab. Read
|
|
1186
|
+
walks take the whole-view route instead: the view is gathered
|
|
1187
|
+
once, row by row, and the slabs are walked in the copy.
|
|
1188
|
+
|
|
1189
|
+
Shortcut SHIFT_WHOLE_VIEW.
|
|
1190
|
+
Layer data; the fiber layer (ki_fiber_contig_*) sits on top.
|
|
1191
|
+
Speeds up read walks of a CAWindow / CAShift whose slab has a
|
|
1192
|
+
SHIFT axis and not the innermost axis.
|
|
1193
|
+
Why about 1.5x on a median along axis 0 of a 2000x2000
|
|
1194
|
+
shift view.
|
|
1195
|
+
Enters a non-stride slab axis, no INDEX slab axis, the
|
|
1196
|
+
innermost axis outside the slab, a read walk.
|
|
1197
|
+
Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
|
|
1198
|
+
only when the view has one).
|
|
1199
|
+
Switch ca_fast_paths_enabled(): off, the per-slab gather.
|
|
1200
|
+
Checked by the slab-axis rows of the "shift" and "window" views
|
|
1201
|
+
in the delivery matrix. */
|
|
1202
|
+
int whole_view = slab_has_non_stride && ! slab_has_index
|
|
1203
|
+
&& ! in_slab[src->ndim - 1]
|
|
1204
|
+
&& ! ( flags & CA_KERNEL_WRITE )
|
|
1205
|
+
&& ca_fast_paths_enabled();
|
|
1206
|
+
|
|
1207
|
+
/* The descriptor engine's gathers (per slab and whole view) read
|
|
1208
|
+
only. A write takes the transfer walk instead: the view is
|
|
1209
|
+
gathered whole and the changed cells go back through its own
|
|
1210
|
+
transfer (promise 3). */
|
|
1211
|
+
if ( (slab_has_non_stride && ! whole_view) || outer_has_shift ) {
|
|
1212
|
+
if ( flags & CA_KERNEL_WRITE ) {
|
|
1213
|
+
plan->src_kind = CA_ITER_SRC_ATTACH;
|
|
1214
|
+
goto by_transfer;
|
|
1335
1215
|
}
|
|
1216
|
+
}
|
|
1217
|
+
plan->src_kind = CA_ITER_SRC_DESCRIPTOR;
|
|
1218
|
+
if ( slab_has_non_stride && ! whole_view ) {
|
|
1219
|
+
plan->route = KI_L2_PER_SLAB;
|
|
1220
|
+
}
|
|
1221
|
+
else if ( outer_has_shift || whole_view ) {
|
|
1222
|
+
plan->route = KI_L2_DESC_WHOLE;
|
|
1223
|
+
}
|
|
1224
|
+
else {
|
|
1225
|
+
plan->route = KI_L2_DESC_STRIDED;
|
|
1226
|
+
}
|
|
1227
|
+
return CA_ITER_OK;
|
|
1228
|
+
}
|
|
1336
1229
|
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1230
|
+
if ( plan->src_kind == CA_ITER_SRC_ATTACH ) {
|
|
1231
|
+
by_transfer:
|
|
1232
|
+
/* Shortcut STACK.
|
|
1233
|
+
Layer data and fiber: the slab is cut contiguous from the K
|
|
1234
|
+
parents through a tile cache.
|
|
1235
|
+
Speeds up read walks of an unmasked CAStack whose one slab axis
|
|
1236
|
+
is the stack axis.
|
|
1237
|
+
Why about 2x on a mean along the stack axis of 4 parents
|
|
1238
|
+
of 500x2000.
|
|
1239
|
+
Enters a CAStack, no mask, a read walk, one slab axis that is
|
|
1240
|
+
k_axis.
|
|
1241
|
+
Promises 1 (the slab is contiguous), 6 (no mask exists to touch).
|
|
1242
|
+
Switch ca_fast_paths_enabled(): off, the transfer walk.
|
|
1243
|
+
Checked by the slab-axis and fiber rows of the stack parent along
|
|
1244
|
+
axis 0 in the delivery matrix. */
|
|
1245
|
+
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1246
|
+
&& !ca_has_mask(src)
|
|
1247
|
+
&& !(flags & CA_KERNEL_WRITE)
|
|
1248
|
+
&& naxes == 1 && axes[0] == ((CAStack *) src)->k_axis
|
|
1249
|
+
&& ca_fast_paths_enabled() ) {
|
|
1250
|
+
plan->route = KI_L2_STACK;
|
|
1353
1251
|
return CA_ITER_OK;
|
|
1354
1252
|
}
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
}
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
/* PROPOSAL_CASTACK_LOOP_INTERCHANGE Vector A rev4 (direct per-
|
|
1405
|
-
parent ptr access path + tile cache). Engages when:
|
|
1406
|
-
- source.attach is ca_stack_func.attach (= CAStack identity)
|
|
1407
|
-
- no mask (= horizontal mask propagation per-slab gather is
|
|
1408
|
-
out of scope; falls back to SRC_ATTACH whole-view path)
|
|
1409
|
-
- naxes == 1 && axes[0] == k_axis (= K-axis-only slab, the
|
|
1410
|
-
demand-driving case = reduce along the stacked axis like
|
|
1411
|
-
view.mean(axis: k_axis))
|
|
1412
|
-
init attaches K parents up front (O(1) per entity parent),
|
|
1413
|
-
caches parent->ptr aliases + uniform parent-native byte strides,
|
|
1414
|
-
allocates a slab-sized scratch (= K * bytes) plus a tile cache
|
|
1415
|
-
sized to fit the L1d budget. next_slab_axes refills TILE fibers
|
|
1416
|
-
per K contig parent reads (rev4 2026-06-19) and serves the next
|
|
1417
|
-
TILE-1 calls from the L1d-resident cache, beating SRC_ATTACH
|
|
1418
|
-
across all measured sizes. No size-threshold gate: rev3's gate
|
|
1419
|
-
was a perf trade-off justification that the tile cache erased.
|
|
1420
|
-
|
|
1421
|
-
K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): engage predicate
|
|
1422
|
-
generalised from `axes[0] == 0` to `axes[0] == k_axis`. The
|
|
1423
|
-
tile cache mechanism + outer_idx -> parent axis mapping are
|
|
1424
|
-
already k_axis-agnostic: outer_idx[m] maps to parent axis m
|
|
1425
|
-
regardless of k_axis position (= for k_axis = 0 outer is stack
|
|
1426
|
-
axes 1..N-1 = parent axes 0..N-2; for k_axis > 0 outer is stack
|
|
1427
|
-
axes [0..k_axis-1, k_axis+1..N-1] = parent axes [0..k_axis-1,
|
|
1428
|
-
k_axis..N-2]; either way outer_idx[m] = parent axis m). */
|
|
1429
|
-
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1430
|
-
&& !ca_has_mask(src)
|
|
1431
|
-
&& !(flags & CA_KERNEL_WRITE)
|
|
1432
|
-
&& naxes == 1 && axes[0] == ((CAStack *) src)->k_axis ) {
|
|
1433
|
-
CAStack *stack = (CAStack *) src;
|
|
1434
|
-
int8_t parent_ndim = src->ndim - 1;
|
|
1435
|
-
int8_t sp = 0, op = 0;
|
|
1436
|
-
|
|
1437
|
-
st->slab_elements = 1;
|
|
1438
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1439
|
-
if ( in_slab[k] ) {
|
|
1440
|
-
st->slab_axes_buf[sp] = k;
|
|
1441
|
-
st->slab_dims[sp] = src->dim[k];
|
|
1442
|
-
st->slab_elements *= src->dim[k];
|
|
1443
|
-
sp++;
|
|
1444
|
-
} else {
|
|
1445
|
-
st->outer_axes[op] = k;
|
|
1446
|
-
st->outer_dims[op] = src->dim[k];
|
|
1447
|
-
st->outer_strides[op] = row_byte_strides[k]; /* unused on STACK path */
|
|
1448
|
-
st->outer_mask_strides[op] = row_elem_strides[k]; /* unused on STACK path */
|
|
1449
|
-
op++;
|
|
1450
|
-
}
|
|
1451
|
-
}
|
|
1452
|
-
st->slab_ndim = sp;
|
|
1453
|
-
st->outer_ndim = op;
|
|
1454
|
-
st->composed_base = 0;
|
|
1455
|
-
st->desc_ndim = 0;
|
|
1456
|
-
/* slab_strides on STACK path: slab_axes == [0], scratch is
|
|
1457
|
-
packed contig over K elements (= K * bytes), so single-axis
|
|
1458
|
-
slab walk uses bytes stride. */
|
|
1459
|
-
st->slab_strides[0] = src->bytes;
|
|
1460
|
-
st->slab_mask_strides[0] = 1; /* harmless on no-mask path */
|
|
1461
|
-
|
|
1462
|
-
ca_size_t total = 1;
|
|
1463
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1464
|
-
st->total_slabs = total;
|
|
1465
|
-
st->slab_n = st->slab_elements;
|
|
1466
|
-
st->slabs_emitted = 0;
|
|
1467
|
-
st->chunk_size = st->slab_n;
|
|
1468
|
-
|
|
1469
|
-
if ( st->outer_ndim > 0 ) {
|
|
1470
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1471
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1472
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1473
|
-
} else {
|
|
1474
|
-
st->outer_idx = NULL;
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
/* Attach K parents and cache their ptr aliases. CAStack
|
|
1478
|
-
guarantees uniform shape (MEMO §3.2) so one set of parent
|
|
1479
|
-
native byte strides covers all K (= parents[0]->dim is
|
|
1480
|
-
canonical). */
|
|
1481
|
-
st->stack_n_parents = stack->n_parents;
|
|
1482
|
-
st->stack_parent_ptrs =
|
|
1483
|
-
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
1484
|
-
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1485
|
-
ca_attach(stack->parents[kk]);
|
|
1486
|
-
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
1487
|
-
}
|
|
1488
|
-
{
|
|
1489
|
-
ca_size_t s = src->bytes;
|
|
1490
|
-
for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
|
|
1491
|
-
st->stack_parent_strides[kk] = s;
|
|
1492
|
-
s *= stack->parents[0]->dim[kk];
|
|
1493
|
-
}
|
|
1494
|
-
}
|
|
1495
|
-
|
|
1496
|
-
/* Slab-sized scratch (= K * bytes for slab_axes == [0]). */
|
|
1497
|
-
st->scratch_cap = (ca_size_t) st->slab_elements * src->bytes;
|
|
1498
|
-
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
1499
|
-
|
|
1500
|
-
/* Tile cache (pilot/castack-axis0-loop-interchange): refill TILE
|
|
1501
|
-
fibers per K parent reads, serve next TILE-1 fibers from L1d.
|
|
1502
|
-
TILE budget = ~32 KB (half of M2 L1d). Empirically tested
|
|
1503
|
-
values; clamped to [8, 64] to keep the refill loop tight. */
|
|
1504
|
-
{
|
|
1505
|
-
ca_size_t K = (ca_size_t) stack->n_parents;
|
|
1506
|
-
ca_size_t bytes = (ca_size_t) src->bytes;
|
|
1507
|
-
ca_size_t tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
|
|
1508
|
-
if ( tile < 8 ) tile = 8;
|
|
1509
|
-
if ( tile > 64 ) tile = 64;
|
|
1510
|
-
st->stack_tile_cap = tile;
|
|
1511
|
-
st->stack_tile_pos = 0;
|
|
1512
|
-
st->stack_tile_have = 0; /* force refill on first call */
|
|
1513
|
-
st->stack_tile_cache = (char *) xmalloc(K * tile * bytes);
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
st->alias_mode = CA_ITER_ALIAS_STACK;
|
|
1517
|
-
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1518
|
-
st->alias_mask = NULL;
|
|
1519
|
-
return CA_ITER_OK;
|
|
1520
|
-
}
|
|
1521
|
-
|
|
1522
|
-
/* PROPOSAL_CASTACK_XFER_OPT_LAYERING P.2 Case A (2026-06-18):
|
|
1523
|
-
CAStack source + slab_axes excludes axis 0 (= K-axis stays in
|
|
1524
|
-
outer iter, e.g. view.mean(axis: 1), view.mean(axis: 2),
|
|
1525
|
-
view.mean(axis: 1, 2)). Each slab corresponds to a region
|
|
1526
|
-
inside ONE parent selected by outer_idx[K_outer_pos]. Engine
|
|
1527
|
-
pre-attaches K parents (+ K parent masks if mask present),
|
|
1528
|
-
caches their ptrs, and next_slab_axes aliases parents[k]->ptr +
|
|
1529
|
-
parent_off directly -- no scratch, no materialise, parent
|
|
1530
|
-
entity case = eager-equivalent memory bandwidth.
|
|
1531
|
-
|
|
1532
|
-
Layering: engine detects CAStack identity by function-pointer
|
|
1533
|
-
comparison and accesses parents[] / mask via ca_func[STACK]
|
|
1534
|
-
operation table where possible. CAStack downcast occurs only
|
|
1535
|
-
for the n_parents / parents[] read in init_l2 (= same scope as
|
|
1536
|
-
rev3 STACK; AC3 forbids parents[] walk in next_slab_axes only).
|
|
1537
|
-
Q1 disposition: option (i) minimal-diff inline in SLAB_AXES
|
|
1538
|
-
branch -- evaluated as smallest diff with acceptable layering
|
|
1539
|
-
(proposal §3.1). */
|
|
1540
|
-
/* STACK_OUTER_K: K-axis stays in outer iter (slab carved out of
|
|
1541
|
-
one parent at a time). K.3 (2026-06-20): engage predicate +
|
|
1542
|
-
stack-axis -> parent-axis mapping generalised for arbitrary
|
|
1543
|
-
k_axis. stack axis s maps to parent axis (s if s < k_axis else
|
|
1544
|
-
s - 1); only the K stack axis (s == k_axis) has no parent
|
|
1545
|
-
counterpart. */
|
|
1546
|
-
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1547
|
-
&& !(flags & CA_KERNEL_WRITE)
|
|
1548
|
-
&& naxes >= 1 && !in_slab[((CAStack *) src)->k_axis] ) {
|
|
1549
|
-
CAStack *stack = (CAStack *) src;
|
|
1550
|
-
int8_t k_axis = stack->k_axis;
|
|
1551
|
-
int8_t parent_ndim = src->ndim - 1;
|
|
1552
|
-
int8_t sp = 0, op = 0;
|
|
1553
|
-
|
|
1554
|
-
st->slab_elements = 1;
|
|
1555
|
-
for ( k = 0; k < src->ndim; k++ ) {
|
|
1556
|
-
if ( in_slab[k] ) {
|
|
1557
|
-
st->slab_axes_buf[sp] = k;
|
|
1558
|
-
st->slab_dims[sp] = src->dim[k];
|
|
1559
|
-
st->slab_elements *= src->dim[k];
|
|
1560
|
-
/* slab_strides[sp] = parent native byte stride at parent
|
|
1561
|
-
axis (k if k < k_axis else k - 1). k == k_axis cannot
|
|
1562
|
-
appear here -- the engage predicate above excludes it. */
|
|
1563
|
-
sp++;
|
|
1564
|
-
} else {
|
|
1565
|
-
st->outer_axes[op] = k;
|
|
1566
|
-
st->outer_dims[op] = src->dim[k];
|
|
1567
|
-
st->outer_strides[op] = row_byte_strides[k];
|
|
1568
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
1569
|
-
op++;
|
|
1570
|
-
}
|
|
1571
|
-
}
|
|
1572
|
-
st->slab_ndim = sp;
|
|
1573
|
-
st->outer_ndim = op;
|
|
1574
|
-
st->composed_base = 0;
|
|
1575
|
-
st->desc_ndim = 0;
|
|
1576
|
-
|
|
1577
|
-
/* Locate the K-axis (= stack axis k_axis) within outer_axes. */
|
|
1578
|
-
st->stack_k_outer_pos = -1;
|
|
1579
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
1580
|
-
if ( st->outer_axes[m] == k_axis ) {
|
|
1581
|
-
st->stack_k_outer_pos = m;
|
|
1582
|
-
break;
|
|
1583
|
-
}
|
|
1584
|
-
}
|
|
1585
|
-
|
|
1586
|
-
/* Uniform parent native byte strides + parent element strides
|
|
1587
|
-
for mask (= MEMO §3.2 uniform shape across parents,
|
|
1588
|
-
parents[0]->dim is canonical). Mask strides cached so
|
|
1589
|
-
next_slab_axes can compute mask_off without downcasting
|
|
1590
|
-
(= AC3 layering goal). */
|
|
1591
|
-
{
|
|
1592
|
-
ca_size_t sb = src->bytes;
|
|
1593
|
-
ca_size_t se = 1;
|
|
1594
|
-
for ( int8_t kk = parent_ndim - 1; kk >= 0; kk-- ) {
|
|
1595
|
-
st->stack_parent_strides[kk] = sb;
|
|
1596
|
-
st->stack_parent_mask_strides[kk] = se;
|
|
1597
|
-
sb *= stack->parents[0]->dim[kk];
|
|
1598
|
-
se *= stack->parents[0]->dim[kk];
|
|
1599
|
-
}
|
|
1600
|
-
}
|
|
1601
|
-
|
|
1602
|
-
/* Fill slab_strides + slab_mask_strides for slab axes (all of
|
|
1603
|
-
which are parent inner axes since K is in outer). stack ax
|
|
1604
|
-
-> parent ax: s if s < k_axis else s - 1. */
|
|
1605
|
-
for ( int8_t s_i = 0; s_i < st->slab_ndim; s_i++ ) {
|
|
1606
|
-
int8_t stack_ax = st->slab_axes_buf[s_i];
|
|
1607
|
-
int8_t parent_ax = (stack_ax < k_axis) ? stack_ax : (stack_ax - 1);
|
|
1608
|
-
st->slab_strides[s_i] = st->stack_parent_strides[parent_ax];
|
|
1609
|
-
st->slab_mask_strides[s_i] = st->stack_parent_mask_strides[parent_ax];
|
|
1610
|
-
}
|
|
1611
|
-
|
|
1612
|
-
ca_size_t total = 1;
|
|
1613
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1614
|
-
st->total_slabs = total;
|
|
1615
|
-
st->slab_n = st->slab_elements;
|
|
1616
|
-
st->slabs_emitted = 0;
|
|
1617
|
-
st->chunk_size = st->slab_n;
|
|
1618
|
-
|
|
1619
|
-
if ( st->outer_ndim > 0 ) {
|
|
1620
|
-
CA_ASSUME(st->outer_ndim <= CA_RANK_MAX); /* bound alloc over rank */
|
|
1621
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1622
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1623
|
-
} else {
|
|
1624
|
-
st->outer_idx = NULL;
|
|
1625
|
-
}
|
|
1253
|
+
/* Shortcut STACK_OUTER_K.
|
|
1254
|
+
Layer data; the fiber layer sits on top.
|
|
1255
|
+
Speeds up read walks of a CAStack whose stack axis is an outer
|
|
1256
|
+
axis: each slab is cut from the one parent the outer
|
|
1257
|
+
index picks, without gathering the stack.
|
|
1258
|
+
Why about 11x on a mean along axis 1 of 4 parents of
|
|
1259
|
+
500x2000.
|
|
1260
|
+
Enters a CAStack, a read walk, the stack axis outside the slab.
|
|
1261
|
+
Promises 2 (the parents' masks are copied), 6 (masks are read
|
|
1262
|
+
only when a parent has one), 8 (alias_ptr stays NULL,
|
|
1263
|
+
so no slab is lent as a window).
|
|
1264
|
+
Switch ca_fast_paths_enabled(): off, the transfer walk.
|
|
1265
|
+
Checked by the slab-axis, fiber and two-slab-axes rows of the
|
|
1266
|
+
stack parent along axes 1 and 2 in the delivery matrix. */
|
|
1267
|
+
if ( ca_func[src->obj_type].attach == ca_stack_func.attach
|
|
1268
|
+
&& !(flags & CA_KERNEL_WRITE)
|
|
1269
|
+
&& naxes >= 1 && !in_slab[((CAStack *) src)->k_axis]
|
|
1270
|
+
&& ca_fast_paths_enabled() ) {
|
|
1271
|
+
plan->route = KI_L2_STACK_OUTER_K;
|
|
1272
|
+
return CA_ITER_OK;
|
|
1273
|
+
}
|
|
1274
|
+
/* Shortcut PER_FIBER_FUSED (conversion views).
|
|
1275
|
+
Layer all three, per fiber: each fiber is fetched with the
|
|
1276
|
+
view's ca_xfer_stride into the fiber scratch, without
|
|
1277
|
+
gathering the whole view.
|
|
1278
|
+
Speeds up a read of a fiber of a fake or byte-swap view, or of a
|
|
1279
|
+
lazy monop that only casts, along its innermost axis.
|
|
1280
|
+
Why no difference measured on a sort along the last axis
|
|
1281
|
+
of a 2000x2000 fake view.
|
|
1282
|
+
Enters CA_KERNEL_FIBER_CONTIG, one slab axis, a read walk, and
|
|
1283
|
+
ca_iter_should_per_fiber_fused.
|
|
1284
|
+
Promises 1, 2 and 4, as the CAWindow / CAShift hook above. A
|
|
1285
|
+
write takes the transfer walk (promise 3).
|
|
1286
|
+
Switch ca_fast_paths_enabled(): off, the transfer walk.
|
|
1287
|
+
Checked by the fiber rows of the fake parent in the delivery
|
|
1288
|
+
matrix. */
|
|
1289
|
+
if ( (flags & CA_KERNEL_FIBER_CONTIG) && naxes == 1
|
|
1290
|
+
&& ca_fast_paths_enabled()
|
|
1291
|
+
&& ca_iter_should_per_fiber_fused(src, CA_ITER_SRC_ATTACH, axes[0],
|
|
1292
|
+
row_bytes[axes[0]], flags) ) {
|
|
1293
|
+
plan->route = KI_L2_FIBER_FUSED;
|
|
1294
|
+
return CA_ITER_OK;
|
|
1295
|
+
}
|
|
1296
|
+
plan->route = KI_L2_TRANSFER;
|
|
1297
|
+
return CA_ITER_OK;
|
|
1298
|
+
}
|
|
1626
1299
|
|
|
1627
|
-
|
|
1628
|
-
Case B; for entity parents attach is O(1). */
|
|
1629
|
-
st->stack_n_parents = stack->n_parents;
|
|
1630
|
-
st->stack_parent_ptrs =
|
|
1631
|
-
(char **) xmalloc(stack->n_parents * sizeof(char *));
|
|
1632
|
-
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1633
|
-
ca_attach(stack->parents[kk]);
|
|
1634
|
-
st->stack_parent_ptrs[kk] = (char *) stack->parents[kk]->ptr;
|
|
1635
|
-
}
|
|
1300
|
+
if ( plan->src_kind != CA_ITER_SRC_CASTRIDE ) return CA_ITER_ERR_POLICY;
|
|
1636
1301
|
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
st->scratch_cap = 0;
|
|
1655
|
-
st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
|
|
1656
|
-
st->alias_ptr = NULL;
|
|
1657
|
-
st->alias_mask = NULL;
|
|
1658
|
-
return CA_ITER_OK;
|
|
1659
|
-
}
|
|
1302
|
+
if ( ca_iter_can_alias(src, 1) ) {
|
|
1303
|
+
plan->route = KI_L2_CONTIG;
|
|
1304
|
+
return CA_ITER_OK;
|
|
1305
|
+
}
|
|
1306
|
+
/* A non-contiguous CAStride: compose the strides up to the root. A
|
|
1307
|
+
root that holds nothing to read through would cost one whole-root
|
|
1308
|
+
materialise no matter how few cells the view touches -- the cliff
|
|
1309
|
+
the xfer path avoids by asking for a region -- so that view is
|
|
1310
|
+
gathered by its own region protocol instead: ca_copy_data walks the
|
|
1311
|
+
leaf's request up the chain, so a 1000x1000 slice of a 2014x3040
|
|
1312
|
+
CAObject asks for exactly that block. */
|
|
1313
|
+
ca_stride_compose_to_root((CAStride *) src, &plan->root, plan->composed,
|
|
1314
|
+
&plan->base);
|
|
1315
|
+
plan->route = ca_root_lends_no_memory(plan->root) ? KI_L2_VIEW_SCRATCH
|
|
1316
|
+
: KI_L2_ROOT_STRIDED;
|
|
1317
|
+
return CA_ITER_OK;
|
|
1318
|
+
}
|
|
1660
1319
|
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
int8_t fiber_ax = axes[0];
|
|
1669
|
-
ca_size_t fiber_ax_stride = row_byte_strides[fiber_ax];
|
|
1670
|
-
if ( ca_iter_should_per_fiber_fused(src, src_kind, fiber_ax,
|
|
1671
|
-
fiber_ax_stride, flags) ) {
|
|
1672
|
-
per_fiber_fused = 1;
|
|
1673
|
-
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
1674
|
-
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1675
|
-
st->fiber_axis = fiber_ax;
|
|
1676
|
-
for ( int8_t kk = 0; kk < src->ndim; kk++ ) {
|
|
1677
|
-
st->fiber_native_strides[kk] = row_byte_strides[kk];
|
|
1678
|
-
}
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1320
|
+
/* Builds the state for the chosen route. */
|
|
1321
|
+
static void
|
|
1322
|
+
ki_open_l2 (ca_iter_state *st, CArray *src, ca_slab_policy_t policy,
|
|
1323
|
+
int8_t *axes, int8_t naxes, uint32_t flags, ki_l2_plan *plan)
|
|
1324
|
+
{
|
|
1325
|
+
ki_axis_layout lay;
|
|
1326
|
+
int8_t k;
|
|
1681
1327
|
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1691
|
-
st->alias_ptr = st->scratch_ptr;
|
|
1692
|
-
}
|
|
1328
|
+
st->src = src;
|
|
1329
|
+
st->src_kind = plan->src_kind;
|
|
1330
|
+
st->level = 2;
|
|
1331
|
+
st->policy = policy;
|
|
1332
|
+
st->ndim = src->ndim;
|
|
1333
|
+
st->flags = flags;
|
|
1334
|
+
st->bytes = src->bytes;
|
|
1335
|
+
st->naxes = naxes;
|
|
1693
1336
|
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
}
|
|
1711
|
-
}
|
|
1712
|
-
st->slab_ndim = sp;
|
|
1713
|
-
st->outer_ndim = op;
|
|
1714
|
-
st->composed_base = 0;
|
|
1715
|
-
st->desc_ndim = 0; /* row-major STRIDE everywhere */
|
|
1716
|
-
|
|
1717
|
-
ca_size_t total = 1;
|
|
1718
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
1719
|
-
st->total_slabs = total;
|
|
1720
|
-
st->slab_n = st->slab_elements;
|
|
1721
|
-
st->slabs_emitted = 0;
|
|
1722
|
-
st->chunk_size = st->slab_n;
|
|
1723
|
-
|
|
1724
|
-
if ( st->outer_ndim > 0 ) {
|
|
1725
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
1726
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
1727
|
-
} else {
|
|
1728
|
-
st->outer_idx = NULL;
|
|
1729
|
-
}
|
|
1337
|
+
/* The default layout: the view's own row-major strides, which is how
|
|
1338
|
+
a scratch the walk gathered is laid out and how the mask copy is
|
|
1339
|
+
laid out on every route that takes one. */
|
|
1340
|
+
for ( k = 0; k < src->ndim; k++ ) lay.dims[k] = src->dim[k];
|
|
1341
|
+
ki_view_row_major(src, lay.slab_strides, lay.mask_strides);
|
|
1342
|
+
memcpy(lay.outer_strides, lay.slab_strides, src->ndim * sizeof(ca_size_t));
|
|
1343
|
+
|
|
1344
|
+
switch ( plan->route ) {
|
|
1345
|
+
|
|
1346
|
+
case KI_L2_FIBER_FUSED:
|
|
1347
|
+
st->alias_mode = CA_ITER_ALIAS_PER_FIBER_FUSED;
|
|
1348
|
+
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1349
|
+
st->fiber_axis = axes[0];
|
|
1350
|
+
memcpy(st->fiber_native_strides, lay.slab_strides,
|
|
1351
|
+
src->ndim * sizeof(ca_size_t));
|
|
1352
|
+
break;
|
|
1730
1353
|
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1354
|
+
case KI_L2_PER_SLAB: {
|
|
1355
|
+
/* One slab is gathered into a slab-sized scratch per outer index, so
|
|
1356
|
+
a slab axis steps through that scratch (packed row-major over the
|
|
1357
|
+
slab axes) and an outer axis moves nothing in it. The descriptors
|
|
1358
|
+
are kept for next_slab_axes to build each slab's subset. */
|
|
1359
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1360
|
+
ca_size_t b = src->bytes;
|
|
1361
|
+
ca_attach(parent); /* window: finish detaches st->root */
|
|
1362
|
+
st->root = parent;
|
|
1363
|
+
memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1364
|
+
memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
|
|
1365
|
+
{
|
|
1366
|
+
ca_size_t s = src->bytes;
|
|
1367
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1368
|
+
st->pstrides[k] = s;
|
|
1369
|
+
s *= plan->pdims[k];
|
|
1745
1370
|
}
|
|
1746
|
-
return CA_ITER_OK;
|
|
1747
1371
|
}
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1372
|
+
st->desc_ndim = src->ndim;
|
|
1373
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1374
|
+
lay.dims[k] = plan->descs[k].count;
|
|
1375
|
+
lay.outer_strides[k] = 0;
|
|
1376
|
+
if ( plan->in_slab[k] ) {
|
|
1377
|
+
lay.slab_strides[k] = b;
|
|
1378
|
+
b *= plan->descs[k].count;
|
|
1379
|
+
}
|
|
1751
1380
|
}
|
|
1381
|
+
break;
|
|
1752
1382
|
}
|
|
1753
1383
|
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
scratch + xfer_all GET/PUT. Yield as one 1-D strided slab. */
|
|
1767
|
-
if ( src_kind == CA_ITER_SRC_ATTACH ) {
|
|
1768
|
-
st->src = src;
|
|
1769
|
-
st->src_kind = CA_ITER_SRC_ATTACH;
|
|
1770
|
-
st->level = 2;
|
|
1771
|
-
st->policy = policy;
|
|
1772
|
-
st->ndim = 1; /* logical 1-D L2 layout */
|
|
1773
|
-
st->flags = flags;
|
|
1774
|
-
st->bytes = src->bytes;
|
|
1775
|
-
st->axes = axes;
|
|
1776
|
-
st->naxes = naxes;
|
|
1777
|
-
|
|
1778
|
-
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
1779
|
-
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0 ? st->scratch_cap : 1);
|
|
1780
|
-
if ( src->elements > 0 ) {
|
|
1781
|
-
ca_xfer_all(src, st->scratch_ptr, CA_XFER_GET);
|
|
1782
|
-
}
|
|
1783
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1784
|
-
st->alias_ptr = st->scratch_ptr;
|
|
1785
|
-
st->composed_strides[0] = src->bytes;
|
|
1786
|
-
st->composed_base = 0;
|
|
1787
|
-
st->slab_n = src->elements;
|
|
1788
|
-
st->total_slabs = 1;
|
|
1789
|
-
st->slabs_emitted = 0;
|
|
1790
|
-
st->chunk_size = st->slab_n;
|
|
1791
|
-
st->outer_idx = NULL;
|
|
1792
|
-
|
|
1793
|
-
if ( ca_has_mask(src) ) {
|
|
1794
|
-
ca_size_t mcap = src->elements > 0 ? src->elements : 1;
|
|
1795
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
1796
|
-
if ( src->elements > 0 ) {
|
|
1797
|
-
ca_xfer_all(src->mask, (char *) st->scratch_mask, CA_XFER_GET);
|
|
1798
|
-
}
|
|
1799
|
-
st->alias_mask = st->scratch_mask;
|
|
1384
|
+
case KI_L2_DESC_WHOLE: {
|
|
1385
|
+
/* The view gathered whole into a row-major scratch by the
|
|
1386
|
+
descriptor engine (which fills out-of-bounds cells), then walked
|
|
1387
|
+
with the default layout. */
|
|
1388
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1389
|
+
ca_axis_desc_t local_descs[CA_RANK_MAX];
|
|
1390
|
+
const void *bound_fill = NULL;
|
|
1391
|
+
ca_attach(parent); /* window: finish detaches st->root */
|
|
1392
|
+
st->root = parent;
|
|
1393
|
+
memcpy(local_descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1394
|
+
if ( ca_func[src->obj_type].attach == ca_window_func.attach ) {
|
|
1395
|
+
bound_fill = ((CAWindow *) src)->fill;
|
|
1800
1396
|
}
|
|
1801
|
-
|
|
1397
|
+
st->scratch_cap = src->elements * src->bytes;
|
|
1398
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1399
|
+
ca_axis_dispatch_gather(parent, plan->pdims, local_descs, src->ndim,
|
|
1400
|
+
src->bytes, src->elements, bound_fill,
|
|
1401
|
+
st->scratch_ptr);
|
|
1402
|
+
st->total_elements = src->elements;
|
|
1403
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1404
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1405
|
+
break;
|
|
1802
1406
|
}
|
|
1803
1407
|
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
on the pre-classified prefix[]. next_slab_strided branches on
|
|
1814
|
-
src_kind to dispatch to this offset formula. */
|
|
1815
|
-
if ( src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
1816
|
-
int8_t nd = raw_ndim;
|
|
1817
|
-
int8_t k;
|
|
1818
|
-
|
|
1819
|
-
st->src = src;
|
|
1820
|
-
st->src_kind = CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE;
|
|
1821
|
-
st->level = 2;
|
|
1822
|
-
st->policy = policy;
|
|
1823
|
-
st->ndim = nd;
|
|
1824
|
-
st->flags = flags;
|
|
1825
|
-
st->bytes = src->bytes;
|
|
1826
|
-
st->axes = axes;
|
|
1827
|
-
st->naxes = naxes;
|
|
1828
|
-
|
|
1829
|
-
/* Persist descs / parent_axis_dims in state; build parent row-major
|
|
1830
|
-
pstrides so prefix_offset / inner offset share the same byte space. */
|
|
1831
|
-
memcpy(st->descs, raw_descs, nd * sizeof(ca_axis_desc_t));
|
|
1832
|
-
memcpy(st->parent_axis_dims, raw_pdims, nd * sizeof(ca_size_t));
|
|
1833
|
-
st->desc_ndim = nd;
|
|
1408
|
+
case KI_L2_DESC_STRIDED: {
|
|
1409
|
+
/* The parent's memory is walked in place. Each slab axis is a
|
|
1410
|
+
STRIDE axis; an outer axis is STRIDE or INDEX (an outer SHIFT went
|
|
1411
|
+
to KI_L2_DESC_WHOLE). The start of every STRIDE axis is folded
|
|
1412
|
+
into composed_base, so a slab or STRIDE outer offset is step *
|
|
1413
|
+
index; an INDEX outer axis is looked up in its indices[] by
|
|
1414
|
+
next_slab_axes, which needs the descriptors kept. */
|
|
1415
|
+
CArray *parent = CAVIEW(src)->parent;
|
|
1416
|
+
ca_size_t base = 0;
|
|
1834
1417
|
{
|
|
1835
1418
|
ca_size_t s = src->bytes;
|
|
1836
|
-
for ( k =
|
|
1419
|
+
for ( k = src->ndim - 1; k >= 0; k-- ) {
|
|
1837
1420
|
st->pstrides[k] = s;
|
|
1838
|
-
s *=
|
|
1421
|
+
s *= plan->pdims[k];
|
|
1839
1422
|
}
|
|
1840
1423
|
}
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
st->total_slabs = outer_total;
|
|
1856
|
-
st->slab_n = inner_n;
|
|
1857
|
-
st->slabs_emitted = 0;
|
|
1858
|
-
st->chunk_size = inner_n;
|
|
1859
|
-
st->total_elements = src->elements;
|
|
1860
|
-
|
|
1861
|
-
/* composed_base holds inner_byte_start; composed_strides[nd-1] holds
|
|
1862
|
-
inner stride so next_slab_strided's existing inner_st extraction
|
|
1863
|
-
(= composed_strides[nd-1]) works without per-iter recomputation.
|
|
1864
|
-
Outer slots of composed_strides are unused by the descriptor
|
|
1865
|
-
branch in next_slab_strided (= prefix[] drives the offset). */
|
|
1866
|
-
st->composed_base = inner_start;
|
|
1867
|
-
st->composed_strides[nd - 1] = inner_stride;
|
|
1868
|
-
|
|
1869
|
-
if ( nd > 1 ) {
|
|
1870
|
-
CA_ASSUME(nd <= CA_RANK_MAX); /* with nd > 1: nd-1 in [1, CA_RANK_MAX-1] */
|
|
1871
|
-
st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
|
|
1872
|
-
for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
|
|
1873
|
-
} else {
|
|
1874
|
-
st->outer_idx = NULL;
|
|
1424
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1425
|
+
const ca_axis_desc_t *d = &plan->descs[k];
|
|
1426
|
+
lay.dims[k] = d->count;
|
|
1427
|
+
if ( plan->in_slab[k] ) {
|
|
1428
|
+
base += d->start * st->pstrides[k];
|
|
1429
|
+
lay.slab_strides[k] = d->step * st->pstrides[k];
|
|
1430
|
+
}
|
|
1431
|
+
else if ( d->kind == CA_AXIS_KIND_STRIDE ) {
|
|
1432
|
+
base += d->start * st->pstrides[k];
|
|
1433
|
+
lay.outer_strides[k] = d->step * st->pstrides[k];
|
|
1434
|
+
}
|
|
1435
|
+
else {
|
|
1436
|
+
lay.outer_strides[k] = st->pstrides[k];
|
|
1437
|
+
}
|
|
1875
1438
|
}
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1439
|
+
st->composed_base = base;
|
|
1440
|
+
memcpy(st->descs, plan->descs, src->ndim * sizeof(ca_axis_desc_t));
|
|
1441
|
+
memcpy(st->parent_axis_dims, plan->pdims, src->ndim * sizeof(ca_size_t));
|
|
1442
|
+
st->desc_ndim = src->ndim;
|
|
1443
|
+
ca_attach(parent); /* window: finish detaches st->root */
|
|
1879
1444
|
st->root = parent;
|
|
1880
1445
|
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
1881
1446
|
st->alias_ptr = (char *) parent->ptr;
|
|
1447
|
+
break;
|
|
1448
|
+
}
|
|
1882
1449
|
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1450
|
+
case KI_L2_STACK: {
|
|
1451
|
+
/* The K parents are attached and read directly; the slab (the K cells
|
|
1452
|
+
along the stack axis) is cut into a slab-sized scratch, refilled
|
|
1453
|
+
from a tile cache of TILE fibers that the parents fill in one pass.
|
|
1454
|
+
CAStack guarantees one shape across its parents, so parents[0]
|
|
1455
|
+
gives the native strides of all of them. */
|
|
1456
|
+
CAStack *stack = (CAStack *) src;
|
|
1457
|
+
int8_t parent_ndim = src->ndim - 1;
|
|
1458
|
+
ca_size_t K = (ca_size_t) stack->n_parents;
|
|
1459
|
+
ca_size_t bytes = (ca_size_t) src->bytes;
|
|
1460
|
+
ca_size_t tile;
|
|
1461
|
+
lay.slab_strides[axes[0]] = src->bytes;
|
|
1462
|
+
lay.mask_strides[axes[0]] = 1;
|
|
1463
|
+
ki_attach_stack_parents(st, stack);
|
|
1464
|
+
{
|
|
1465
|
+
ca_size_t s = src->bytes;
|
|
1466
|
+
for ( k = parent_ndim - 1; k >= 0; k-- ) {
|
|
1467
|
+
st->stack_parent_strides[k] = s;
|
|
1468
|
+
s *= stack->parents[0]->dim[k];
|
|
1890
1469
|
}
|
|
1891
|
-
st->alias_mask = st->scratch_mask;
|
|
1892
1470
|
}
|
|
1893
|
-
|
|
1471
|
+
st->scratch_cap = (ca_size_t) src->dim[axes[0]] * src->bytes;
|
|
1472
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1473
|
+
/* TILE budget = ~32 KB (half of M2 L1d), clamped to [8, 64] to keep
|
|
1474
|
+
the refill loop tight. */
|
|
1475
|
+
tile = K > 0 ? (32 * 1024) / (K * bytes) : 0;
|
|
1476
|
+
if ( tile < 8 ) tile = 8;
|
|
1477
|
+
if ( tile > 64 ) tile = 64;
|
|
1478
|
+
st->stack_tile_cap = tile;
|
|
1479
|
+
st->stack_tile_pos = 0;
|
|
1480
|
+
st->stack_tile_have = 0; /* force refill on first call */
|
|
1481
|
+
st->stack_tile_cache = (char *) ki_data_alloc(src->data_type, K * tile * bytes);
|
|
1482
|
+
st->alias_mode = CA_ITER_ALIAS_STACK;
|
|
1483
|
+
st->alias_ptr = NULL; /* no whole-view buffer */
|
|
1484
|
+
break;
|
|
1894
1485
|
}
|
|
1895
1486
|
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
st->policy = policy;
|
|
1913
|
-
st->ndim = src->ndim;
|
|
1914
|
-
st->flags = flags;
|
|
1915
|
-
st->bytes = src->bytes;
|
|
1916
|
-
st->axes = axes;
|
|
1917
|
-
st->naxes = naxes;
|
|
1918
|
-
|
|
1919
|
-
memcpy(st->parent_axis_dims, raw_pdims, raw_ndim * sizeof(ca_size_t));
|
|
1920
|
-
ca_axis_dispatch_prepare(st->parent_axis_dims, raw_descs, raw_ndim,
|
|
1921
|
-
st->bytes, st->descs, st->pstrides,
|
|
1922
|
-
st->mdim, &st->desc_ndim);
|
|
1923
|
-
ca_axis_dispatch_layout(st->descs, st->pstrides, st->mdim,
|
|
1924
|
-
st->desc_ndim, st->bytes,
|
|
1925
|
-
&st->slab_start, &st->slab_bytes_desc,
|
|
1926
|
-
&st->slab_base);
|
|
1927
|
-
if ( st->slab_start > 0 ) {
|
|
1928
|
-
ca_axis_dispatch_classify_prefix(st->descs, st->pstrides,
|
|
1929
|
-
st->slab_start, st->prefix);
|
|
1487
|
+
case KI_L2_STACK_OUTER_K: {
|
|
1488
|
+
/* Each slab is a region inside ONE parent, picked by
|
|
1489
|
+
outer_idx[stack_k_outer_pos]; next_slab_axes points into
|
|
1490
|
+
parents[k]->ptr + parent_off. A slab axis s of the stack is parent
|
|
1491
|
+
axis s, or s - 1 past the stack axis. */
|
|
1492
|
+
CAStack *stack = (CAStack *) src;
|
|
1493
|
+
int8_t k_axis = stack->k_axis;
|
|
1494
|
+
int8_t parent_ndim = src->ndim - 1;
|
|
1495
|
+
{
|
|
1496
|
+
ca_size_t sb = src->bytes, se = 1;
|
|
1497
|
+
for ( k = parent_ndim - 1; k >= 0; k-- ) {
|
|
1498
|
+
st->stack_parent_strides[k] = sb;
|
|
1499
|
+
st->stack_parent_mask_strides[k] = se;
|
|
1500
|
+
sb *= stack->parents[0]->dim[k];
|
|
1501
|
+
se *= stack->parents[0]->dim[k];
|
|
1502
|
+
}
|
|
1930
1503
|
}
|
|
1931
|
-
st->
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
bound_fill = ((CAWindow *) src)->fill;
|
|
1504
|
+
st->stack_k_outer_pos = 0;
|
|
1505
|
+
for ( k = 0; k < src->ndim; k++ ) {
|
|
1506
|
+
if ( plan->in_slab[k] ) {
|
|
1507
|
+
int8_t parent_ax = (k < k_axis) ? k : (k - 1);
|
|
1508
|
+
lay.slab_strides[k] = st->stack_parent_strides[parent_ax];
|
|
1509
|
+
lay.mask_strides[k] = st->stack_parent_mask_strides[parent_ax];
|
|
1510
|
+
}
|
|
1511
|
+
else if ( k < k_axis ) {
|
|
1512
|
+
st->stack_k_outer_pos++;
|
|
1513
|
+
}
|
|
1942
1514
|
}
|
|
1943
|
-
st
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
bound_fill);
|
|
1949
|
-
|
|
1950
|
-
/* Lay out as a single 1-D L2 strided slab: ptr = scratch,
|
|
1951
|
-
n = total_elements, stride = bytes. next_slab_strided reads
|
|
1952
|
-
inner_stride from composed_strides[ndim - 1] and skips the
|
|
1953
|
-
outer loop when outer_idx == NULL, so a 1-D logical layout
|
|
1954
|
-
(st->ndim = 1, composed_strides[0] = bytes) yields exactly one
|
|
1955
|
-
contig run. Note st->ndim diverges from src->ndim here — the
|
|
1956
|
-
iterator's logical ndim is 1, but the view itself can be N-D
|
|
1957
|
-
(the materialised buffer is flat). */
|
|
1958
|
-
st->ndim = 1;
|
|
1959
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1960
|
-
st->alias_ptr = st->scratch_ptr;
|
|
1961
|
-
st->composed_strides[0] = src->bytes;
|
|
1962
|
-
st->composed_base = 0;
|
|
1963
|
-
st->slab_n = src->elements;
|
|
1964
|
-
st->total_slabs = 1;
|
|
1965
|
-
st->slabs_emitted = 0;
|
|
1966
|
-
st->chunk_size = st->slab_n;
|
|
1967
|
-
st->outer_idx = NULL;
|
|
1968
|
-
|
|
1969
|
-
/* Step 6: mask gather, same as L1 descriptor branch. */
|
|
1515
|
+
ki_attach_stack_parents(st, stack);
|
|
1516
|
+
/* A masked stack: copy the K parent masks as they are at the start
|
|
1517
|
+
of the walk, and next_slab_axes points into the copy of parent k
|
|
1518
|
+
(promise 2). The stack's mask may still read stand-ins for
|
|
1519
|
+
parents that have gained a mask since; read through to those. */
|
|
1970
1520
|
if ( ca_has_mask(src) ) {
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1521
|
+
ca_multi_parent_mask_refresh(src->mask, stack->parents,
|
|
1522
|
+
stack->mask_standins,
|
|
1523
|
+
stack->n_parents);
|
|
1524
|
+
/* Zero-filled: a slot is set once its copy is made, and finish
|
|
1525
|
+
frees the copies whose slot is set. */
|
|
1526
|
+
st->stack_parent_mask_ptrs =
|
|
1527
|
+
(boolean8_t **) xmalloc(stack->n_parents * sizeof(boolean8_t *));
|
|
1528
|
+
memset(st->stack_parent_mask_ptrs, 0,
|
|
1529
|
+
stack->n_parents * sizeof(boolean8_t *));
|
|
1530
|
+
for ( int32_t kk = 0; kk < stack->n_parents; kk++ ) {
|
|
1531
|
+
CArray *pm = ((CAStack *) src->mask)->parents[kk];
|
|
1532
|
+
ca_size_t n = pm->elements;
|
|
1533
|
+
boolean8_t *copy = (boolean8_t *) xmalloc(n > 0 ? n : 1);
|
|
1534
|
+
st->stack_parent_mask_ptrs[kk] = copy;
|
|
1535
|
+
ca_attach(pm); /* window: only a memcpy before the detach */
|
|
1536
|
+
memcpy(copy, pm->ptr, n);
|
|
1537
|
+
ca_detach(pm);
|
|
1975
1538
|
}
|
|
1976
|
-
st->alias_mask = st->scratch_mask;
|
|
1977
1539
|
}
|
|
1978
|
-
|
|
1540
|
+
st->alias_mode = CA_ITER_ALIAS_STACK_OUTER_K;
|
|
1541
|
+
st->alias_ptr = NULL;
|
|
1542
|
+
break;
|
|
1979
1543
|
}
|
|
1980
1544
|
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
st->ndim = src->ndim;
|
|
1987
|
-
st->flags = flags;
|
|
1988
|
-
st->bytes = src->bytes;
|
|
1989
|
-
st->axes = axes;
|
|
1990
|
-
st->naxes = naxes;
|
|
1991
|
-
|
|
1992
|
-
/* Compute composed strides + base. For entity / CAStride contig we
|
|
1993
|
-
synthesise row-major byte strides so next_slab_strided's offset
|
|
1994
|
-
math is uniform across alias modes; for CAStride non-contig we
|
|
1995
|
-
compose leaf strides up to the root entity via the substrate. */
|
|
1996
|
-
int8_t nd = src->ndim;
|
|
1997
|
-
int use_strided = 0;
|
|
1998
|
-
int use_view_scratch = 0;
|
|
1999
|
-
CArray *root = NULL;
|
|
1545
|
+
case KI_L2_TRANSFER:
|
|
1546
|
+
ki_gather_view(st, src, flags, 0);
|
|
1547
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1548
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1549
|
+
break;
|
|
2000
1550
|
|
|
2001
|
-
|
|
2002
|
-
|
|
1551
|
+
case KI_L2_ROOT_STRIDED:
|
|
1552
|
+
memcpy(st->composed_strides, plan->composed, src->ndim * sizeof(ca_size_t));
|
|
1553
|
+
st->composed_base = plan->base;
|
|
1554
|
+
memcpy(lay.slab_strides, plan->composed, src->ndim * sizeof(ca_size_t));
|
|
1555
|
+
memcpy(lay.outer_strides, plan->composed, src->ndim * sizeof(ca_size_t));
|
|
1556
|
+
ca_attach(plan->root); /* window: finish detaches st->root */
|
|
1557
|
+
st->root = plan->root; /* recorded once attached */
|
|
1558
|
+
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
1559
|
+
st->alias_ptr = (char *) plan->root->ptr;
|
|
1560
|
+
break;
|
|
1561
|
+
|
|
1562
|
+
case KI_L2_VIEW_SCRATCH:
|
|
2003
1563
|
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
2004
|
-
src->dim,
|
|
2005
|
-
st
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
1564
|
+
src->dim, src->ndim, src->bytes);
|
|
1565
|
+
ki_gather_view(st, src, flags, 1);
|
|
1566
|
+
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
1567
|
+
st->alias_ptr = st->scratch_ptr;
|
|
1568
|
+
break;
|
|
1569
|
+
|
|
1570
|
+
case KI_L2_CONTIG:
|
|
1571
|
+
ca_iter_build_rowmajor_strides(st->composed_strides,
|
|
1572
|
+
src->dim, src->ndim, src->bytes);
|
|
1573
|
+
ca_attach(src);
|
|
1574
|
+
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
1575
|
+
st->alias_ptr = (char *) src->ptr;
|
|
1576
|
+
break;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
ki_lay_out_slabs(st, src, plan->in_slab, &lay);
|
|
1580
|
+
|
|
1581
|
+
if ( plan->route == KI_L2_PER_SLAB ) {
|
|
1582
|
+
/* One max-slab buffer refilled per outer index, sized now that the
|
|
1583
|
+
slab is known.
|
|
1584
|
+
|
|
1585
|
+
Shortcut PER_SLAB_HOIST.
|
|
1586
|
+
Layer data; the fiber layer sits on top.
|
|
1587
|
+
Speeds up per-slab gathers whose innermost slab axis is STRIDE
|
|
1588
|
+
and with no SHIFT axis in the view: the outer and
|
|
1589
|
+
non-innermost slab offsets are hoisted and the inner
|
|
1590
|
+
run is copied with memcpy.
|
|
1591
|
+
Why no difference measured on a row-selected 2-D read.
|
|
1592
|
+
Enters at least one slab axis, the innermost slab axis
|
|
1593
|
+
STRIDE, no SHIFT axis.
|
|
1594
|
+
Promises 2 and 6 (the mask is copied by ki_take_mask_copy, and
|
|
1595
|
+
only when the view has one).
|
|
1596
|
+
Switch ca_fast_paths_enabled(): off, the PER_SLAB gather.
|
|
1597
|
+
Checked by the two-slab-axes rows of the grid parent in the
|
|
1598
|
+
delivery matrix. */
|
|
1599
|
+
ca_size_t slab_bytes = st->slab_elements * src->bytes;
|
|
1600
|
+
int hoist = (st->slab_ndim > 0) && ca_fast_paths_enabled()
|
|
1601
|
+
&& plan->descs[st->slab_axes_buf[st->slab_ndim - 1]].kind
|
|
1602
|
+
== CA_AXIS_KIND_STRIDE;
|
|
1603
|
+
for ( k = 0; hoist && k < src->ndim; k++ ) {
|
|
1604
|
+
if ( plan->descs[k].kind == CA_AXIS_KIND_SHIFT ) hoist = 0;
|
|
2029
1605
|
}
|
|
1606
|
+
st->scratch_cap = slab_bytes > 0 ? slab_bytes : 1;
|
|
1607
|
+
st->scratch_ptr = (char *) ki_data_alloc(src->data_type, st->scratch_cap);
|
|
1608
|
+
st->alias_ptr = st->scratch_ptr; /* refilled per slab in next_slab_axes */
|
|
1609
|
+
st->alias_mode = hoist ? CA_ITER_ALIAS_PER_SLAB_HOIST
|
|
1610
|
+
: CA_ITER_ALIAS_PER_SLAB;
|
|
2030
1611
|
}
|
|
2031
1612
|
|
|
2032
|
-
/*
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
1613
|
+
/* The mask cursor (promise 2): fiber by fiber through fiber_mask_scratch
|
|
1614
|
+
on PER_FIBER_FUSED, the parents' copies on STACK_OUTER_K, none on
|
|
1615
|
+
STACK (an unmasked stack), and the walk's copy of the whole view's
|
|
1616
|
+
mask everywhere else. */
|
|
1617
|
+
if ( plan->route == KI_L2_FIBER_FUSED || plan->route == KI_L2_STACK
|
|
1618
|
+
|| plan->route == KI_L2_STACK_OUTER_K ) {
|
|
1619
|
+
st->alias_mask = NULL;
|
|
1620
|
+
}
|
|
1621
|
+
else {
|
|
1622
|
+
ki_take_mask_copy(st, src);
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
2037
1625
|
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
for ( k = 0; k < CA_RANK_MAX; k++ ) in_slab[k] = 0;
|
|
2049
|
-
for ( k = 0; k < naxes; k++ ) {
|
|
2050
|
-
int8_t ax = axes[k];
|
|
2051
|
-
if ( ax < 0 || ax >= nd ) return CA_ITER_ERR_POLICY;
|
|
2052
|
-
if ( in_slab[ax] ) return CA_ITER_ERR_POLICY; /* duplicate */
|
|
2053
|
-
in_slab[ax] = 1;
|
|
2054
|
-
}
|
|
1626
|
+
static int
|
|
1627
|
+
ca_iter_state_init_l2_unprotected (ca_iter_state *st,
|
|
1628
|
+
struct _CArray *src,
|
|
1629
|
+
ca_slab_policy_t policy,
|
|
1630
|
+
int8_t *axes,
|
|
1631
|
+
int8_t naxes,
|
|
1632
|
+
uint32_t flags)
|
|
1633
|
+
{
|
|
1634
|
+
ki_l2_plan plan;
|
|
1635
|
+
int rc;
|
|
2055
1636
|
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
}
|
|
2064
|
-
}
|
|
1637
|
+
/* Leave the state defined before anything can return. Every error
|
|
1638
|
+
return below hands control back to a caller that may not look at the
|
|
1639
|
+
code, so what it is left holding has to be a state that iterates zero
|
|
1640
|
+
times rather than whatever its stack frame happened to contain. */
|
|
1641
|
+
if ( st == NULL ) return CA_ITER_ERR_FLAGS;
|
|
1642
|
+
memset(st, 0, sizeof(*st));
|
|
1643
|
+
if ( src == NULL ) return CA_ITER_ERR_FLAGS;
|
|
2065
1644
|
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
st->slab_strides[sp] = st->composed_strides[k];
|
|
2074
|
-
st->slab_mask_strides[sp] = row_elem_strides[k];
|
|
2075
|
-
st->slab_elements *= src->dim[k];
|
|
2076
|
-
sp++;
|
|
2077
|
-
} else {
|
|
2078
|
-
st->outer_axes[op] = k;
|
|
2079
|
-
st->outer_dims[op] = src->dim[k];
|
|
2080
|
-
st->outer_strides[op] = st->composed_strides[k];
|
|
2081
|
-
st->outer_mask_strides[op] = row_elem_strides[k];
|
|
2082
|
-
op++;
|
|
2083
|
-
}
|
|
2084
|
-
}
|
|
2085
|
-
st->slab_ndim = sp;
|
|
2086
|
-
st->outer_ndim = op;
|
|
2087
|
-
|
|
2088
|
-
/* total_slabs = Π outer_dims. Empty product (all-axes case) = 1
|
|
2089
|
-
→ single slab = whole array (D1.4 WHOLE-equivalent). */
|
|
2090
|
-
ca_size_t total = 1;
|
|
2091
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) total *= st->outer_dims[m];
|
|
2092
|
-
st->total_slabs = total;
|
|
2093
|
-
st->slab_n = st->slab_elements; /* mirror to legacy field */
|
|
2094
|
-
st->slabs_emitted = 0;
|
|
2095
|
-
st->chunk_size = st->slab_n;
|
|
2096
|
-
|
|
2097
|
-
if ( st->outer_ndim > 0 ) {
|
|
2098
|
-
st->outer_idx = ALLOC_N(ca_size_t, st->outer_ndim);
|
|
2099
|
-
for ( int8_t m = 0; m < st->outer_ndim; m++ ) st->outer_idx[m] = 0;
|
|
2100
|
-
} else {
|
|
2101
|
-
st->outer_idx = NULL;
|
|
2102
|
-
}
|
|
1645
|
+
/* A Face only layers a semantic identifier and a CALazyMarker only
|
|
1646
|
+
layers "leaf of a lazy chain"; storage is identical to the parent
|
|
1647
|
+
either way, so strip at the kernel-compute entry and descend.
|
|
1648
|
+
Routing and alias decisions belong to the parent. Strip before
|
|
1649
|
+
validate_inputs: left on, both would classify as SRC_ATTACH (they
|
|
1650
|
+
have attach slots) and be materialised whole (see
|
|
1651
|
+
ca_iter_strip_storage_wrapper).
|
|
2103
1652
|
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
st->alias_mode = CA_ITER_ALIAS_STRIDED;
|
|
2108
|
-
st->alias_ptr = (char *) root->ptr;
|
|
2109
|
-
} else if ( use_view_scratch ) {
|
|
2110
|
-
st->root = NULL;
|
|
2111
|
-
st->scratch_cap = (ca_size_t) src->elements * src->bytes;
|
|
2112
|
-
st->scratch_ptr = (char *) xmalloc(st->scratch_cap > 0
|
|
2113
|
-
? st->scratch_cap : 1);
|
|
2114
|
-
if ( src->elements > 0 ) {
|
|
2115
|
-
ca_copy_data(src, st->scratch_ptr);
|
|
2116
|
-
}
|
|
2117
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
2118
|
-
st->alias_ptr = st->scratch_ptr;
|
|
2119
|
-
} else {
|
|
2120
|
-
st->root = NULL;
|
|
2121
|
-
ca_attach(src);
|
|
2122
|
-
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
2123
|
-
st->alias_ptr = (char *) src->ptr;
|
|
2124
|
-
}
|
|
1653
|
+
The Face identifier is re-wrapped onto the result by the caller's lift
|
|
1654
|
+
hook (= primary operators / reductions / etc.). The marker is not:
|
|
1655
|
+
a reduction over a lazy marker yields a plain entity.
|
|
2125
1656
|
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
st->scratch_mask = (boolean8_t *) xmalloc(mcap);
|
|
2132
|
-
if ( src->elements > 0 ) {
|
|
2133
|
-
ca_copy_data(src->mask, (char *) st->scratch_mask);
|
|
2134
|
-
}
|
|
2135
|
-
st->alias_mask = st->scratch_mask;
|
|
2136
|
-
}
|
|
2137
|
-
return CA_ITER_OK;
|
|
1657
|
+
WRITE is rejected against the wrapper, before the strip — a marker is
|
|
1658
|
+
read-only while its parent is not. validate_inputs re-checks the
|
|
1659
|
+
stripped source, which is harmless. */
|
|
1660
|
+
if ( (flags & CA_KERNEL_WRITE) && ca_is_readonly(src) ) {
|
|
1661
|
+
return CA_ITER_ERR_READONLY;
|
|
2138
1662
|
}
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
the slab. 0-d / 1-d sources collapse to a single yield. */
|
|
2142
|
-
if ( nd <= 1 ) {
|
|
2143
|
-
st->total_slabs = 1;
|
|
2144
|
-
st->slab_n = src->elements;
|
|
2145
|
-
st->outer_idx = NULL;
|
|
2146
|
-
} else {
|
|
2147
|
-
ca_size_t total = 1;
|
|
2148
|
-
int8_t k;
|
|
2149
|
-
for ( k = 0; k < nd - 1; k++ ) total *= src->dim[k];
|
|
2150
|
-
st->total_slabs = total;
|
|
2151
|
-
st->slab_n = src->dim[nd - 1];
|
|
2152
|
-
st->outer_idx = ALLOC_N(ca_size_t, nd - 1);
|
|
2153
|
-
for ( k = 0; k < nd - 1; k++ ) st->outer_idx[k] = 0;
|
|
1663
|
+
if ( ca_is_face(src) || ca_is_lazy_marker(src) ) {
|
|
1664
|
+
src = ca_iter_strip_storage_wrapper(src);
|
|
2154
1665
|
}
|
|
2155
|
-
st->slabs_emitted = 0;
|
|
2156
|
-
st->chunk_size = st->slab_n;
|
|
2157
1666
|
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
if ( src->elements > 0 ) {
|
|
2169
|
-
ca_copy_data(src, st->scratch_ptr);
|
|
2170
|
-
}
|
|
2171
|
-
st->alias_mode = CA_ITER_ALIAS_NONE;
|
|
2172
|
-
st->alias_ptr = st->scratch_ptr;
|
|
2173
|
-
} else {
|
|
2174
|
-
st->root = NULL;
|
|
2175
|
-
ca_attach(src);
|
|
2176
|
-
st->alias_mode = CA_ITER_ALIAS_CONTIG;
|
|
2177
|
-
st->alias_ptr = (char *) src->ptr;
|
|
1667
|
+
rc = ca_iter_validate_inputs(st, src, policy, flags);
|
|
1668
|
+
if ( rc != CA_ITER_OK ) return rc;
|
|
1669
|
+
/* L2 walks a slab of named axes. The whole-array walk is L1
|
|
1670
|
+
(init_l1 with CA_SLAB_WHOLE); one strided run per innermost row is
|
|
1671
|
+
CA_SLAB_AXES with the last axis. */
|
|
1672
|
+
if ( policy != CA_SLAB_AXES ) return CA_ITER_ERR_POLICY;
|
|
1673
|
+
|
|
1674
|
+
/* The slab axes: in range, no repeats. */
|
|
1675
|
+
if ( axes == NULL || naxes <= 0 || naxes > src->ndim ) {
|
|
1676
|
+
return CA_ITER_ERR_POLICY;
|
|
2178
1677
|
}
|
|
1678
|
+
memset(plan.in_slab, 0, sizeof(plan.in_slab));
|
|
1679
|
+
for ( int8_t k = 0; k < naxes; k++ ) {
|
|
1680
|
+
int8_t ax = axes[k];
|
|
1681
|
+
if ( ax < 0 || ax >= src->ndim ) return CA_ITER_ERR_POLICY;
|
|
1682
|
+
if ( plan.in_slab[ax] ) return CA_ITER_ERR_POLICY;
|
|
1683
|
+
plan.in_slab[ax] = 1;
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
/* route_source returns the refined src_kind and, for descriptor
|
|
1687
|
+
sources, the descriptors, so the choice below does not describe the
|
|
1688
|
+
axes again. */
|
|
1689
|
+
plan.ndesc = 0;
|
|
1690
|
+
plan.root = NULL;
|
|
1691
|
+
plan.base = 0;
|
|
1692
|
+
plan.src_kind = ca_iter_route_source(src, plan.descs, plan.pdims,
|
|
1693
|
+
&plan.ndesc);
|
|
1694
|
+
rc = ki_choose_l2_route(src, axes, naxes, flags, &plan);
|
|
1695
|
+
if ( rc != CA_ITER_OK ) return rc;
|
|
1696
|
+
|
|
1697
|
+
ki_open_l2(st, src, policy, axes, naxes, flags, &plan);
|
|
2179
1698
|
return CA_ITER_OK;
|
|
2180
1699
|
}
|
|
2181
1700
|
|
|
@@ -2202,73 +1721,54 @@ ca_iter_state_next_slab_unprotected (ca_iter_state *st,
|
|
|
2202
1721
|
return 1;
|
|
2203
1722
|
}
|
|
2204
1723
|
|
|
1724
|
+
|
|
1725
|
+
/* CA_KERNEL_FIBER_CONTIG: a fiber (a walk with one slab axis) reaches the
|
|
1726
|
+
kernel as a contiguous run of cells and of mask flags. Each path of
|
|
1727
|
+
next_slab_axes finds the fiber's first cell and the step along it, and
|
|
1728
|
+
hands both through these two functions, which copy a fiber with any
|
|
1729
|
+
other step into the walk's fiber scratch. Other walks get the pointer
|
|
1730
|
+
back unchanged. */
|
|
2205
1731
|
static int
|
|
2206
|
-
|
|
2207
|
-
char **out_ptr,
|
|
2208
|
-
boolean8_t **out_mask,
|
|
2209
|
-
ca_size_t *out_n,
|
|
2210
|
-
ca_size_t *out_stride_bytes)
|
|
1732
|
+
ki_fiber_contig_wanted (const ca_iter_state *st)
|
|
2211
1733
|
{
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
if ( out_mask ) *out_mask = NULL;
|
|
2216
|
-
if ( out_n ) *out_n = 0;
|
|
2217
|
-
if ( out_stride_bytes ) *out_stride_bytes = 0;
|
|
2218
|
-
return 0;
|
|
2219
|
-
}
|
|
1734
|
+
return (st->flags & CA_KERNEL_FIBER_CONTIG)
|
|
1735
|
+
&& st->naxes == 1 && st->slab_ndim == 1;
|
|
1736
|
+
}
|
|
2220
1737
|
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
ca_size_t
|
|
2226
|
-
|
|
1738
|
+
static char *
|
|
1739
|
+
ki_fiber_contig_data (ca_iter_state *st, char *data, ca_size_t step)
|
|
1740
|
+
{
|
|
1741
|
+
ca_size_t n = st->slab_dims[0];
|
|
1742
|
+
ca_size_t bytes = st->bytes;
|
|
1743
|
+
ca_size_t need = n * bytes;
|
|
1744
|
+
if ( ! ki_fiber_contig_wanted(st) || step == bytes ) {
|
|
1745
|
+
return data;
|
|
1746
|
+
}
|
|
1747
|
+
if ( st->fiber_data_scratch_cap < need ) {
|
|
1748
|
+
if ( st->fiber_data_scratch ) ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
|
|
1749
|
+
st->fiber_data_scratch = (char *) ki_data_alloc(st->src->data_type, need);
|
|
1750
|
+
st->fiber_data_scratch_cap = need;
|
|
1751
|
+
}
|
|
1752
|
+
ca_stride_gather_run(st->fiber_data_scratch, data, bytes, n, step);
|
|
1753
|
+
return st->fiber_data_scratch;
|
|
1754
|
+
}
|
|
2227
1755
|
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
composed_base. */
|
|
2235
|
-
if ( st->outer_idx != NULL ) {
|
|
2236
|
-
if ( st->src_kind == CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE ) {
|
|
2237
|
-
int oob = 0;
|
|
2238
|
-
off += ca_axis_dispatch_prefix_offset(st->prefix, st->outer_idx,
|
|
2239
|
-
nd - 1, &oob);
|
|
2240
|
-
/* oob unreachable: outer SHIFT was downgraded in init_l2. */
|
|
2241
|
-
} else {
|
|
2242
|
-
for ( k = 0; k < nd - 1; k++ ) {
|
|
2243
|
-
off += st->outer_idx[k] * st->composed_strides[k];
|
|
2244
|
-
}
|
|
2245
|
-
}
|
|
1756
|
+
static boolean8_t *
|
|
1757
|
+
ki_fiber_contig_mask (ca_iter_state *st, boolean8_t *mask, ca_size_t step)
|
|
1758
|
+
{
|
|
1759
|
+
ca_size_t n = st->slab_dims[0];
|
|
1760
|
+
if ( mask == NULL || ! ki_fiber_contig_wanted(st) || step == 1 ) {
|
|
1761
|
+
return mask;
|
|
2246
1762
|
}
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
gather mask into scratch_mask for uniformity). Per-slab mask
|
|
2252
|
-
offset = i in the outer cursor (= slabs_emitted at this point). */
|
|
2253
|
-
if ( out_mask ) {
|
|
2254
|
-
*out_mask = st->alias_mask
|
|
2255
|
-
? st->alias_mask + st->slabs_emitted * st->slab_n
|
|
2256
|
-
: NULL;
|
|
1763
|
+
if ( st->fiber_mask_scratch_cap < n ) {
|
|
1764
|
+
if ( st->fiber_mask_scratch ) xfree(st->fiber_mask_scratch);
|
|
1765
|
+
st->fiber_mask_scratch = (boolean8_t *) xmalloc(n > 0 ? n : 1);
|
|
1766
|
+
st->fiber_mask_scratch_cap = n;
|
|
2257
1767
|
}
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
st->slabs_emitted += 1;
|
|
2262
|
-
|
|
2263
|
-
/* Advance outer_idx row-major (least-significant axis innermost,
|
|
2264
|
-
so we tick outer_idx[ndim-2] first). */
|
|
2265
|
-
if ( st->outer_idx != NULL ) {
|
|
2266
|
-
for ( k = nd - 2; k >= 0; k-- ) {
|
|
2267
|
-
if ( ++st->outer_idx[k] < st->src->dim[k] ) break;
|
|
2268
|
-
st->outer_idx[k] = 0;
|
|
2269
|
-
}
|
|
1768
|
+
for ( ca_size_t i = 0; i < n; i++ ) {
|
|
1769
|
+
st->fiber_mask_scratch[i] = mask[i * step];
|
|
2270
1770
|
}
|
|
2271
|
-
return
|
|
1771
|
+
return st->fiber_mask_scratch;
|
|
2272
1772
|
}
|
|
2273
1773
|
|
|
2274
1774
|
static int
|
|
@@ -2283,12 +1783,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2283
1783
|
return 0;
|
|
2284
1784
|
}
|
|
2285
1785
|
|
|
2286
|
-
/*
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
engine per-cell dispatch). Target: 1.5-1.8x win for INDEX slab
|
|
2291
|
-
with innermost STRIDE (= grid / select sparse projection use cases). */
|
|
1786
|
+
/* PER_SLAB_HOIST (see its name tag in ki_open_l2): the innermost slab
|
|
1787
|
+
axis is STRIDE and no axis is SHIFT. The outer and non-innermost
|
|
1788
|
+
slab offsets are computed once per slab row, and the inner run is a
|
|
1789
|
+
plain memcpy instead of the descriptor engine's per-cell walk. */
|
|
2292
1790
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB_HOIST ) {
|
|
2293
1791
|
int8_t inner_view_ax = st->slab_axes_buf[st->slab_ndim - 1];
|
|
2294
1792
|
ca_size_t inner_count = st->descs[inner_view_ax].count;
|
|
@@ -2370,7 +1868,11 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2370
1868
|
}
|
|
2371
1869
|
|
|
2372
1870
|
if ( out_ptr ) *out_ptr = st->scratch_ptr;
|
|
2373
|
-
if ( out_mask )
|
|
1871
|
+
if ( out_mask ) {
|
|
1872
|
+
*out_mask = ki_fiber_contig_mask(st,
|
|
1873
|
+
st->scratch_mask ? st->scratch_mask + mask_off : NULL,
|
|
1874
|
+
st->slab_mask_strides[0]);
|
|
1875
|
+
}
|
|
2374
1876
|
|
|
2375
1877
|
st->slabs_emitted += 1;
|
|
2376
1878
|
for ( int8_t m = st->outer_ndim - 1; m >= 0; m-- ) {
|
|
@@ -2380,17 +1882,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2380
1882
|
return 1;
|
|
2381
1883
|
}
|
|
2382
1884
|
|
|
2383
|
-
/*
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
k_axis -- stack axis s != k_axis maps to parent axis s if
|
|
2393
|
-
s < k_axis else s - 1. */
|
|
1885
|
+
/* CAStack source with the stack axis k_axis in the outer iteration.
|
|
1886
|
+
Each slab is a region inside parents[k]->ptr where k =
|
|
1887
|
+
outer_idx[K_outer_pos]. Parent inner byte offset = Σ outer_idx[m] *
|
|
1888
|
+
stack_parent_strides[parent_ax] over all outer axes except the K-axis
|
|
1889
|
+
itself. Mask: a pointer into the copy of parent k's mask that init
|
|
1890
|
+
took (stack_parent_mask_ptrs[k] + mask_off); a write through it
|
|
1891
|
+
reaches nothing. The data is yielded in place unless FIBER_CONTIG
|
|
1892
|
+
needs a strided fiber gathered (ki_fiber_contig_data). A stack axis
|
|
1893
|
+
s other than k_axis is parent axis s, or s - 1 past k_axis. */
|
|
2394
1894
|
if ( st->alias_mode == CA_ITER_ALIAS_STACK_OUTER_K ) {
|
|
2395
1895
|
int8_t kpos = st->stack_k_outer_pos;
|
|
2396
1896
|
int32_t k = (int32_t) st->outer_idx[kpos];
|
|
@@ -2406,11 +1906,15 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2406
1906
|
mask_off += st->outer_idx[m] * st->stack_parent_mask_strides[parent_ax];
|
|
2407
1907
|
}
|
|
2408
1908
|
|
|
2409
|
-
if ( out_ptr )
|
|
1909
|
+
if ( out_ptr ) {
|
|
1910
|
+
*out_ptr = ki_fiber_contig_data(st, st->stack_parent_ptrs[k] + parent_off,
|
|
1911
|
+
st->slab_strides[0]);
|
|
1912
|
+
}
|
|
2410
1913
|
if ( out_mask ) {
|
|
2411
|
-
*out_mask = st
|
|
2412
|
-
|
|
2413
|
-
|
|
1914
|
+
*out_mask = ki_fiber_contig_mask(st,
|
|
1915
|
+
st->stack_parent_mask_ptrs
|
|
1916
|
+
? st->stack_parent_mask_ptrs[k] + mask_off : NULL,
|
|
1917
|
+
st->slab_mask_strides[0]);
|
|
2414
1918
|
}
|
|
2415
1919
|
|
|
2416
1920
|
st->slabs_emitted += 1;
|
|
@@ -2421,20 +1925,19 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2421
1925
|
return 1;
|
|
2422
1926
|
}
|
|
2423
1927
|
|
|
2424
|
-
/*
|
|
2425
|
-
|
|
1928
|
+
/* CAStack read along its stack axis, from the parents' memory. For
|
|
1929
|
+
each outer index, compute the
|
|
2426
1930
|
parent inner byte offset (= same across all K parents, uniform
|
|
2427
|
-
shape),
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
stack axes
|
|
2434
|
-
maps
|
|
2435
|
-
stack axis m+1 = parent axis m). */
|
|
1931
|
+
shape), and yield the K cells of that fiber out of stack_tile_cache,
|
|
1932
|
+
which is refilled TILE fibers at a time from the cached parent ptrs
|
|
1933
|
+
(below). No ca_xfer_stride dispatch. The scratch_ptr init
|
|
1934
|
+
allocates is not used by this path.
|
|
1935
|
+
|
|
1936
|
+
init engages this path only for slab_axes == [k_axis], so the outer
|
|
1937
|
+
axes are the stack axes other than k_axis, which are the parent axes
|
|
1938
|
+
in order: outer_idx[m] maps to parent axis m. */
|
|
2436
1939
|
if ( st->alias_mode == CA_ITER_ALIAS_STACK ) {
|
|
2437
|
-
/*
|
|
1940
|
+
/* Tile cache. Outer iter
|
|
2438
1941
|
walks parent storage in row-major order so parent_off increments
|
|
2439
1942
|
by `bytes` per call monotonically. Refill TILE fibers at once
|
|
2440
1943
|
via K contig parent streams (= TILE consecutive cells from each
|
|
@@ -2504,11 +2007,10 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2504
2007
|
return 1;
|
|
2505
2008
|
}
|
|
2506
2009
|
|
|
2507
|
-
/*
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
for one max slab and reused across iters (D1.1 (B)). */
|
|
2010
|
+
/* PER_SLAB: one slab gathered by ca_axis_dispatch_gather, from a
|
|
2011
|
+
subset of the descriptors with the outer axes pinned at the current
|
|
2012
|
+
outer_idx position. scratch_ptr is sized for one slab and reused for
|
|
2013
|
+
every slab. */
|
|
2512
2014
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_SLAB ) {
|
|
2513
2015
|
ca_axis_desc_t subset_descs[CA_RANK_MAX];
|
|
2514
2016
|
memcpy(subset_descs, st->descs, st->ndim * sizeof(ca_axis_desc_t));
|
|
@@ -2543,7 +2045,7 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2543
2045
|
}
|
|
2544
2046
|
}
|
|
2545
2047
|
|
|
2546
|
-
/* CAWindow fill value
|
|
2048
|
+
/* CAWindow fill value (the same lookup as in ki_open_l2). */
|
|
2547
2049
|
const void *bound_fill = NULL;
|
|
2548
2050
|
if ( ca_func[st->src->obj_type].attach == ca_window_func.attach ) {
|
|
2549
2051
|
bound_fill = ((CAWindow *) st->src)->fill;
|
|
@@ -2553,15 +2055,19 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2553
2055
|
st->ndim, st->bytes, st->slab_elements,
|
|
2554
2056
|
bound_fill, st->scratch_ptr);
|
|
2555
2057
|
|
|
2556
|
-
/* Mask offset into whole-view
|
|
2557
|
-
|
|
2058
|
+
/* Mask offset into the whole-view mask copy, which is view
|
|
2059
|
+
row-major to match slab_mask_strides. */
|
|
2558
2060
|
ca_size_t mask_off = 0;
|
|
2559
2061
|
for ( int8_t m = 0; m < st->outer_ndim; m++ ) {
|
|
2560
2062
|
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
2561
2063
|
}
|
|
2562
2064
|
|
|
2563
2065
|
if ( out_ptr ) *out_ptr = st->scratch_ptr;
|
|
2564
|
-
if ( out_mask )
|
|
2066
|
+
if ( out_mask ) {
|
|
2067
|
+
*out_mask = ki_fiber_contig_mask(st,
|
|
2068
|
+
st->scratch_mask ? st->scratch_mask + mask_off : NULL,
|
|
2069
|
+
st->slab_mask_strides[0]);
|
|
2070
|
+
}
|
|
2565
2071
|
|
|
2566
2072
|
st->slabs_emitted += 1;
|
|
2567
2073
|
/* Advance outer_idx row-major (innermost outer axis ticks first). */
|
|
@@ -2577,10 +2083,9 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2577
2083
|
(element units, view row-major). Both are zero when outer_ndim
|
|
2578
2084
|
== 0 (= all-axes WHOLE-equivalent case).
|
|
2579
2085
|
|
|
2580
|
-
STRIDE outer axes (
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
|
|
2086
|
+
STRIDE outer axes (SRC_CASTRIDE, and the STRIDE axes of a descriptor
|
|
2087
|
+
view) use outer_idx[m] directly as the multiplier. INDEX outer axes
|
|
2088
|
+
of a descriptor view need an indices[] lookup: multiplier = descs[axis].indices[outer_idx[m]].
|
|
2584
2089
|
Mask offset uses outer_idx[m] directly in both cases (= mask
|
|
2585
2090
|
scratch is in view row-major order, outer_idx walks view-axis
|
|
2586
2091
|
positions). */
|
|
@@ -2600,13 +2105,12 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2600
2105
|
mask_off += st->outer_idx[m] * st->outer_mask_strides[m];
|
|
2601
2106
|
}
|
|
2602
2107
|
|
|
2603
|
-
/*
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
region path.
|
|
2108
|
+
/* PER_FIBER_FUSED: there is no whole-view buffer to alias from. Build
|
|
2109
|
+
the fiber region from outer_idx + fiber_axis and call
|
|
2110
|
+
ca_xfer_stride(src, ..., GET) directly into fiber_data_scratch; the
|
|
2111
|
+
view's region transfer (out-of-bounds filling for CAWindow / CAShift,
|
|
2112
|
+
the conversion for a fake or byte-swap view) delivers it in one
|
|
2113
|
+
pass.
|
|
2610
2114
|
|
|
2611
2115
|
last_data_off carries the fiber region's outer footprint so
|
|
2612
2116
|
sync_slab can reconstruct the same region for WRITE PUT. Encode
|
|
@@ -2622,16 +2126,16 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2622
2126
|
ca_size_t counts[CA_RANK_MAX];
|
|
2623
2127
|
|
|
2624
2128
|
if ( st->fiber_data_scratch_cap < need ) {
|
|
2625
|
-
if ( st->fiber_data_scratch )
|
|
2626
|
-
st->fiber_data_scratch = (char *)
|
|
2129
|
+
if ( st->fiber_data_scratch ) ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
|
|
2130
|
+
st->fiber_data_scratch = (char *) ki_data_alloc(st->src->data_type, need);
|
|
2627
2131
|
st->fiber_data_scratch_cap = need;
|
|
2628
2132
|
}
|
|
2629
2133
|
|
|
2630
2134
|
/* Build fiber region: fiber_axis spans the full fiber, all other
|
|
2631
2135
|
axes pinned to outer_idx position (count=1). Cache starts[] in
|
|
2632
2136
|
state so sync_slab can rebuild the same region for WRITE PUT
|
|
2633
|
-
(= captured BEFORE outer_idx advance below, same hazard
|
|
2634
|
-
|
|
2137
|
+
(= captured BEFORE the outer_idx advance below, the same hazard as
|
|
2138
|
+
last_data_off; see sync_slab). */
|
|
2635
2139
|
{
|
|
2636
2140
|
int8_t op = 0;
|
|
2637
2141
|
for ( int8_t k = 0; k < nd; k++ ) {
|
|
@@ -2691,82 +2195,23 @@ ca_iter_state_next_slab_axes_unprotected (ca_iter_state *st,
|
|
|
2691
2195
|
return 1;
|
|
2692
2196
|
}
|
|
2693
2197
|
|
|
2694
|
-
/*
|
|
2695
|
-
For naxes==1 (= single slab axis = fiber) the catalog contract
|
|
2198
|
+
/* Contiguous fiber delivery (promise 1). For naxes==1 (= single slab axis = fiber) the catalog contract
|
|
2696
2199
|
CA_FOR_EACH_FIBER promises contig data delivery. When the fiber
|
|
2697
2200
|
is not innermost-contig (= slab_strides[0] != bytes), the engine
|
|
2698
2201
|
gathers the fiber into fiber_data_scratch before yielding so the
|
|
2699
2202
|
author can write p[i] without stride math.
|
|
2700
2203
|
|
|
2701
2204
|
Capture last_data_off BEFORE the outer_idx advance below; sync_slab
|
|
2702
|
-
consumes it for WRITE scatter. See header field
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
st->last_data_off = data_off;
|
|
2713
|
-
|
|
2714
|
-
if ( data_step == (ca_size_t) bytes ) {
|
|
2715
|
-
/* Fast path: fiber is already contig (= innermost-axis or stride
|
|
2716
|
-
coincidentally == bytes). No gather needed. */
|
|
2717
|
-
yield_ptr = src_data;
|
|
2718
|
-
} else {
|
|
2719
|
-
/* Per-fiber gather via the typed-store inline helper
|
|
2720
|
-
(ca_iter_substrate.h). For bytes in {1,2,4,8} this uses a
|
|
2721
|
-
compiler-vectorize-friendly `*dp++ = v; sp += step` loop;
|
|
2722
|
-
other sizes fall back to per-element memcpy. Lazy-alloc
|
|
2723
|
-
scratch sized to max fiber bytes (slab_dims[0] constant per
|
|
2724
|
-
walk → single alloc in practice). */
|
|
2725
|
-
ca_size_t need = n * bytes;
|
|
2726
|
-
if ( st->fiber_data_scratch_cap < need ) {
|
|
2727
|
-
if ( st->fiber_data_scratch ) xfree(st->fiber_data_scratch);
|
|
2728
|
-
st->fiber_data_scratch = (char *) xmalloc(need);
|
|
2729
|
-
st->fiber_data_scratch_cap = need;
|
|
2730
|
-
}
|
|
2731
|
-
ca_stride_gather_run(st->fiber_data_scratch, src_data,
|
|
2732
|
-
bytes, n, data_step);
|
|
2733
|
-
yield_ptr = st->fiber_data_scratch;
|
|
2734
|
-
}
|
|
2735
|
-
} else {
|
|
2736
|
-
yield_ptr = st->alias_ptr + data_off;
|
|
2737
|
-
}
|
|
2738
|
-
|
|
2739
|
-
/* PROPOSAL_FIBER_DELIVERY F.1b: per-fiber contig mask delivery.
|
|
2740
|
-
Symmetric to F.1a data path above. When the source has a mask
|
|
2741
|
-
(= alias_mask != NULL) and the fiber's mask is not innermost-contig
|
|
2742
|
-
(= slab_mask_strides[0] != 1), gather it into fiber_mask_scratch
|
|
2743
|
-
so the author can write m[i] without stride math. Mask is
|
|
2744
|
-
read-only here (= L2 WRITE never propagates to mask state), so no
|
|
2745
|
-
scatter is needed in sync_slab. */
|
|
2746
|
-
boolean8_t *yield_mask;
|
|
2747
|
-
if ( (st->flags & CA_KERNEL_FIBER_CONTIG)
|
|
2748
|
-
&& st->naxes == 1 && st->slab_ndim == 1
|
|
2749
|
-
&& st->alias_mask != NULL ) {
|
|
2750
|
-
ca_size_t n = st->slab_dims[0];
|
|
2751
|
-
ca_size_t mask_step = st->slab_mask_strides[0];
|
|
2752
|
-
boolean8_t *src_mask = st->alias_mask + mask_off;
|
|
2753
|
-
|
|
2754
|
-
if ( mask_step == 1 ) {
|
|
2755
|
-
yield_mask = src_mask;
|
|
2756
|
-
} else {
|
|
2757
|
-
if ( st->fiber_mask_scratch_cap < (ca_size_t) n ) {
|
|
2758
|
-
if ( st->fiber_mask_scratch ) xfree(st->fiber_mask_scratch);
|
|
2759
|
-
st->fiber_mask_scratch = (boolean8_t *) xmalloc(n);
|
|
2760
|
-
st->fiber_mask_scratch_cap = n;
|
|
2761
|
-
}
|
|
2762
|
-
for ( ca_size_t i = 0; i < n; i++ ) {
|
|
2763
|
-
st->fiber_mask_scratch[i] = src_mask[i * mask_step];
|
|
2764
|
-
}
|
|
2765
|
-
yield_mask = st->fiber_mask_scratch;
|
|
2766
|
-
}
|
|
2767
|
-
} else {
|
|
2768
|
-
yield_mask = st->alias_mask ? st->alias_mask + mask_off : NULL;
|
|
2769
|
-
}
|
|
2205
|
+
consumes it for WRITE scatter. See the header field and the hazard
|
|
2206
|
+
comment in sync_slab. */
|
|
2207
|
+
/* sync_slab finds the slab here: the fiber it scatters back, and the
|
|
2208
|
+
cells of a whole-view scratch it compares. */
|
|
2209
|
+
st->last_data_off = data_off;
|
|
2210
|
+
char *yield_ptr = ki_fiber_contig_data(st, st->alias_ptr + data_off,
|
|
2211
|
+
st->slab_strides[0]);
|
|
2212
|
+
boolean8_t *yield_mask = ki_fiber_contig_mask(st,
|
|
2213
|
+
st->alias_mask ? st->alias_mask + mask_off : NULL,
|
|
2214
|
+
st->slab_mask_strides[0]);
|
|
2770
2215
|
|
|
2771
2216
|
if ( out_ptr ) *out_ptr = yield_ptr;
|
|
2772
2217
|
if ( out_mask ) *out_mask = yield_mask;
|
|
@@ -2805,8 +2250,6 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
2805
2250
|
/* ======================================================================
|
|
2806
2251
|
* !!! CORRECTNESS HAZARD - DO NOT MOVE !!!
|
|
2807
2252
|
*
|
|
2808
|
-
* (PROPOSAL_FIBER_DELIVERY F.1a)
|
|
2809
|
-
*
|
|
2810
2253
|
* Per-fiber scratch reuse + scatter correctness depends on the strict
|
|
2811
2254
|
* evaluation order of the CA_FOR_EACH_FIBER_* macro sandwich:
|
|
2812
2255
|
*
|
|
@@ -2833,14 +2276,13 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
2833
2276
|
* instead of reusing -- separate phase, separate design.
|
|
2834
2277
|
* ====================================================================== */
|
|
2835
2278
|
|
|
2836
|
-
/*
|
|
2837
|
-
When alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED, the author
|
|
2279
|
+
/* PER_FIBER_FUSED write: the author
|
|
2838
2280
|
wrote into fiber_data_scratch and there is no whole-view buffer
|
|
2839
2281
|
to xfer_all PUT. Rebuild the same fiber region from cached
|
|
2840
2282
|
fiber_region_starts[] (= captured pre-advance in next_slab_axes,
|
|
2841
|
-
same hazard
|
|
2842
|
-
|
|
2843
|
-
|
|
2283
|
+
the same hazard as last_data_off) and call ca_xfer_stride(src, ...,
|
|
2284
|
+
PUT), which puts the fiber back through the view's region transfer
|
|
2285
|
+
in one pass. */
|
|
2844
2286
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
|
|
2845
2287
|
ca_size_t fiber_n = st->slab_dims[0];
|
|
2846
2288
|
int8_t nd = st->ndim;
|
|
@@ -2855,7 +2297,7 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
2855
2297
|
return;
|
|
2856
2298
|
}
|
|
2857
2299
|
|
|
2858
|
-
/* Per-fiber gather path scatter
|
|
2300
|
+
/* Per-fiber gather path scatter.
|
|
2859
2301
|
When next_slab_axes gathered the fiber into fiber_data_scratch
|
|
2860
2302
|
(= naxes==1 + slab_strides[0] != bytes), scatter it back to the
|
|
2861
2303
|
source layout via the strided write. When the fiber was the
|
|
@@ -2880,87 +2322,57 @@ ca_iter_state_sync_slab_unprotected (ca_iter_state *st)
|
|
|
2880
2322
|
it fires for any source whose fiber is not innermost-contig -- so
|
|
2881
2323
|
for SRC_ATTACH this has just written the fiber back into the
|
|
2882
2324
|
whole-view scratch, and the push-back below carries it to the
|
|
2883
|
-
source. For the alias paths scratch_ptr is NULL and the
|
|
2884
|
-
below
|
|
2885
|
-
}
|
|
2886
|
-
|
|
2887
|
-
/* SRC_ATTACH path (step 9 + 2026-05-31 refactor): kernel wrote into
|
|
2888
|
-
iterator-owned scratch (= scratch_ptr). Push back via xfer_all PUT
|
|
2889
|
-
which routes through the view's xfer_all slot -- handles CAFake
|
|
2890
|
-
(cast back), CAByteSwap (swap back), CABitfield/CABitarray (bit
|
|
2891
|
-
pack back), CAReduce (broadcast across reduce window). Inherits
|
|
2892
|
-
transform-fused / partial materialise / etc. automatically.
|
|
2893
|
-
|
|
2894
|
-
This has to stay below the two per-fiber blocks: consulting
|
|
2895
|
-
src_kind first would push back the whole-view buffer while the
|
|
2896
|
-
fiber the author actually wrote sat unscattered in
|
|
2897
|
-
fiber_data_scratch, and would push back a NULL one for
|
|
2898
|
-
PER_FIBER_FUSED, which owns no whole-view buffer at all. */
|
|
2899
|
-
if ( st->src_kind == CA_ITER_SRC_ATTACH ) {
|
|
2900
|
-
if ( st->src->elements > 0 ) {
|
|
2901
|
-
ca_xfer_all(st->src, st->scratch_ptr, CA_XFER_PUT);
|
|
2902
|
-
}
|
|
2903
|
-
st->write_dirty = 0;
|
|
2904
|
-
return;
|
|
2325
|
+
source. For the alias paths scratch_ptr is NULL and the push-back
|
|
2326
|
+
below is skipped. */
|
|
2905
2327
|
}
|
|
2906
2328
|
|
|
2907
|
-
/* alias path: kernel wrote through alias_ptr into
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
stays NULL and we no-op correctly). */
|
|
2329
|
+
/* alias path: the kernel wrote through alias_ptr into the memory the
|
|
2330
|
+
walk borrowed, so nothing is scattered. Applies to all CAStride alias and to
|
|
2331
|
+
STRIDE-only descriptor alias (SRC_DESCRIPTOR_L2_ALIASABLE), whose
|
|
2332
|
+
scratch_ptr is NULL too. An alias walk over a view root is written
|
|
2333
|
+
back once, by finish (ca_sync_detach on the root). */
|
|
2913
2334
|
if ( st->scratch_ptr == NULL ) return;
|
|
2914
2335
|
|
|
2915
|
-
/* scratch path:
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
destination), CAShift WRAP/REFLECT bounds map back to interior
|
|
2935
|
-
and are written normally. Iteration order is engine-defined;
|
|
2936
|
-
user kernels must not rely on it.
|
|
2937
|
-
|
|
2938
|
-
scatter wants the pre-merge raw descriptors (the engine re-runs
|
|
2939
|
-
_prepare internally with whatever we give it). init cached
|
|
2940
|
-
the post-merge axes in st->descs for next_slab_strided's
|
|
2941
|
-
offset math; we re-call describe_axes here for the scatter
|
|
2942
|
-
call rather than caching a second copy in the state struct. */
|
|
2943
|
-
ca_axis_desc_t raw_descs[CA_RANK_MAX];
|
|
2944
|
-
ca_size_t raw_pdims[CA_RANK_MAX];
|
|
2945
|
-
int8_t raw_ndim = 0;
|
|
2946
|
-
ca_iter_describe_axes(st->src, raw_descs, raw_pdims, &raw_ndim);
|
|
2947
|
-
ca_axis_dispatch_scatter(st->root /* parent */,
|
|
2948
|
-
raw_pdims,
|
|
2949
|
-
raw_descs, raw_ndim,
|
|
2950
|
-
st->bytes, st->total_elements,
|
|
2951
|
-
st->scratch_ptr);
|
|
2952
|
-
break;
|
|
2953
|
-
}
|
|
2954
|
-
}
|
|
2336
|
+
/* scratch path: the kernel wrote into a whole-view scratch, and the
|
|
2337
|
+
changed cells go back through the view's own transfer, which
|
|
2338
|
+
converts, packs or scatters them as the view requires (promise 3).
|
|
2339
|
+
Every write walk that reads a scratch gathered it with
|
|
2340
|
+
ki_gather_view, which keeps the copy the comparison needs: the
|
|
2341
|
+
transfer walk (SRC_ATTACH) and the CAStride paths that read a
|
|
2342
|
+
gathered copy. The descriptor engine's own gathers serve reads
|
|
2343
|
+
only; a write over a descriptor view is sent to the transfer walk
|
|
2344
|
+
at init.
|
|
2345
|
+
|
|
2346
|
+
This has to stay below the two per-fiber blocks: pushing back first
|
|
2347
|
+
would send the whole-view buffer while the fiber the author actually
|
|
2348
|
+
wrote sat unscattered in fiber_data_scratch. */
|
|
2349
|
+
if ( st->src_kind != CA_ITER_SRC_ATTACH
|
|
2350
|
+
&& st->src_kind != CA_ITER_SRC_CASTRIDE ) {
|
|
2351
|
+
rb_bug("kernel iterator: a write walk over a scratch with nothing to "
|
|
2352
|
+
"compare it against (src_kind %d)", (int) st->src_kind);
|
|
2353
|
+
}
|
|
2354
|
+
ki_scratch_push_changed(st);
|
|
2955
2355
|
st->write_dirty = 0;
|
|
2956
2356
|
}
|
|
2957
2357
|
|
|
2358
|
+
static VALUE ca_iter_finish_sync (VALUE arg);
|
|
2359
|
+
|
|
2360
|
+
static VALUE
|
|
2361
|
+
ki_sync_detach_root (VALUE root)
|
|
2362
|
+
{
|
|
2363
|
+
ca_sync_detach((CArray *) root);
|
|
2364
|
+
return Qnil;
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2958
2367
|
void
|
|
2959
2368
|
ca_iter_state_finish (ca_iter_state *st)
|
|
2960
2369
|
{
|
|
2961
2370
|
if ( st == NULL || st->src == NULL ) {
|
|
2962
2371
|
return;
|
|
2963
2372
|
}
|
|
2373
|
+
/* A slab still handed out is dropped: finish abandons it. The block
|
|
2374
|
+
macros close with ca_iter_state_close, which writes it back first. */
|
|
2375
|
+
st->write_dirty = 0;
|
|
2964
2376
|
/* composed_strides is inline — no free needed. outer_idx is heap
|
|
2965
2377
|
for L2 multi-d sources (NULL on L1 paths and on L2 0/1-d). */
|
|
2966
2378
|
if ( st->outer_idx ) {
|
|
@@ -2970,14 +2382,14 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
2970
2382
|
/* Lifecycle cleanup — orders matter slightly (free scratch before
|
|
2971
2383
|
detaching parent so the kernel iterator's resources are released
|
|
2972
2384
|
symmetrically with init):
|
|
2973
|
-
- scratch_ptr: owned by
|
|
2974
|
-
descriptor
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
ca_detach.
|
|
2385
|
+
- scratch_ptr: owned by the iterator (every gather into scratch:
|
|
2386
|
+
CAStride non-contig, descriptor, SRC_ATTACH, per-slab, stack).
|
|
2387
|
+
ki_data_free.
|
|
2388
|
+
- root: descriptor parent or CAStride compose-fold root attached
|
|
2389
|
+
at init. ca_detach, or ca_sync_detach on a WRITE walk.
|
|
2978
2390
|
- else (alias paths): src was attached at init, detach. */
|
|
2979
2391
|
if ( st->scratch_ptr ) {
|
|
2980
|
-
|
|
2392
|
+
ki_data_free(st->scratch_ptr, st->scratch_cap);
|
|
2981
2393
|
st->scratch_ptr = NULL;
|
|
2982
2394
|
st->scratch_cap = 0;
|
|
2983
2395
|
}
|
|
@@ -2985,9 +2397,9 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
2985
2397
|
xfree(st->scratch_mask);
|
|
2986
2398
|
st->scratch_mask = NULL;
|
|
2987
2399
|
}
|
|
2988
|
-
/*
|
|
2400
|
+
/* The per-fiber scratch. */
|
|
2989
2401
|
if ( st->fiber_data_scratch ) {
|
|
2990
|
-
|
|
2402
|
+
ki_data_free(st->fiber_data_scratch, st->fiber_data_scratch_cap);
|
|
2991
2403
|
st->fiber_data_scratch = NULL;
|
|
2992
2404
|
st->fiber_data_scratch_cap = 0;
|
|
2993
2405
|
}
|
|
@@ -2996,9 +2408,10 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
2996
2408
|
st->fiber_mask_scratch = NULL;
|
|
2997
2409
|
st->fiber_mask_scratch_cap = 0;
|
|
2998
2410
|
}
|
|
2999
|
-
/*
|
|
3000
|
-
|
|
3001
|
-
|
|
2411
|
+
/* Detach the K parents init attached and free the cached ptr array
|
|
2412
|
+
(symmetric with init_l2's per-parent ca_attach loop). The parent
|
|
2413
|
+
mask copies are freed separately below; they were never attached
|
|
2414
|
+
for the walk. */
|
|
3002
2415
|
if ( st->stack_parent_ptrs ) {
|
|
3003
2416
|
CAStack *stack = (CAStack *) st->src;
|
|
3004
2417
|
for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
|
|
@@ -3007,37 +2420,87 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3007
2420
|
xfree(st->stack_parent_ptrs);
|
|
3008
2421
|
st->stack_parent_ptrs = NULL;
|
|
3009
2422
|
}
|
|
3010
|
-
/*
|
|
2423
|
+
/* The STACK tile cache. */
|
|
3011
2424
|
if ( st->stack_tile_cache ) {
|
|
3012
|
-
|
|
2425
|
+
ki_data_free(st->stack_tile_cache, 0);
|
|
3013
2426
|
st->stack_tile_cache = NULL;
|
|
3014
2427
|
st->stack_tile_cap = 0;
|
|
3015
2428
|
st->stack_tile_pos = 0;
|
|
3016
2429
|
st->stack_tile_have = 0;
|
|
3017
2430
|
}
|
|
3018
2431
|
if ( st->stack_parent_mask_ptrs ) {
|
|
3019
|
-
CAStack *stack = (CAStack *) st->src;
|
|
3020
2432
|
for ( int32_t kk = 0; kk < st->stack_n_parents; kk++ ) {
|
|
3021
|
-
|
|
2433
|
+
if ( st->stack_parent_mask_ptrs[kk] ) {
|
|
2434
|
+
xfree(st->stack_parent_mask_ptrs[kk]);
|
|
2435
|
+
}
|
|
3022
2436
|
}
|
|
3023
2437
|
xfree(st->stack_parent_mask_ptrs);
|
|
3024
2438
|
st->stack_parent_mask_ptrs = NULL;
|
|
3025
2439
|
}
|
|
3026
2440
|
st->stack_n_parents = 0;
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
2441
|
+
{
|
|
2442
|
+
/* The state is emptied before anything here can raise: syncing a
|
|
2443
|
+
write walk's root goes through the root's slots. A finish that
|
|
2444
|
+
raised is then finished already, and a second call returns at once
|
|
2445
|
+
instead of detaching src again. */
|
|
2446
|
+
CArray *root = st->root;
|
|
2447
|
+
CArray *src = st->src;
|
|
2448
|
+
CArray *own = (st->flags & CA_ITER_FLAG_OWNS_SRC) ? src : NULL;
|
|
2449
|
+
int write = (st->flags & CA_KERNEL_WRITE) != 0;
|
|
2450
|
+
int detach_src = root == NULL
|
|
2451
|
+
&& ( st->alias_mode == CA_ITER_ALIAS_CONTIG
|
|
2452
|
+
|| st->alias_mode == CA_ITER_ALIAS_STRIDED );
|
|
2453
|
+
int tag = 0;
|
|
2454
|
+
st->root = NULL;
|
|
2455
|
+
st->src = NULL;
|
|
2456
|
+
st->alias_ptr = NULL;
|
|
2457
|
+
st->flags &= ~CA_ITER_FLAG_OWNS_SRC;
|
|
2458
|
+
if ( root ) {
|
|
2459
|
+
/* A write walk wrote into the root's attach buffer, which is the root
|
|
2460
|
+
itself only for an entity: a view root (a transpose, a non-contig
|
|
2461
|
+
block, a view lending no memory) gave a buffer of its own. */
|
|
2462
|
+
if ( write ) {
|
|
2463
|
+
/* protect: frees the iterator's own copy before the raise passes on */
|
|
2464
|
+
rb_protect(ki_sync_detach_root, (VALUE) root, &tag);
|
|
2465
|
+
}
|
|
2466
|
+
else {
|
|
2467
|
+
ca_detach(root);
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
else if ( detach_src ) {
|
|
2471
|
+
/* alias path (CONTIG/STRIDED for CAStride family): we attached
|
|
2472
|
+
src directly at init, so detach it here. A SRC_ATTACH walk owns
|
|
2473
|
+
its own scratch (alias_mode = NONE, freed above) and attached
|
|
2474
|
+
nothing to detach. */
|
|
2475
|
+
ca_detach(src);
|
|
2476
|
+
}
|
|
2477
|
+
if ( own ) {
|
|
2478
|
+
if ( ca_is_object_type(own) ) {
|
|
2479
|
+
ca_gc_release_buffer(own->ptr);
|
|
2480
|
+
}
|
|
2481
|
+
ca_free(own);
|
|
2482
|
+
}
|
|
2483
|
+
if ( tag ) {
|
|
2484
|
+
rb_jump_tag(tag);
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
/* Finish a walk the way the block macros leave it: a slab of a write walk
|
|
2490
|
+
still handed out -- the body left with `break` -- is written back
|
|
2491
|
+
first. A raise from the write-back is passed on once the walk is
|
|
2492
|
+
finished. */
|
|
2493
|
+
void
|
|
2494
|
+
ca_iter_state_close (ca_iter_state *st)
|
|
2495
|
+
{
|
|
2496
|
+
int tag = 0;
|
|
2497
|
+
if ( st != NULL && st->src != NULL && st->write_dirty ) {
|
|
2498
|
+
rb_protect(ca_iter_finish_sync, (VALUE) st, &tag);
|
|
2499
|
+
}
|
|
2500
|
+
ca_iter_state_finish(st);
|
|
2501
|
+
if ( tag ) {
|
|
2502
|
+
rb_jump_tag(tag);
|
|
2503
|
+
}
|
|
3041
2504
|
}
|
|
3042
2505
|
|
|
3043
2506
|
/* ---- Raise-safe entry points ----------------------------------------
|
|
@@ -3053,10 +2516,9 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3053
2516
|
* and ca_iter_state_finish is idempotent (it clears st->src), so the
|
|
3054
2517
|
* caller's own finish after an ordinary walk stays correct.
|
|
3055
2518
|
*
|
|
3056
|
-
* Not covered: a kernel body that raises. It runs in the caller's
|
|
2519
|
+
* Not covered here: a kernel body that raises. It runs in the caller's
|
|
3057
2520
|
* frame, between two calls of these functions, where the engine has no
|
|
3058
|
-
* hold on it.
|
|
3059
|
-
* to watch.
|
|
2521
|
+
* hold on it. A body that can raise runs through ca_iter_ensure below.
|
|
3060
2522
|
*
|
|
3061
2523
|
* init runs protected always -- once per walk, next to a materialise.
|
|
3062
2524
|
* next_slab / sync_slab are called once per slab, in walks whose whole
|
|
@@ -3072,6 +2534,7 @@ ca_iter_state_finish (ca_iter_state *st)
|
|
|
3072
2534
|
header. */
|
|
3073
2535
|
#define CA_ITER_FLAG_WALK_MAY_RAISE 0x80000000u
|
|
3074
2536
|
|
|
2537
|
+
|
|
3075
2538
|
typedef struct {
|
|
3076
2539
|
ca_iter_state *st;
|
|
3077
2540
|
CArray *src;
|
|
@@ -3082,15 +2545,31 @@ typedef struct {
|
|
|
3082
2545
|
char **out_ptr;
|
|
3083
2546
|
boolean8_t **out_mask;
|
|
3084
2547
|
ca_size_t *out_n;
|
|
3085
|
-
ca_size_t *out_stride_bytes;
|
|
3086
2548
|
int rc;
|
|
3087
2549
|
} ca_iter_call_t;
|
|
3088
2550
|
|
|
3089
|
-
/* Whether next_slab / sync_slab on this walk need the protection.
|
|
2551
|
+
/* Whether next_slab / sync_slab on this walk need the protection.
|
|
2552
|
+
|
|
2553
|
+
Shortcut PROTECT_ONLY_MAY_RAISE.
|
|
2554
|
+
Layer the exits (promise 4): which walks run next_slab /
|
|
2555
|
+
sync_slab under rb_protect.
|
|
2556
|
+
Speeds up walks that cannot raise there: they skip rb_protect.
|
|
2557
|
+
Why about 1.5x on a per-axis sum over 200k short fibers.
|
|
2558
|
+
Enters the route is not PER_FIBER_FUSED and the walk does not
|
|
2559
|
+
write back out of scratch.
|
|
2560
|
+
Promises 4 (a walk left out cannot raise between init and finish).
|
|
2561
|
+
Switch ca_fast_paths_enabled(): off, every walk is protected.
|
|
2562
|
+
Checked by test_iter_raise_cleanup.rb; the raise rows of
|
|
2563
|
+
test_iter_delivery_detectors.rb. */
|
|
3090
2564
|
static void
|
|
3091
2565
|
ca_iter_mark_walk_may_raise (ca_iter_state *st)
|
|
3092
2566
|
{
|
|
3093
2567
|
if ( st == NULL ) return;
|
|
2568
|
+
if ( ! ca_fast_paths_enabled() ) {
|
|
2569
|
+
/* the general path protects every walk */
|
|
2570
|
+
st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
|
|
2571
|
+
return;
|
|
2572
|
+
}
|
|
3094
2573
|
if ( st->alias_mode == CA_ITER_ALIAS_PER_FIBER_FUSED ) {
|
|
3095
2574
|
/* gathers each fiber through the source's xfer_stride slot */
|
|
3096
2575
|
st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
|
|
@@ -3099,19 +2578,35 @@ ca_iter_mark_walk_may_raise (ca_iter_state *st)
|
|
|
3099
2578
|
&& (st->scratch_ptr != NULL
|
|
3100
2579
|
|| st->src_kind == CA_ITER_SRC_ATTACH) ) {
|
|
3101
2580
|
/* writes back out of iterator-owned scratch, through the source's
|
|
3102
|
-
|
|
2581
|
+
transfer (ki_scratch_push_changed: ca_xfer_addrs / ca_xfer_all PUT)
|
|
2582
|
+
or the descriptor scatter. A write into aliased memory needs no
|
|
3103
2583
|
push-back at all: sync_slab returns at once. */
|
|
3104
2584
|
st->flags |= CA_ITER_FLAG_WALK_MAY_RAISE;
|
|
3105
2585
|
}
|
|
3106
2586
|
}
|
|
3107
2587
|
|
|
2588
|
+
/* When body raises, finish st -- and the state it is paired with (see
|
|
2589
|
+
ca_iter_state_init_l2_paired), which is open in the same frame and
|
|
2590
|
+
would otherwise be jumped over. */
|
|
2591
|
+
static VALUE ca_iter_ensure_finish_one (VALUE arg);
|
|
2592
|
+
|
|
3108
2593
|
static void
|
|
3109
2594
|
ca_iter_protect (ca_iter_state *st, VALUE (*body)(VALUE), VALUE arg)
|
|
3110
2595
|
{
|
|
3111
2596
|
int tag = 0;
|
|
3112
2597
|
rb_protect(body, arg, &tag);
|
|
3113
2598
|
if ( tag ) {
|
|
3114
|
-
|
|
2599
|
+
ca_iter_state *pair = (ca_iter_state *) st->pair;
|
|
2600
|
+
volatile VALUE error = rb_errinfo();
|
|
2601
|
+
int ignored = 0;
|
|
2602
|
+
/* protect: finishes the pair even when finishing st raises; the
|
|
2603
|
+
body's exception is the one passed on */
|
|
2604
|
+
rb_protect(ca_iter_ensure_finish_one, (VALUE) st, &ignored);
|
|
2605
|
+
if ( pair != NULL ) {
|
|
2606
|
+
/* protect: as above */
|
|
2607
|
+
rb_protect(ca_iter_ensure_finish_one, (VALUE) pair, &ignored);
|
|
2608
|
+
}
|
|
2609
|
+
rb_set_errinfo(error);
|
|
3115
2610
|
rb_jump_tag(tag);
|
|
3116
2611
|
}
|
|
3117
2612
|
}
|
|
@@ -3143,15 +2638,6 @@ ca_iter_call_next_slab (VALUE arg)
|
|
|
3143
2638
|
return Qnil;
|
|
3144
2639
|
}
|
|
3145
2640
|
|
|
3146
|
-
static VALUE
|
|
3147
|
-
ca_iter_call_next_slab_strided (VALUE arg)
|
|
3148
|
-
{
|
|
3149
|
-
ca_iter_call_t *c = (ca_iter_call_t *) arg;
|
|
3150
|
-
c->rc = ca_iter_state_next_slab_strided_unprotected(c->st, c->out_ptr,
|
|
3151
|
-
c->out_mask, c->out_n,
|
|
3152
|
-
c->out_stride_bytes);
|
|
3153
|
-
return Qnil;
|
|
3154
|
-
}
|
|
3155
2641
|
|
|
3156
2642
|
static VALUE
|
|
3157
2643
|
ca_iter_call_next_slab_axes (VALUE arg)
|
|
@@ -3181,6 +2667,9 @@ ca_iter_state_init_l1 (ca_iter_state *st,
|
|
|
3181
2667
|
ca_iter_call_t c;
|
|
3182
2668
|
c.st = st; c.src = src; c.policy = policy;
|
|
3183
2669
|
c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
|
|
2670
|
+
if ( st != NULL ) {
|
|
2671
|
+
st->pair = NULL; /* read by ca_iter_protect if the init raises */
|
|
2672
|
+
}
|
|
3184
2673
|
ca_iter_protect(st, ca_iter_call_init_l1, (VALUE) &c);
|
|
3185
2674
|
if ( c.rc == CA_ITER_OK ) ca_iter_mark_walk_may_raise(st);
|
|
3186
2675
|
return c.rc;
|
|
@@ -3197,11 +2686,32 @@ ca_iter_state_init_l2 (ca_iter_state *st,
|
|
|
3197
2686
|
ca_iter_call_t c;
|
|
3198
2687
|
c.st = st; c.src = src; c.policy = policy;
|
|
3199
2688
|
c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
|
|
2689
|
+
if ( st != NULL ) {
|
|
2690
|
+
st->pair = NULL; /* read by ca_iter_protect if the init raises */
|
|
2691
|
+
}
|
|
3200
2692
|
ca_iter_protect(st, ca_iter_call_init_l2, (VALUE) &c);
|
|
3201
2693
|
if ( c.rc == CA_ITER_OK ) ca_iter_mark_walk_may_raise(st);
|
|
3202
2694
|
return c.rc;
|
|
3203
2695
|
}
|
|
3204
2696
|
|
|
2697
|
+
/* A slab handed out by a write walk is dirty until sync_slab writes it
|
|
2698
|
+
back; finish writes back one the caller left with `break`. */
|
|
2699
|
+
static int
|
|
2700
|
+
ca_iter_mark_dirty (ca_iter_state *st, int rc)
|
|
2701
|
+
{
|
|
2702
|
+
if ( rc && st != NULL && (st->flags & CA_KERNEL_WRITE) ) {
|
|
2703
|
+
st->write_dirty = 1;
|
|
2704
|
+
}
|
|
2705
|
+
return rc;
|
|
2706
|
+
}
|
|
2707
|
+
|
|
2708
|
+
static VALUE
|
|
2709
|
+
ca_iter_finish_sync (VALUE arg)
|
|
2710
|
+
{
|
|
2711
|
+
ca_iter_state_sync_slab_unprotected((ca_iter_state *) arg);
|
|
2712
|
+
return Qnil;
|
|
2713
|
+
}
|
|
2714
|
+
|
|
3205
2715
|
int
|
|
3206
2716
|
ca_iter_state_next_slab (ca_iter_state *st,
|
|
3207
2717
|
char **out_ptr,
|
|
@@ -3210,31 +2720,15 @@ ca_iter_state_next_slab (ca_iter_state *st,
|
|
|
3210
2720
|
{
|
|
3211
2721
|
ca_iter_call_t c;
|
|
3212
2722
|
if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
|
|
3213
|
-
return
|
|
2723
|
+
return ca_iter_mark_dirty(st,
|
|
2724
|
+
ca_iter_state_next_slab_unprotected(st, out_ptr, out_mask, out_n));
|
|
3214
2725
|
}
|
|
3215
2726
|
c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.out_n = out_n;
|
|
3216
2727
|
c.rc = 0;
|
|
3217
2728
|
ca_iter_protect(st, ca_iter_call_next_slab, (VALUE) &c);
|
|
3218
|
-
return c.rc;
|
|
2729
|
+
return ca_iter_mark_dirty(st, c.rc);
|
|
3219
2730
|
}
|
|
3220
2731
|
|
|
3221
|
-
int
|
|
3222
|
-
ca_iter_state_next_slab_strided (ca_iter_state *st,
|
|
3223
|
-
char **out_ptr,
|
|
3224
|
-
boolean8_t **out_mask,
|
|
3225
|
-
ca_size_t *out_n,
|
|
3226
|
-
ca_size_t *out_stride_bytes)
|
|
3227
|
-
{
|
|
3228
|
-
ca_iter_call_t c;
|
|
3229
|
-
if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
|
|
3230
|
-
return ca_iter_state_next_slab_strided_unprotected(st, out_ptr, out_mask,
|
|
3231
|
-
out_n, out_stride_bytes);
|
|
3232
|
-
}
|
|
3233
|
-
c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.out_n = out_n;
|
|
3234
|
-
c.out_stride_bytes = out_stride_bytes; c.rc = 0;
|
|
3235
|
-
ca_iter_protect(st, ca_iter_call_next_slab_strided, (VALUE) &c);
|
|
3236
|
-
return c.rc;
|
|
3237
|
-
}
|
|
3238
2732
|
|
|
3239
2733
|
int
|
|
3240
2734
|
ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
@@ -3243,11 +2737,12 @@ ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
|
3243
2737
|
{
|
|
3244
2738
|
ca_iter_call_t c;
|
|
3245
2739
|
if ( st == NULL || !(st->flags & CA_ITER_FLAG_WALK_MAY_RAISE) ) {
|
|
3246
|
-
return
|
|
2740
|
+
return ca_iter_mark_dirty(st,
|
|
2741
|
+
ca_iter_state_next_slab_axes_unprotected(st, out_ptr, out_mask));
|
|
3247
2742
|
}
|
|
3248
2743
|
c.st = st; c.out_ptr = out_ptr; c.out_mask = out_mask; c.rc = 0;
|
|
3249
2744
|
ca_iter_protect(st, ca_iter_call_next_slab_axes, (VALUE) &c);
|
|
3250
|
-
return c.rc;
|
|
2745
|
+
return ca_iter_mark_dirty(st, c.rc);
|
|
3251
2746
|
}
|
|
3252
2747
|
|
|
3253
2748
|
void
|
|
@@ -3262,6 +2757,168 @@ ca_iter_state_sync_slab (ca_iter_state *st)
|
|
|
3262
2757
|
ca_iter_protect(st, ca_iter_call_sync_slab, (VALUE) &c);
|
|
3263
2758
|
}
|
|
3264
2759
|
|
|
2760
|
+
typedef struct {
|
|
2761
|
+
ca_iter_state *st;
|
|
2762
|
+
CArray *src;
|
|
2763
|
+
CArray *copy;
|
|
2764
|
+
} ca_iter_reopen_t;
|
|
2765
|
+
|
|
2766
|
+
static VALUE
|
|
2767
|
+
ca_iter_reopen_copy (VALUE arg)
|
|
2768
|
+
{
|
|
2769
|
+
ca_iter_reopen_t *r = (ca_iter_reopen_t *) arg;
|
|
2770
|
+
r->copy = ca_copy(r->src);
|
|
2771
|
+
return Qnil;
|
|
2772
|
+
}
|
|
2773
|
+
|
|
2774
|
+
/* A read walk whose source lives in the root a paired write walk writes
|
|
2775
|
+
into would read cells the kernel has already written: an aliased fiber
|
|
2776
|
+
is the written memory itself, and a gathered one is read from the root
|
|
2777
|
+
after earlier fibers were written back. Such a read walk is reopened
|
|
2778
|
+
on a copy of its source, taken before anything is written and owned by
|
|
2779
|
+
the walk. Returns the init's rc; a raise finishes `first` first. */
|
|
2780
|
+
static int
|
|
2781
|
+
ca_iter_reopen_on_copy (ca_iter_state *first)
|
|
2782
|
+
{
|
|
2783
|
+
ca_iter_reopen_t r;
|
|
2784
|
+
ca_iter_call_t c;
|
|
2785
|
+
int8_t axes[CA_RANK_MAX];
|
|
2786
|
+
int8_t naxes = first->naxes;
|
|
2787
|
+
uint32_t flags = first->flags & ~(CA_ITER_FLAG_WALK_MAY_RAISE
|
|
2788
|
+
| CA_ITER_FLAG_OWNS_SRC);
|
|
2789
|
+
ca_slab_policy_t policy = first->policy;
|
|
2790
|
+
int tag = 0;
|
|
2791
|
+
|
|
2792
|
+
memcpy(axes, first->slab_axes_buf, (size_t) naxes);
|
|
2793
|
+
r.st = first; r.src = first->src; r.copy = NULL;
|
|
2794
|
+
rb_protect(ca_iter_reopen_copy, (VALUE) &r, &tag);
|
|
2795
|
+
if ( tag ) {
|
|
2796
|
+
ca_iter_state_finish(first);
|
|
2797
|
+
rb_jump_tag(tag);
|
|
2798
|
+
}
|
|
2799
|
+
if ( ca_is_object_type(r.copy) ) {
|
|
2800
|
+
/* the walk overwrites the cells that held these VALUEs */
|
|
2801
|
+
ca_gc_hold_buffer(r.copy->ptr, r.copy->elements);
|
|
2802
|
+
}
|
|
2803
|
+
ca_iter_state_finish(first);
|
|
2804
|
+
|
|
2805
|
+
c.st = first; c.src = r.copy; c.policy = policy;
|
|
2806
|
+
c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
|
|
2807
|
+
rb_protect(ca_iter_call_init_l2, (VALUE) &c, &tag);
|
|
2808
|
+
if ( tag || c.rc != CA_ITER_OK ) {
|
|
2809
|
+
ca_iter_state_finish(first);
|
|
2810
|
+
if ( ca_is_object_type(r.copy) ) {
|
|
2811
|
+
ca_gc_release_buffer(r.copy->ptr);
|
|
2812
|
+
}
|
|
2813
|
+
ca_free(r.copy);
|
|
2814
|
+
if ( tag ) {
|
|
2815
|
+
rb_jump_tag(tag);
|
|
2816
|
+
}
|
|
2817
|
+
return c.rc;
|
|
2818
|
+
}
|
|
2819
|
+
first->flags |= CA_ITER_FLAG_OWNS_SRC;
|
|
2820
|
+
ca_iter_mark_walk_may_raise(first);
|
|
2821
|
+
return CA_ITER_OK;
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
int
|
|
2825
|
+
ca_iter_state_init_l2_paired (ca_iter_state *st,
|
|
2826
|
+
ca_iter_state *first,
|
|
2827
|
+
struct _CArray *src,
|
|
2828
|
+
ca_slab_policy_t policy,
|
|
2829
|
+
int8_t *axes,
|
|
2830
|
+
int8_t naxes,
|
|
2831
|
+
uint32_t flags)
|
|
2832
|
+
{
|
|
2833
|
+
ca_iter_call_t c;
|
|
2834
|
+
int tag = 0;
|
|
2835
|
+
if ( (flags & CA_KERNEL_WRITE) && first->src != NULL &&
|
|
2836
|
+
! (first->flags & CA_KERNEL_WRITE) &&
|
|
2837
|
+
policy == CA_SLAB_AXES && first->policy == CA_SLAB_AXES &&
|
|
2838
|
+
ca_shares_root(first->src, src) ) {
|
|
2839
|
+
int rc = ca_iter_reopen_on_copy(first);
|
|
2840
|
+
if ( rc != CA_ITER_OK ) {
|
|
2841
|
+
return rc;
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
c.st = st; c.src = src; c.policy = policy;
|
|
2845
|
+
c.axes = axes; c.naxes = naxes; c.flags = flags; c.rc = CA_ITER_OK;
|
|
2846
|
+
rb_protect(ca_iter_call_init_l2, (VALUE) &c, &tag);
|
|
2847
|
+
if ( tag ) {
|
|
2848
|
+
ca_iter_state_finish(st);
|
|
2849
|
+
ca_iter_state_finish(first);
|
|
2850
|
+
rb_jump_tag(tag);
|
|
2851
|
+
}
|
|
2852
|
+
if ( c.rc != CA_ITER_OK ) {
|
|
2853
|
+
ca_iter_state_finish(first);
|
|
2854
|
+
return c.rc;
|
|
2855
|
+
}
|
|
2856
|
+
ca_iter_mark_walk_may_raise(st);
|
|
2857
|
+
st->pair = first;
|
|
2858
|
+
first->pair = st;
|
|
2859
|
+
return c.rc;
|
|
2860
|
+
}
|
|
2861
|
+
|
|
2862
|
+
typedef struct {
|
|
2863
|
+
int32_t n;
|
|
2864
|
+
ca_iter_state *states;
|
|
2865
|
+
} ca_iter_ensure_t;
|
|
2866
|
+
|
|
2867
|
+
static VALUE
|
|
2868
|
+
ca_iter_ensure_finish_one (VALUE arg)
|
|
2869
|
+
{
|
|
2870
|
+
ca_iter_state_finish((ca_iter_state *) arg);
|
|
2871
|
+
return Qnil;
|
|
2872
|
+
}
|
|
2873
|
+
|
|
2874
|
+
/* Finishes the walks in reverse order of their slots, as nested windows
|
|
2875
|
+
close. A finish can raise: on a WRITE walk whose root is a view, it
|
|
2876
|
+
syncs the root (ca_sync_detach) through the view's slots. Each finish
|
|
2877
|
+
is therefore run under rb_protect, so a raise from one does not leave
|
|
2878
|
+
the states before it unfinished; the first raise is re-raised once
|
|
2879
|
+
every state is closed. */
|
|
2880
|
+
static VALUE
|
|
2881
|
+
ca_iter_ensure_close (VALUE arg)
|
|
2882
|
+
{
|
|
2883
|
+
ca_iter_ensure_t *e = (ca_iter_ensure_t *) arg;
|
|
2884
|
+
int first = 0;
|
|
2885
|
+
volatile VALUE first_err = Qnil;
|
|
2886
|
+
for ( int32_t i = e->n - 1; i >= 0; i-- ) {
|
|
2887
|
+
int status = 0;
|
|
2888
|
+
rb_protect(ca_iter_ensure_finish_one, (VALUE) &e->states[i], &status);
|
|
2889
|
+
if ( status ) {
|
|
2890
|
+
if ( ! first ) {
|
|
2891
|
+
first = status;
|
|
2892
|
+
first_err = rb_errinfo();
|
|
2893
|
+
}
|
|
2894
|
+
rb_set_errinfo(Qnil);
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
if ( first ) {
|
|
2898
|
+
rb_set_errinfo(first_err);
|
|
2899
|
+
rb_jump_tag(first);
|
|
2900
|
+
}
|
|
2901
|
+
return Qnil;
|
|
2902
|
+
}
|
|
2903
|
+
|
|
2904
|
+
VALUE
|
|
2905
|
+
ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
2906
|
+
VALUE (*body)(VALUE), VALUE arg)
|
|
2907
|
+
{
|
|
2908
|
+
ca_iter_ensure_t e;
|
|
2909
|
+
if ( n < 0 ) {
|
|
2910
|
+
rb_raise(rb_eArgError, "ca_iter_ensure: negative state count %d", (int) n);
|
|
2911
|
+
}
|
|
2912
|
+
/* Unopened: finish returns at once on a state whose src is NULL, and
|
|
2913
|
+
init clears the whole state before it takes anything. */
|
|
2914
|
+
for ( int32_t i = 0; i < n; i++ ) {
|
|
2915
|
+
states[i].src = NULL;
|
|
2916
|
+
}
|
|
2917
|
+
e.n = n;
|
|
2918
|
+
e.states = states;
|
|
2919
|
+
return rb_ensure(body, arg, ca_iter_ensure_close, (VALUE) &e);
|
|
2920
|
+
}
|
|
2921
|
+
|
|
3265
2922
|
#ifdef CARRAY_DEV_BUILD
|
|
3266
2923
|
/* ============================================================
|
|
3267
2924
|
* smoke surface (dev-only, stripped in release)
|
|
@@ -3270,10 +2927,27 @@ ca_iter_state_sync_slab (ca_iter_state *st)
|
|
|
3270
2927
|
* or `CARRAY_DEV=1 rake build_ext`). These helpers expose internal
|
|
3271
2928
|
* engine state to Ruby for spec_ai regression pins. Do not consume
|
|
3272
2929
|
* from user code.
|
|
3273
|
-
*
|
|
3274
|
-
* See devel/PROPOSAL_SMOKE_DEV_BUILD_GATE.md
|
|
3275
2930
|
* ============================================================ */
|
|
3276
2931
|
|
|
2932
|
+
/* The innermost-axis walk the L2 smokes run: one strided run per
|
|
2933
|
+
innermost row, handed out as (pointer, cell count, byte stride). */
|
|
2934
|
+
static int
|
|
2935
|
+
t1_smoke_open_rows (ca_iter_state *st, CArray *src, uint32_t flags)
|
|
2936
|
+
{
|
|
2937
|
+
int8_t ax = (int8_t) (src->ndim - 1);
|
|
2938
|
+
return ca_iter_state_init_l2(st, src, CA_SLAB_AXES, &ax, 1, flags);
|
|
2939
|
+
}
|
|
2940
|
+
|
|
2941
|
+
static int
|
|
2942
|
+
t1_smoke_next_row (ca_iter_state *st, char **p, ca_size_t *n,
|
|
2943
|
+
ca_size_t *stride)
|
|
2944
|
+
{
|
|
2945
|
+
if ( ! ca_iter_state_next_slab_axes(st, p, NULL) ) return 0;
|
|
2946
|
+
*n = st->slab_dims[0];
|
|
2947
|
+
*stride = st->slab_strides[0];
|
|
2948
|
+
return 1;
|
|
2949
|
+
}
|
|
2950
|
+
|
|
3277
2951
|
/* ---- Ruby smoke surface --------------------------------------------- */
|
|
3278
2952
|
|
|
3279
2953
|
/* L1 smoke (steps 1+2): CArray.t1_smoke(ca) -> Hash with
|
|
@@ -3318,7 +2992,7 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
|
|
|
3318
2992
|
return result;
|
|
3319
2993
|
}
|
|
3320
2994
|
|
|
3321
|
-
/* L2 smoke
|
|
2995
|
+
/* L2 smoke: CArray.t1_smoke_strided(ca) -> Hash with
|
|
3322
2996
|
rc => Integer
|
|
3323
2997
|
slabs => Integer (= total_slabs)
|
|
3324
2998
|
total_elems => Integer
|
|
@@ -3327,8 +3001,6 @@ rb_t1_smoke (VALUE klass, VALUE vsrc)
|
|
|
3327
3001
|
reported (ptr, n, stride_bytes) tuples, in
|
|
3328
3002
|
iteration order; should equal view.to_ca.dump_binary
|
|
3329
3003
|
strides => Array<Integer> — stride_bytes per yielded slab
|
|
3330
|
-
(constant across yields under WHOLE policy in
|
|
3331
|
-
step 3; surfaced for inspection)
|
|
3332
3004
|
On error rc != OK, the walk fields are 0 / empty. */
|
|
3333
3005
|
static VALUE
|
|
3334
3006
|
rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
|
|
@@ -3347,9 +3019,9 @@ rb_t1_smoke_strided (VALUE klass, VALUE vsrc)
|
|
|
3347
3019
|
data = rb_str_new(0, 0);
|
|
3348
3020
|
strides_arr = rb_ary_new();
|
|
3349
3021
|
|
|
3350
|
-
rc =
|
|
3022
|
+
rc = t1_smoke_open_rows(&st, src, 0);
|
|
3351
3023
|
if ( rc == CA_ITER_OK ) {
|
|
3352
|
-
while (
|
|
3024
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
|
|
3353
3025
|
if ( p != NULL && n > 0 ) {
|
|
3354
3026
|
ca_size_t i;
|
|
3355
3027
|
for ( i = 0; i < n; i++ ) {
|
|
@@ -3396,15 +3068,15 @@ rb_t1_smoke_sum_strided_f64 (VALUE klass, VALUE vsrc)
|
|
|
3396
3068
|
"t1_smoke_sum_strided_f64 expects a float64 source");
|
|
3397
3069
|
}
|
|
3398
3070
|
|
|
3399
|
-
rc =
|
|
3071
|
+
rc = t1_smoke_open_rows(&st, src, 0);
|
|
3400
3072
|
if ( rc != CA_ITER_OK ) {
|
|
3401
3073
|
rb_raise(rb_eRuntimeError, "ca_iter_state_init L2 failed (rc=%d)", rc);
|
|
3402
3074
|
}
|
|
3403
3075
|
|
|
3404
|
-
while (
|
|
3405
|
-
/*
|
|
3406
|
-
|
|
3407
|
-
|
|
3076
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
|
|
3077
|
+
/* CA_L2_FOR_EACH takes a contiguous loop when stride ==
|
|
3078
|
+
sizeof(double), which the compiler can vectorise, and a strided
|
|
3079
|
+
loop otherwise. */
|
|
3408
3080
|
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3409
3081
|
acc += *dp;
|
|
3410
3082
|
});
|
|
@@ -3445,9 +3117,8 @@ rb_t1_smoke_write_fill_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
|
3445
3117
|
return INT2NUM(CA_ITER_OK);
|
|
3446
3118
|
}
|
|
3447
3119
|
|
|
3448
|
-
/* WRITE smoke: in-place fill via L2
|
|
3449
|
-
|
|
3450
|
-
already in t1_smoke_write_fill_f64). Kernel walks the strided
|
|
3120
|
+
/* WRITE smoke: in-place fill via an L2 walk (the L1 form is
|
|
3121
|
+
t1_smoke_write_fill_f64). Kernel walks the strided
|
|
3451
3122
|
slab and writes val at every position; sync_slab scatters back. */
|
|
3452
3123
|
static VALUE
|
|
3453
3124
|
rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
@@ -3464,14 +3135,12 @@ rb_t1_smoke_write_fill_strided_f64 (VALUE klass, VALUE vsrc, VALUE vval)
|
|
|
3464
3135
|
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3465
3136
|
}
|
|
3466
3137
|
|
|
3467
|
-
rc =
|
|
3468
|
-
CA_KERNEL_WRITE);
|
|
3138
|
+
rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
|
|
3469
3139
|
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3470
3140
|
|
|
3471
|
-
while (
|
|
3472
|
-
/*
|
|
3473
|
-
|
|
3474
|
-
hit +18-22% in step 5.5 — macro removes that overhead). */
|
|
3141
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
|
|
3142
|
+
/* Contiguous loop when stride == sizeof(double) (a gathered
|
|
3143
|
+
scratch), strided otherwise. */
|
|
3475
3144
|
CA_L2_FOR_EACH(double, p, n, stride, dp, {
|
|
3476
3145
|
*dp = v;
|
|
3477
3146
|
});
|
|
@@ -3539,13 +3208,11 @@ cmp_double (const void *a, const void *b)
|
|
|
3539
3208
|
|
|
3540
3209
|
/* WRITE smoke: per-row sort via L2 strided dispatch. For a 2D
|
|
3541
3210
|
float64 src of shape [m, n], sort each of the m rows in ascending
|
|
3542
|
-
order.
|
|
3211
|
+
order. Walks the innermost axis so each row is a strided slab; the
|
|
3543
3212
|
kernel materialises into a tight contig scratch, qsorts, then
|
|
3544
3213
|
writes back via the same stride. This exercises L2 WRITE
|
|
3545
3214
|
mechanics (multi-slab walk, per-slab fill of strided cells)
|
|
3546
|
-
without
|
|
3547
|
-
Pattern H specialised op, out of step 4 scope (reviewer advice #3).
|
|
3548
|
-
*/
|
|
3215
|
+
without a strided qsort of its own. */
|
|
3549
3216
|
static VALUE
|
|
3550
3217
|
rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
3551
3218
|
{
|
|
@@ -3561,17 +3228,16 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
|
3561
3228
|
rb_raise(rb_eTypeError, "expects a float64 source");
|
|
3562
3229
|
}
|
|
3563
3230
|
|
|
3564
|
-
rc =
|
|
3565
|
-
CA_KERNEL_WRITE);
|
|
3231
|
+
rc = t1_smoke_open_rows(&st, src, CA_KERNEL_WRITE);
|
|
3566
3232
|
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3567
3233
|
|
|
3568
|
-
while (
|
|
3234
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride_bytes) ) {
|
|
3569
3235
|
if ( n > (ca_size_t) (sizeof(scratch) / sizeof(double)) ) {
|
|
3570
3236
|
ca_iter_state_finish(&st);
|
|
3571
3237
|
rb_raise(rb_eRuntimeError, "row too large for smoke scratch");
|
|
3572
3238
|
}
|
|
3573
|
-
/* strided -> contig scratch
|
|
3574
|
-
stride == bytes
|
|
3239
|
+
/* strided -> contig scratch (CA_L2_FOR_EACH takes the contiguous
|
|
3240
|
+
loop when stride == bytes). */
|
|
3575
3241
|
ca_size_t _sc_i = 0;
|
|
3576
3242
|
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3577
3243
|
scratch[_sc_i++] = *dp;
|
|
@@ -3579,7 +3245,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
|
3579
3245
|
/* in-place qsort on the contig scratch */
|
|
3580
3246
|
qsort(scratch, n, sizeof(double), cmp_double);
|
|
3581
3247
|
/* contig scratch -> strided (writes back to parent via alias_ptr,
|
|
3582
|
-
|
|
3248
|
+
a direct write). */
|
|
3583
3249
|
ca_size_t _wb_i = 0;
|
|
3584
3250
|
CA_L2_FOR_EACH(double, p, n, stride_bytes, dp, {
|
|
3585
3251
|
*dp = scratch[_wb_i++];
|
|
@@ -3590,7 +3256,7 @@ rb_t1_smoke_sort_row_f64 (VALUE klass, VALUE vsrc)
|
|
|
3590
3256
|
return INT2NUM(CA_ITER_OK);
|
|
3591
3257
|
}
|
|
3592
3258
|
|
|
3593
|
-
/*
|
|
3259
|
+
/* Mask smoke: L1 walk that exposes BOTH the value slab and the
|
|
3594
3260
|
mask slab. Used to pin "mask is delivered to kernel" semantics.
|
|
3595
3261
|
Returns rc + mask_seen (Boolean: was out_mask non-NULL at first
|
|
3596
3262
|
yield) + mask_bytes (String: concatenated mask bytes). */
|
|
@@ -3666,9 +3332,9 @@ rb_t1_smoke_attach (VALUE klass, VALUE vsrc)
|
|
|
3666
3332
|
return SIZE2NUM(total);
|
|
3667
3333
|
}
|
|
3668
3334
|
|
|
3669
|
-
/* L2 bench-grade smoke
|
|
3670
|
-
but
|
|
3671
|
-
exercised through the L2 path. */
|
|
3335
|
+
/* L2 bench-grade smoke. Same shape as t1_smoke_attach
|
|
3336
|
+
but walks the innermost axis through init_l2 so descriptor sources
|
|
3337
|
+
can be exercised through the L2 path. */
|
|
3672
3338
|
static VALUE
|
|
3673
3339
|
rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
3674
3340
|
{
|
|
@@ -3681,10 +3347,10 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
|
3681
3347
|
volatile char sink = 0;
|
|
3682
3348
|
|
|
3683
3349
|
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
3684
|
-
rc =
|
|
3350
|
+
rc = t1_smoke_open_rows(&st, src, 0);
|
|
3685
3351
|
if ( rc != CA_ITER_OK ) return INT2NUM(rc);
|
|
3686
3352
|
|
|
3687
|
-
while (
|
|
3353
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
|
|
3688
3354
|
if ( p != NULL && n > 0 ) sink ^= p[0];
|
|
3689
3355
|
total += n;
|
|
3690
3356
|
}
|
|
@@ -3693,7 +3359,7 @@ rb_t1_smoke_attach_strided (VALUE klass, VALUE vsrc)
|
|
|
3693
3359
|
return SIZE2NUM(total);
|
|
3694
3360
|
}
|
|
3695
3361
|
|
|
3696
|
-
/*
|
|
3362
|
+
/* Flags smoke: minimal smoke that exposes the `flags` argument to Ruby
|
|
3697
3363
|
tests. Just runs init_l1 with the requested flags and returns
|
|
3698
3364
|
the rc — used to verify NO_MASK enforcement (and any future
|
|
3699
3365
|
flag-gated rejection paths) without needing a full kernel walk. */
|
|
@@ -3703,6 +3369,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
|
|
|
3703
3369
|
CArray *src;
|
|
3704
3370
|
ca_iter_state st;
|
|
3705
3371
|
int rc;
|
|
3372
|
+
/* fits: a development smoke argument, not a cell */
|
|
3706
3373
|
uint32_t flags = (uint32_t) NUM2UINT(vflags);
|
|
3707
3374
|
|
|
3708
3375
|
TypedData_Get_Struct(vsrc, CArray, &carray_data_type, src);
|
|
@@ -3711,7 +3378,7 @@ rb_t1_smoke_init_rc (VALUE klass, VALUE vsrc, VALUE vflags)
|
|
|
3711
3378
|
return INT2NUM(rc);
|
|
3712
3379
|
}
|
|
3713
3380
|
|
|
3714
|
-
/* ----
|
|
3381
|
+
/* ---- CA_SLAB_AXES smoke ----------------------------------------------
|
|
3715
3382
|
sum a float64 source over user-specified slab axes via init_l2 +
|
|
3716
3383
|
next_slab_axes. K-D walk over slab dims using slab_strides for data
|
|
3717
3384
|
and slab_mask_strides for mask (so non-contig CAStride sources work
|
|
@@ -3741,10 +3408,11 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
3741
3408
|
rb_raise(rb_eTypeError, "t1_smoke_sum_axes_f64 expects a float64 source");
|
|
3742
3409
|
}
|
|
3743
3410
|
naxes = (int8_t) (argc - 1);
|
|
3744
|
-
if ( naxes > CA_RANK_MAX ) {
|
|
3411
|
+
if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
|
|
3745
3412
|
rb_raise(rb_eArgError, "too many axes");
|
|
3746
3413
|
}
|
|
3747
3414
|
for ( i = 0; i < naxes; i++ ) {
|
|
3415
|
+
/* fits: an axis number, checked by the engine's init */
|
|
3748
3416
|
slab_axes[i] = (int8_t) NUM2INT(argv[i + 1]);
|
|
3749
3417
|
}
|
|
3750
3418
|
|
|
@@ -3780,7 +3448,7 @@ rb_t1_smoke_sum_axes_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
3780
3448
|
return DBL2NUM(acc);
|
|
3781
3449
|
}
|
|
3782
3450
|
|
|
3783
|
-
/* ----
|
|
3451
|
+
/* ---- helper smoke: rb_ca_parse_reduce_axes ---------------------------
|
|
3784
3452
|
Calls the helper and returns the parsed axes as a Ruby Array of
|
|
3785
3453
|
Integer. Lets tests inspect parsing behaviour directly.
|
|
3786
3454
|
Signature: CArray.t1_test_parse_reduce_axes(ca, *args) */
|
|
@@ -3806,7 +3474,7 @@ rb_t1_test_parse_reduce_axes (int argc, VALUE *argv, VALUE klass)
|
|
|
3806
3474
|
return arr;
|
|
3807
3475
|
}
|
|
3808
3476
|
|
|
3809
|
-
/* ----
|
|
3477
|
+
/* ---- helper smoke: rb_ca_parse_reduce_axes_kw ------------------------
|
|
3810
3478
|
Calls the kwarg helper and returns the parsed axes as a Ruby Array.
|
|
3811
3479
|
Lets tests inspect kwarg-form parsing (Qnil / Integer / Array) +
|
|
3812
3480
|
validation parity with the variadic entry.
|
|
@@ -3834,7 +3502,7 @@ rb_test_parse_reduce_axes_kw (int argc, VALUE *argv, VALUE klass)
|
|
|
3834
3502
|
return arr;
|
|
3835
3503
|
}
|
|
3836
3504
|
|
|
3837
|
-
/* ----
|
|
3505
|
+
/* ---- helper smoke: rb_ca_new_reduced ---------------------------------
|
|
3838
3506
|
Exposes rb_ca_new_reduced (carray_core.c) for unit testing. Returns
|
|
3839
3507
|
the newly allocated output CArray (= tests verify shape + data_type).
|
|
3840
3508
|
Signature: CArray.t1_test_new_reduced(ca, data_type, axis1, axis2, ...) */
|
|
@@ -3849,21 +3517,21 @@ rb_t1_test_new_reduced (int argc, VALUE *argv, VALUE klass)
|
|
|
3849
3517
|
rb_raise(rb_eArgError,
|
|
3850
3518
|
"t1_test_new_reduced expects (ca, data_type, axis...) with >= 1 axis");
|
|
3851
3519
|
}
|
|
3852
|
-
/*
|
|
3853
|
-
|
|
3854
|
-
and legacy Integer-code callers both work. */
|
|
3520
|
+
/* Accept Symbol / Integer / Class / String alike through
|
|
3521
|
+
rb_ca_guess_type. */
|
|
3855
3522
|
data_type = (int32_t) rb_ca_guess_type(argv[1]);
|
|
3856
3523
|
naxes = (int8_t) (argc - 2);
|
|
3857
|
-
if ( naxes > CA_RANK_MAX ) {
|
|
3524
|
+
if ( naxes > CA_RANK_MAX ) { /* rank-check: a list of axes, each checked by the engine */
|
|
3858
3525
|
rb_raise(rb_eArgError, "too many axes");
|
|
3859
3526
|
}
|
|
3860
3527
|
for ( int i = 0; i < naxes; i++ ) {
|
|
3528
|
+
/* fits: an axis number, checked by the engine's init */
|
|
3861
3529
|
axes[i] = (int8_t) NUM2INT(argv[2 + i]);
|
|
3862
3530
|
}
|
|
3863
3531
|
return rb_ca_new_reduced(argv[0], axes, naxes, data_type, 0);
|
|
3864
3532
|
}
|
|
3865
3533
|
|
|
3866
|
-
/* ----
|
|
3534
|
+
/* ---- CAReduce smoke --------------------------------------------------
|
|
3867
3535
|
CAReduce has no public Ruby ctor (it's an internal class used by
|
|
3868
3536
|
CARefer mask handling). Tests need to drive kernel_iterator over
|
|
3869
3537
|
a CAReduce, so we expose a thin construction helper plus dedicated
|
|
@@ -3928,8 +3596,9 @@ rb_t1_smoke_reduce_read (VALUE klass, VALUE vparent, VALUE vcount, VALUE voffset
|
|
|
3928
3596
|
}
|
|
3929
3597
|
|
|
3930
3598
|
/* WRITE broadcast smoke for CAReduce: fills the reduce view with
|
|
3931
|
-
`val` (boolean / 0 or 1) and lets sync_slab
|
|
3932
|
-
|
|
3599
|
+
`val` (boolean / 0 or 1) and lets sync_slab send the changed cells
|
|
3600
|
+
back through the view (ki_scratch_push_changed), which broadcasts them
|
|
3601
|
+
to the parent. After the call parent's
|
|
3933
3602
|
[offset..offset+elems*count) bytes should all equal val. Returns
|
|
3934
3603
|
rc; on rc != OK parent is not modified. */
|
|
3935
3604
|
static VALUE
|
|
@@ -3942,6 +3611,7 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
|
|
|
3942
3611
|
ca_size_t n, i;
|
|
3943
3612
|
int rc;
|
|
3944
3613
|
CArray *reduce;
|
|
3614
|
+
/* fits: a development smoke argument, not a cell */
|
|
3945
3615
|
uint8_t v = (uint8_t) NUM2UINT(vval);
|
|
3946
3616
|
|
|
3947
3617
|
TypedData_Get_Struct(vparent, CArray, &carray_data_type, parent);
|
|
@@ -3957,15 +3627,15 @@ rb_t1_smoke_reduce_write_broadcast (VALUE klass, VALUE vparent,
|
|
|
3957
3627
|
}
|
|
3958
3628
|
while ( ca_iter_state_next_slab(&st, &p, NULL, &n) ) {
|
|
3959
3629
|
for ( i = 0; i < n; i++ ) p[i] = v;
|
|
3960
|
-
ca_iter_state_sync_slab(&st); /*
|
|
3630
|
+
ca_iter_state_sync_slab(&st); /* sends the changed cells back */
|
|
3961
3631
|
}
|
|
3962
3632
|
ca_iter_state_finish(&st);
|
|
3963
3633
|
ca_free(reduce);
|
|
3964
3634
|
return INT2NUM(CA_ITER_OK);
|
|
3965
3635
|
}
|
|
3966
3636
|
|
|
3967
|
-
/* ----
|
|
3968
|
-
CARemap has no public Ruby
|
|
3637
|
+
/* ---- CARemap smoke helpers --------------------------------------------
|
|
3638
|
+
CARemap has no public Ruby constructor, so we
|
|
3969
3639
|
construct it from C using the (ref, idx) args. Pattern mirrors
|
|
3970
3640
|
rb_t1_smoke_reduce_*: build the view, run the iterator state
|
|
3971
3641
|
machine, return the standard hash (rc / slabs / total_elems /
|
|
@@ -4032,10 +3702,10 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
|
|
|
4032
3702
|
view = ca_remap_new(ref, idx);
|
|
4033
3703
|
data = rb_str_new(0, 0);
|
|
4034
3704
|
|
|
4035
|
-
rc =
|
|
3705
|
+
rc = t1_smoke_open_rows(&st, view, 0);
|
|
4036
3706
|
if ( rc == CA_ITER_OK ) {
|
|
4037
3707
|
ca_size_t stride;
|
|
4038
|
-
while (
|
|
3708
|
+
while ( t1_smoke_next_row(&st, &p, &n, &stride) ) {
|
|
4039
3709
|
ca_size_t i;
|
|
4040
3710
|
for ( i = 0; i < n; i++ ) {
|
|
4041
3711
|
rb_str_cat(data, p + i * stride, st.bytes);
|
|
@@ -4058,7 +3728,8 @@ rb_t1_smoke_remap_read_strided (VALUE klass, VALUE vref, VALUE vidx)
|
|
|
4058
3728
|
}
|
|
4059
3729
|
|
|
4060
3730
|
/* WRITE smoke: fill view's slab buffer with `val` (float64), let
|
|
4061
|
-
sync_slab
|
|
3731
|
+
sync_slab send the changed cells back to ref through the view
|
|
3732
|
+
(ki_scratch_push_changed: ca_xfer_addrs or ca_xfer_all PUT). */
|
|
4062
3733
|
static VALUE
|
|
4063
3734
|
rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
|
|
4064
3735
|
VALUE vval)
|
|
@@ -4091,17 +3762,17 @@ rb_t1_smoke_remap_write_fill_f64 (VALUE klass, VALUE vref, VALUE vidx,
|
|
|
4091
3762
|
return INT2NUM(CA_ITER_OK);
|
|
4092
3763
|
}
|
|
4093
3764
|
|
|
4094
|
-
/* ----
|
|
3765
|
+
/* ---- CA_FOR_EACH_SLAB macro smoke surfaces ---------------------------
|
|
4095
3766
|
Smoke kernels that exercise the block macros end-to-end. Used by
|
|
4096
3767
|
spec_ai/test_ca_for_each_slab_macros.rb to pin behavioural
|
|
4097
3768
|
correctness of the macro expansion (= same byte-parity result as
|
|
4098
3769
|
the raw API equivalent). */
|
|
4099
3770
|
|
|
4100
|
-
/* sum reduction using CA_FOR_EACH_SLAB
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
3771
|
+
/* sum reduction using CA_FOR_EACH_SLAB over the slab axes given in
|
|
3772
|
+
argv[1..] (K-D slabs). Reads the source as float64 without checking
|
|
3773
|
+
its data_type. Accumulator = scalar (single full-reduction slab when
|
|
3774
|
+
naxes == src->ndim, otherwise per-outer-slab accumulator written to a
|
|
3775
|
+
row-major output buffer). */
|
|
4105
3776
|
static VALUE
|
|
4106
3777
|
rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
4107
3778
|
{
|
|
@@ -4120,6 +3791,7 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
4120
3791
|
rb_raise(rb_eArgError, "bad axes count");
|
|
4121
3792
|
}
|
|
4122
3793
|
for ( int8_t k = 0; k < naxes; k++ ) {
|
|
3794
|
+
/* fits: an axis number, checked by the engine's init */
|
|
4123
3795
|
slab_axes[k] = (int8_t) NUM2INT(argv[1 + k]);
|
|
4124
3796
|
if ( slab_axes[k] < 0 ) slab_axes[k] += ca->ndim;
|
|
4125
3797
|
}
|
|
@@ -4189,8 +3861,8 @@ rb_caf_smoke_sum_f64 (int argc, VALUE *argv, VALUE klass)
|
|
|
4189
3861
|
|
|
4190
3862
|
/* map kernel using CA_FOR_EACH_SLAB_INOUT. Doubles each element of
|
|
4191
3863
|
a float64 source into a fresh same-shape output. Slab axis is
|
|
4192
|
-
always the innermost (
|
|
4193
|
-
|
|
3864
|
+
always the innermost (ca->ndim - 1); there is no axis argument.
|
|
3865
|
+
Demonstrates the 2-iter pattern. */
|
|
4194
3866
|
static VALUE
|
|
4195
3867
|
rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
|
|
4196
3868
|
{
|
|
@@ -4234,11 +3906,11 @@ rb_caf_smoke_double_f64 (VALUE klass, VALUE vsrc)
|
|
|
4234
3906
|
return vout;
|
|
4235
3907
|
}
|
|
4236
3908
|
|
|
4237
|
-
/* ----
|
|
3909
|
+
/* ---- CA_FOR_EACH_FIBER macro smokes ----------------------------------
|
|
4238
3910
|
Exercise each of the 4 catalog forms end-to-end with float64 fibers,
|
|
4239
3911
|
verifying contig delivery for both innermost (= stride==bytes) and
|
|
4240
3912
|
non-innermost (= stride>bytes, gather path) axis positions. Used by
|
|
4241
|
-
spec_ai/test_fiber_delivery.rb
|
|
3913
|
+
spec_ai/test_fiber_delivery.rb. */
|
|
4242
3914
|
|
|
4243
3915
|
/* form 1 (NO_MASK single): per-axis sum, returns total sum.
|
|
4244
3916
|
Catalog: CA_FOR_EACH_FIBER. */
|
|
@@ -4264,8 +3936,7 @@ rb_caf_fiber_smoke_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4264
3936
|
}
|
|
4265
3937
|
|
|
4266
3938
|
/* form 2 (NO_MASK INOUT): per-axis fiber copy * scalar.
|
|
4267
|
-
Catalog: CA_FOR_EACH_FIBER_INOUT.
|
|
4268
|
-
gather/scatter behavior. */
|
|
3939
|
+
Catalog: CA_FOR_EACH_FIBER_INOUT. Doubles each cell of each fiber. */
|
|
4269
3940
|
static VALUE
|
|
4270
3941
|
rb_caf_fiber_smoke_double_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
4271
3942
|
{
|
|
@@ -4349,9 +4020,9 @@ rb_caf_fiber_smoke_zero_masked_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4349
4020
|
return vout;
|
|
4350
4021
|
}
|
|
4351
4022
|
|
|
4352
|
-
/*
|
|
4353
|
-
Functional equivalent of
|
|
4354
|
-
|
|
4023
|
+
/* Bench helper: sort_copy via CA_FOR_EACH_FIBER_INOUT.
|
|
4024
|
+
Functional equivalent of the per-axis sort_copy (rb_ca_sort_copy in
|
|
4025
|
+
ext/carray_sort.c).
|
|
4355
4026
|
The macro form lets us A/B the author-side overhead of stride math
|
|
4356
4027
|
vs the catalog-macro-driven engine gather/scatter. */
|
|
4357
4028
|
static int
|
|
@@ -4394,7 +4065,7 @@ rb_caf_fiber_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4394
4065
|
return vout;
|
|
4395
4066
|
}
|
|
4396
4067
|
|
|
4397
|
-
/*
|
|
4068
|
+
/* Bench helper variant: sort_copy via SLAB macro + manual
|
|
4398
4069
|
gather/scatter (= what hand-rolled sort_copy effectively does, but
|
|
4399
4070
|
driven through the public CA_FOR_EACH_SLAB_INOUT macro instead of
|
|
4400
4071
|
the raw next_slab_axes API). Lets us isolate macro overhead from
|
|
@@ -4418,9 +4089,13 @@ rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4418
4089
|
TypedData_Get_Struct(vout, CArray, &carray_data_type, out);
|
|
4419
4090
|
slab_axes[0] = (int8_t) axis;
|
|
4420
4091
|
|
|
4092
|
+
if ( axis < 0 || axis >= src->ndim ) {
|
|
4093
|
+
rb_raise(rb_eArgError, "caf_slab_bench_sort_copy_f64: axis out of range");
|
|
4094
|
+
}
|
|
4421
4095
|
ca_size_t fiber_n = src->dim[axis];
|
|
4422
4096
|
ca_size_t bytes = sizeof(double);
|
|
4423
|
-
|
|
4097
|
+
volatile VALUE holder; /* the walk can raise while buf is held */
|
|
4098
|
+
char *buf = ALLOCV_N(char, holder, fiber_n * bytes);
|
|
4424
4099
|
|
|
4425
4100
|
CA_FOR_EACH_SLAB_INOUT(st_in, st_out, src, out,
|
|
4426
4101
|
slab_axes, 1,
|
|
@@ -4452,11 +4127,11 @@ rb_caf_slab_bench_sort_copy_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4452
4127
|
}
|
|
4453
4128
|
}
|
|
4454
4129
|
}
|
|
4455
|
-
|
|
4130
|
+
ALLOCV_END(holder);
|
|
4456
4131
|
return vout;
|
|
4457
4132
|
}
|
|
4458
4133
|
|
|
4459
|
-
/*
|
|
4134
|
+
/* Bench helper: per-fiber fused xfer_stride direct into a contig
|
|
4460
4135
|
scratch, skipping the kernel_iterator SRC_ATTACH whole-view materialise.
|
|
4461
4136
|
Tests user's hypothesis: for transform views (CAFake/CAByteSwap etc.),
|
|
4462
4137
|
does bypassing the whole-view scratch_ptr and calling ca_xfer_stride
|
|
@@ -4502,7 +4177,8 @@ rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4502
4177
|
}
|
|
4503
4178
|
|
|
4504
4179
|
/* Fiber-sized contig scratch. */
|
|
4505
|
-
|
|
4180
|
+
volatile VALUE holder;
|
|
4181
|
+
char *buf = ALLOCV_N(char, holder, fiber_n * src->bytes);
|
|
4506
4182
|
|
|
4507
4183
|
for ( ca_size_t f = 0; f < total_fibers; f++ ) {
|
|
4508
4184
|
/* Build region: counts = 1 on all non-axis, fiber_n on axis;
|
|
@@ -4530,21 +4206,33 @@ rb_caf_bench_per_fiber_xfer_sum_f64 (VALUE klass, VALUE vsrc, VALUE vaxis)
|
|
|
4530
4206
|
}
|
|
4531
4207
|
}
|
|
4532
4208
|
|
|
4533
|
-
|
|
4209
|
+
ALLOCV_END(holder);
|
|
4534
4210
|
return rb_float_new(total);
|
|
4535
4211
|
}
|
|
4536
4212
|
|
|
4213
|
+
/* CArray.__iter_buffers__: see ki_buffers_out. */
|
|
4214
|
+
static VALUE
|
|
4215
|
+
rb_ki_buffers_out (VALUE klass)
|
|
4216
|
+
{
|
|
4217
|
+
(void) klass;
|
|
4218
|
+
return LONG2NUM(__atomic_load_n(&ki_buffers_out, __ATOMIC_RELAXED));
|
|
4219
|
+
}
|
|
4220
|
+
|
|
4537
4221
|
#endif /* CARRAY_DEV_BUILD — end smoke surface fence */
|
|
4538
4222
|
|
|
4539
4223
|
void
|
|
4540
4224
|
Init_ca_kernel_iterator (void)
|
|
4541
4225
|
{
|
|
4226
|
+
ki_kept_key = rb_ractor_local_storage_ptr_newkey(&ki_kept_type);
|
|
4227
|
+
|
|
4542
4228
|
#ifdef CARRAY_DEV_BUILD
|
|
4543
4229
|
/* ==== smoke surface registrations (dev-only, stripped in release) ====
|
|
4544
|
-
*
|
|
4230
|
+
* All `t1_smoke_*`, `caf_smoke_*`,
|
|
4545
4231
|
* `caf_fiber_smoke_*`, `caf_*_bench_*`, helper smokes, and `T1_*` test
|
|
4546
4232
|
* constants are gated here together — none of them are consumed by
|
|
4547
4233
|
* production code (lib/ / other ext/), only by spec_ai regression pins. */
|
|
4234
|
+
rb_define_singleton_method(rb_cCArray, "__iter_buffers__",
|
|
4235
|
+
rb_ki_buffers_out, 0);
|
|
4548
4236
|
rb_define_singleton_method(rb_cCArray, "t1_smoke",
|
|
4549
4237
|
rb_t1_smoke, 1);
|
|
4550
4238
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_strided",
|
|
@@ -4567,20 +4255,20 @@ Init_ca_kernel_iterator (void)
|
|
|
4567
4255
|
rb_t1_smoke_with_mask, 1);
|
|
4568
4256
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_init_rc",
|
|
4569
4257
|
rb_t1_smoke_init_rc, 2);
|
|
4570
|
-
/*
|
|
4258
|
+
/* CA_SLAB_AXES smoke (variadic axes) */
|
|
4571
4259
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_sum_axes_f64",
|
|
4572
4260
|
rb_t1_smoke_sum_axes_f64, -1);
|
|
4573
|
-
/*
|
|
4261
|
+
/* rb_ca_new_reduced helper smoke */
|
|
4574
4262
|
rb_define_singleton_method(rb_cCArray, "t1_test_new_reduced",
|
|
4575
4263
|
rb_t1_test_new_reduced, -1);
|
|
4576
|
-
/*
|
|
4264
|
+
/* rb_ca_parse_reduce_axes helper smoke */
|
|
4577
4265
|
rb_define_singleton_method(rb_cCArray, "t1_test_parse_reduce_axes",
|
|
4578
4266
|
rb_t1_test_parse_reduce_axes, -1);
|
|
4579
4267
|
|
|
4580
|
-
/*
|
|
4268
|
+
/* rb_ca_parse_reduce_axes_kw helper smoke */
|
|
4581
4269
|
rb_define_singleton_method(rb_cCArray, "test_parse_reduce_axes_kw",
|
|
4582
4270
|
rb_test_parse_reduce_axes_kw, -1);
|
|
4583
|
-
/* CAReduce
|
|
4271
|
+
/* CAReduce: no public Ruby API */
|
|
4584
4272
|
rb_define_singleton_method(rb_cCArray, "t1_make_reduce",
|
|
4585
4273
|
rb_t1_make_reduce, 3);
|
|
4586
4274
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_read",
|
|
@@ -4588,19 +4276,19 @@ Init_ca_kernel_iterator (void)
|
|
|
4588
4276
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_reduce_write_broadcast",
|
|
4589
4277
|
rb_t1_smoke_reduce_write_broadcast, 4);
|
|
4590
4278
|
|
|
4591
|
-
/*
|
|
4279
|
+
/* CARemap smokes (test-only). */
|
|
4592
4280
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read",
|
|
4593
4281
|
rb_t1_smoke_remap_read, 2);
|
|
4594
4282
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_read_strided",
|
|
4595
4283
|
rb_t1_smoke_remap_read_strided, 2);
|
|
4596
4284
|
rb_define_singleton_method(rb_cCArray, "t1_smoke_remap_write_fill_f64",
|
|
4597
4285
|
rb_t1_smoke_remap_write_fill_f64, 3);
|
|
4598
|
-
/*
|
|
4286
|
+
/* block macro smokes */
|
|
4599
4287
|
rb_define_singleton_method(rb_cCArray, "caf_smoke_sum_f64",
|
|
4600
4288
|
rb_caf_smoke_sum_f64, -1);
|
|
4601
4289
|
rb_define_singleton_method(rb_cCArray, "caf_smoke_double_f64",
|
|
4602
4290
|
rb_caf_smoke_double_f64, 1);
|
|
4603
|
-
/*
|
|
4291
|
+
/* fiber macro smokes */
|
|
4604
4292
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_sum_f64",
|
|
4605
4293
|
rb_caf_fiber_smoke_sum_f64, 2);
|
|
4606
4294
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_double_f64",
|
|
@@ -4609,7 +4297,7 @@ Init_ca_kernel_iterator (void)
|
|
|
4609
4297
|
rb_caf_fiber_smoke_unmasked_sum_f64, 2);
|
|
4610
4298
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_smoke_zero_masked_f64",
|
|
4611
4299
|
rb_caf_fiber_smoke_zero_masked_f64, 2);
|
|
4612
|
-
/*
|
|
4300
|
+
/* bench helper */
|
|
4613
4301
|
rb_define_singleton_method(rb_cCArray, "caf_fiber_bench_sort_copy_f64",
|
|
4614
4302
|
rb_caf_fiber_bench_sort_copy_f64, 2);
|
|
4615
4303
|
rb_define_singleton_method(rb_cCArray, "caf_slab_bench_sort_copy_f64",
|
|
@@ -4629,5 +4317,6 @@ Init_ca_kernel_iterator (void)
|
|
|
4629
4317
|
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_CONTIG", INT2NUM(CA_ITER_ALIAS_CONTIG));
|
|
4630
4318
|
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_STRIDED", INT2NUM(CA_ITER_ALIAS_STRIDED));
|
|
4631
4319
|
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_ATTACH", INT2NUM(CA_ITER_ALIAS_ATTACH));
|
|
4320
|
+
rb_define_const(rb_cCArray, "T1_ITER_ALIAS_PER_SLAB", INT2NUM(CA_ITER_ALIAS_PER_SLAB));
|
|
4632
4321
|
#endif /* CARRAY_DEV_BUILD */
|
|
4633
4322
|
}
|