carray 3.0.5 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +233 -0
- data/README.md +1 -1
- data/carray.gemspec +3 -2
- data/ext/ca_axis_dispatch.c +6 -0
- data/ext/ca_axis_group.c +1 -1
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.c +116 -231
- data/ext/ca_kernel_iterator.h +410 -254
- data/ext/ca_obj_bincmp.c +9 -1
- data/ext/ca_obj_binop.c +10 -1
- data/ext/ca_obj_bitfield.c +2 -0
- data/ext/ca_obj_byte_swap.c +2 -0
- data/ext/ca_obj_face.c +2 -0
- data/ext/ca_obj_fake.c +3 -1
- data/ext/ca_obj_grid.c +167 -12
- data/ext/ca_obj_meld.c +69 -113
- data/ext/ca_obj_moncmp.c +9 -1
- data/ext/ca_obj_monop.c +40 -2
- data/ext/ca_obj_object.c +54 -20
- data/ext/ca_obj_reduce.c +15 -1
- data/ext/ca_obj_refer.c +65 -20
- data/ext/ca_obj_remap.c +69 -10
- data/ext/ca_obj_roll.c +1 -0
- data/ext/ca_obj_select.c +4 -1
- data/ext/ca_obj_select_axis.c +1 -0
- data/ext/ca_obj_shift.c +1 -0
- data/ext/ca_obj_stack.c +310 -111
- data/ext/ca_obj_stride.c +131 -45
- data/ext/ca_obj_string.c +18 -2
- data/ext/ca_obj_tile.c +1 -0
- data/ext/ca_obj_triop.c +9 -1
- data/ext/ca_obj_window.c +1 -0
- data/ext/ca_op_ipower.c +8 -8
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +2 -1
- data/ext/carray_access.c +27 -64
- data/ext/carray_address_basis.c +134 -0
- data/ext/carray_attribute.c +8 -0
- data/ext/carray_broadcast.c +3 -4
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_cast.c +8 -11
- data/ext/carray_conversion.c +9 -5
- data/ext/carray_core.c +811 -72
- data/ext/carray_element.c +13 -9
- data/ext/carray_histogram.c +25 -3
- data/ext/carray_internal.h +147 -12
- data/ext/carray_kernels_bincmp.c +2007 -3
- data/ext/carray_kernels_binop.c +25365 -5199
- data/ext/carray_kernels_init.c +8 -8
- data/ext/carray_kernels_monop.c +10696 -3678
- data/ext/carray_kernels_reduce_aggregate.c +4429 -702
- data/ext/carray_kernels_reduce_boolean.c +1533 -15
- data/ext/carray_kernels_reduce_cumulative.c +1808 -248
- data/ext/carray_kernels_reduce_extreme.c +4508 -511
- data/ext/carray_kernels_reduce_variance.c +2284 -152
- data/ext/carray_kernels_scan.c +240 -240
- data/ext/carray_kernels_search.c +4211 -264
- data/ext/carray_kernels_sort.c +367 -450
- data/ext/carray_lazy.c +47 -0
- data/ext/carray_loop.c +1 -1
- data/ext/carray_mask.c +130 -139
- data/ext/carray_median_percentile.c +4 -4
- data/ext/carray_memory_view.c +13 -7
- data/ext/carray_operator.c +147 -44
- data/ext/carray_order.c +3 -2
- data/ext/carray_random.c +4 -1
- data/ext/carray_slab.c +97 -21
- data/ext/carray_sort.c +1 -1
- data/ext/carray_test.c +74 -19
- data/ext/carray_time_text.c +2 -2
- data/ext/extconf.rb +11 -0
- data/ext/mkkernel.rb +346 -94
- data/ext/ruby_carray.c +4 -1
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +25 -1
- data/lib/carray/autoload_carray.rb +3 -3
- data/lib/carray/autoload_method_extension.rb +5 -3
- data/lib/carray/axis_group.rb +13 -7
- data/lib/carray/basics.rb +1 -1
- data/lib/carray/block_iterator.rb +17 -3
- data/lib/carray/categorical.rb +6 -8
- data/lib/carray/categorical_iterator.rb +2 -2
- data/lib/carray/complex.rb +2 -2
- data/lib/carray/conditional.rb +3 -2
- data/lib/carray/frame/column_verbs.rb +45 -14
- data/lib/carray/frame/convert.rb +19 -0
- data/lib/carray/frame/csv_parser.rb +32 -32
- data/lib/carray/frame/describe.rb +4 -2
- data/lib/carray/frame/frame.rb +399 -78
- data/lib/carray/frame/group.rb +679 -53
- data/lib/carray/frame/io.rb +209 -71
- data/lib/carray/frame/join.rb +146 -37
- data/lib/carray/frame/records.rb +3 -3
- data/lib/carray/frame/reshape.rb +89 -11
- data/lib/carray/frame/sort.rb +57 -9
- data/lib/carray/frame/verbs.rb +384 -49
- data/lib/carray/frame/windows.rb +44 -6
- data/lib/carray/inspect.rb +7 -2
- data/lib/carray/iterator.rb +32 -0
- data/lib/carray/lazy.rb +17 -8
- data/lib/carray/mask_gap_fill.rb +37 -17
- data/lib/carray/meld_reduce.rb +1 -1
- data/lib/carray/methods/choose.rb +3 -2
- data/lib/carray/methods/meshgrid.rb +3 -1
- data/lib/carray/runtime.rb +47 -8
- data/lib/carray/segment_iterator.rb +23 -8
- data/lib/carray/serialize.rb +34 -17
- data/lib/carray/stack.rb +18 -6
- data/lib/carray/time.rb +5 -0
- data/lib/carray/window_iterator.rb +16 -4
- data/yard-stubs/ca_obj_array.rb +26 -1
- data/yard-stubs/carray_cast.rb +8 -5
- data/yard-stubs/carray_conversion.rb +8 -2
- data/yard-stubs/carray_mask.rb +20 -1
- data/yard-stubs/carray_math.rb +31 -16
- data/yard-stubs/carray_memory_view.rb +7 -0
- data/yard-stubs/carray_test.rb +7 -4
- metadata +18 -4
|
@@ -93,24 +93,516 @@ all_ki_stream_bool (VALUE arg)
|
|
|
93
93
|
return BOOL2VAL((boolean8_t) (acc));
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
97
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
98
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
99
|
+
typedef struct {
|
|
100
|
+
ca_iter_state states[1];
|
|
101
|
+
VALUE self;
|
|
102
|
+
CArray *ca;
|
|
103
|
+
int8_t *slab_axes;
|
|
104
|
+
int8_t naxes;
|
|
105
|
+
int keep_axis;
|
|
106
|
+
} all_ki_native_bool_frame;
|
|
107
|
+
|
|
96
108
|
static VALUE
|
|
97
|
-
|
|
109
|
+
all_ki_native_bool_body (VALUE arg)
|
|
98
110
|
{
|
|
111
|
+
all_ki_native_bool_frame *fr = (all_ki_native_bool_frame *) arg;
|
|
112
|
+
VALUE self = fr->self;
|
|
113
|
+
(void) self;
|
|
114
|
+
CArray *ca = fr->ca;
|
|
115
|
+
(void) ca;
|
|
116
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
117
|
+
(void) slab_axes;
|
|
118
|
+
int8_t naxes = fr->naxes;
|
|
119
|
+
(void) naxes;
|
|
120
|
+
int keep_axis = fr->keep_axis;
|
|
121
|
+
(void) keep_axis;
|
|
122
|
+
#define st (fr->states[0])
|
|
99
123
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
100
124
|
ca->elements > 0 &&
|
|
101
125
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
102
|
-
|
|
126
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
103
127
|
all_ki_stream_bool_t __ctx;
|
|
104
128
|
__ctx.chunk = NULL;
|
|
105
129
|
__ctx.ca = ca;
|
|
106
130
|
return ca_lazy_arena_protect(all_ki_stream_bool, ca_lazy_arena_release_held,
|
|
107
131
|
(VALUE) &__ctx);
|
|
108
132
|
}
|
|
133
|
+
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
134
|
+
(PROPOSAL_REDUCTION_LOOP_INTERCHANGE).
|
|
135
|
+
MEMO_REDUCTION_FASTPATH_ENTITY_ONLY_GUARD Tier 1: extend from
|
|
136
|
+
entity-only to any contig row-major buffer. ca_attach_is_alias
|
|
137
|
+
is true for CA_OBJ_ARRAY and for CAStride-family views whose
|
|
138
|
+
composed strides are row-major contiguous (CARefer reshape,
|
|
139
|
+
contig CABlock, CAFarray, alias CAStride, ...); ca_attach then
|
|
140
|
+
exposes ca->ptr as that buffer at O(1) cost (no materialise). */
|
|
141
|
+
if ( naxes == 1
|
|
142
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
143
|
+
&& ! ca_has_mask(ca)
|
|
144
|
+
&& ca_attach_is_alias(ca)
|
|
145
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
146
|
+
int8_t __li_ax = slab_axes[0];
|
|
147
|
+
ca_size_t __li_M = ca->dim[__li_ax];
|
|
148
|
+
ca_size_t __li_INNER = 1;
|
|
149
|
+
for ( int8_t __k = (int8_t)(__li_ax + 1); __k < ca->ndim; __k++ ) __li_INNER *= ca->dim[__k];
|
|
150
|
+
ca_size_t __li_OUTER = 1;
|
|
151
|
+
for ( int8_t __k = 0; __k < __li_ax; __k++ ) __li_OUTER *= ca->dim[__k];
|
|
152
|
+
|
|
153
|
+
/* Threshold gate: skip when too small to amortise alloc + 2-pass overhead. */
|
|
154
|
+
if ( __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
155
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
156
|
+
CArray *co;
|
|
157
|
+
GetCArray(vout, co);
|
|
158
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
159
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
160
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
161
|
+
|
|
162
|
+
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
163
|
+
to L7 TILE = 512 cells regardless of INNER). */
|
|
164
|
+
boolean8_t __li_buf_acc[512];
|
|
165
|
+
|
|
166
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
167
|
+
const ca_size_t __li_TILE = 512;
|
|
168
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
169
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
170
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
171
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
172
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
173
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
174
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
175
|
+
}
|
|
176
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
177
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
178
|
+
_Pragma("omp simd")
|
|
179
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
180
|
+
boolean8_t v = __li_row[__j];
|
|
181
|
+
__li_buf_acc[__j] &= v;
|
|
182
|
+
(void) v;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
187
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
ca_detach(ca);
|
|
192
|
+
|
|
193
|
+
if ( naxes == ca->ndim && !keep_axis ) return BOOL2VAL(op[0]); /* unreachable: ax != innermost && naxes == 1 < ndim */
|
|
194
|
+
return vout;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/* MEMO_REDUCTION_FASTPATH_ENTITY_ONLY_GUARD Tier 2: CAStack source.
|
|
199
|
+
view.reduce(axis: ax>=1) is exactly K independent per-parent
|
|
200
|
+
reduces over (ax - 1) of each contiguous parent; output row k
|
|
201
|
+
= parent k's reduced plane. Run the L.7 core per parent (each
|
|
202
|
+
attach is O(1) for entity parents), no whole-view materialise.
|
|
203
|
+
|
|
204
|
+
K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): gated to k_axis == 0
|
|
205
|
+
because the per-parent output layout in dst assumes K is the
|
|
206
|
+
outermost output axis (= each parent contributes a contiguous
|
|
207
|
+
block of parent_OUTER * parent_INNER cells). For k_axis != 0
|
|
208
|
+
the parents' outputs are interleaved in dst -- generalising
|
|
209
|
+
the output stride is doable but out of K.3 scope. The
|
|
210
|
+
dispatch falls through to ca_iter_state_init_l2 SLAB_AXES
|
|
211
|
+
which engages the K.3-generalised tile cache. */
|
|
212
|
+
if ( naxes == 1
|
|
213
|
+
&& slab_axes[0] >= 1
|
|
214
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
215
|
+
&& ! ca_has_mask(ca)
|
|
216
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
217
|
+
&& ((CAStack *) ca)->k_axis == 0
|
|
218
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
219
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
220
|
+
int8_t __li_ax = slab_axes[0];
|
|
221
|
+
ca_size_t __li_M = ca->dim[__li_ax];
|
|
222
|
+
ca_size_t __li_INNER = 1;
|
|
223
|
+
for ( int8_t __k = (int8_t)(__li_ax + 1); __k < ca->ndim; __k++ ) __li_INNER *= ca->dim[__k];
|
|
224
|
+
ca_size_t __li_OUTER = 1; /* parent outer = dims 1..ax-1 (excludes K = axis 0) */
|
|
225
|
+
for ( int8_t __k = 1; __k < __li_ax; __k++ ) __li_OUTER *= ca->dim[__k];
|
|
226
|
+
|
|
227
|
+
/* Bail to the generic path if any parent carries a mask (the
|
|
228
|
+
core reads parent->ptr without mask awareness). */
|
|
229
|
+
int __li_ok = 1;
|
|
230
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
231
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if ( __li_ok && __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
235
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
236
|
+
CArray *co;
|
|
237
|
+
GetCArray(vout, co);
|
|
238
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
239
|
+
|
|
240
|
+
boolean8_t __li_buf_acc[512];
|
|
241
|
+
|
|
242
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
243
|
+
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
244
|
+
const ca_size_t __li_TILE = 512;
|
|
245
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
246
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
247
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
248
|
+
boolean8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
249
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
250
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
251
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
252
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
253
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
254
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
255
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
256
|
+
}
|
|
257
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
258
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
259
|
+
_Pragma("omp simd")
|
|
260
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
261
|
+
boolean8_t v = __li_row[__j];
|
|
262
|
+
__li_buf_acc[__j] &= v;
|
|
263
|
+
(void) v;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
268
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
ca_detach(__li_st->parents[__kk]);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
return vout;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/* M.1.0 + M.2.0a (PROPOSAL_MKKERNEL_TIER2_K_AXIS_GEN, 2026-06-20):
|
|
280
|
+
Tier 2 fast path for CAStack with k_axis > 0, covering two
|
|
281
|
+
structurally-equivalent cases:
|
|
282
|
+
|
|
283
|
+
M.1 ax > k_axis parent INNER all post-K in output
|
|
284
|
+
M.2a ax < k_axis = ax + 1 parent INNER all post-K in output
|
|
285
|
+
(K directly after reduce axis)
|
|
286
|
+
|
|
287
|
+
In both cases parent INNER axes remain contig at the output
|
|
288
|
+
tail (j-stride 1 preserved -> SIMD tile write maintained),
|
|
289
|
+
parent PRE axes form a scattered OUTER (per-OUTER output base
|
|
290
|
+
computed via output stride table + carry-increment multi-index).
|
|
291
|
+
The k_axis == 0 branch above stays as the tightest fast path
|
|
292
|
+
(PRE contig in output, no scatter).
|
|
293
|
+
|
|
294
|
+
Out of this branch: M.2b (ax < k_axis - 1) where parent INNER
|
|
295
|
+
is split by K in output (= INNER tile write becomes scatter,
|
|
296
|
+
SIMD-impacting). Falls through to the generic init_l2
|
|
297
|
+
SLAB_AXES path. */
|
|
298
|
+
if ( naxes == 1
|
|
299
|
+
&& slab_axes[0] >= 1
|
|
300
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
301
|
+
&& ! ca_has_mask(ca)
|
|
302
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
303
|
+
&& ((CAStack *) ca)->k_axis > 0
|
|
304
|
+
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
305
|
+
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
306
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
307
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
308
|
+
int8_t __li_ax = slab_axes[0];
|
|
309
|
+
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
310
|
+
CArray *__li_p0 = __li_st->parents[0]; /* shape representative */
|
|
311
|
+
int8_t __li_pa = (__li_ax < __li_kax) ? __li_ax : (int8_t)(__li_ax - 1);
|
|
312
|
+
int8_t __li_kax_out = (__li_ax < __li_kax) ? (int8_t)(__li_kax - 1) : __li_kax;
|
|
313
|
+
ca_size_t __li_M = __li_p0->dim[__li_pa]; /* = ca->dim[__li_ax] for ax != kax */
|
|
314
|
+
ca_size_t __li_INNER = 1;
|
|
315
|
+
for ( int8_t __k = (int8_t)(__li_pa + 1); __k < __li_p0->ndim; __k++ ) __li_INNER *= __li_p0->dim[__k];
|
|
316
|
+
ca_size_t __li_OUTER = 1;
|
|
317
|
+
for ( int8_t __k = 0; __k < __li_pa; __k++ ) __li_OUTER *= __li_p0->dim[__k];
|
|
318
|
+
|
|
319
|
+
/* Bail to the generic path if any parent carries a mask. */
|
|
320
|
+
int __li_ok = 1;
|
|
321
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
322
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
if ( __li_ok && __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
326
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
327
|
+
CArray *co;
|
|
328
|
+
GetCArray(vout, co);
|
|
329
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
330
|
+
|
|
331
|
+
/* Output row-major stride table (length = ca->ndim - 1).
|
|
332
|
+
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
333
|
+
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
334
|
+
{
|
|
335
|
+
ca_size_t __li_s = 1;
|
|
336
|
+
for ( int8_t __vo = (int8_t)(ca->ndim - 2); __vo >= 0; __vo-- ) {
|
|
337
|
+
int8_t __vv = (__vo < __li_ax) ? __vo : (int8_t)(__vo + 1);
|
|
338
|
+
__li_out_stride[__vo] = __li_s;
|
|
339
|
+
__li_s *= ca->dim[__vv];
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
ca_size_t __li_k_stride = __li_out_stride[__li_kax_out];
|
|
343
|
+
|
|
344
|
+
/* Per-parent-PRE output strides + dims. PRE view position
|
|
345
|
+
vp = (pd < kax) ? pd : pd + 1. PRE output position
|
|
346
|
+
vo = (vp < ax) ? vp : vp - 1. For M.1 (ax > kax) all PRE
|
|
347
|
+
view pos < ax so vo = vp. For M.2a (ax < kax = ax+1) all
|
|
348
|
+
PRE pd < ax < kax so vp = pd and vo = pd. */
|
|
349
|
+
ca_size_t __li_pre_out_stride[CA_RANK_MAX];
|
|
350
|
+
ca_size_t __li_pre_dim[CA_RANK_MAX];
|
|
351
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
352
|
+
int8_t __vp = (__pd < __li_kax) ? __pd : (int8_t)(__pd + 1);
|
|
353
|
+
int8_t __vo = (__vp < __li_ax) ? __vp : (int8_t)(__vp - 1);
|
|
354
|
+
__li_pre_out_stride[__pd] = __li_out_stride[__vo];
|
|
355
|
+
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
boolean8_t __li_buf_acc[512];
|
|
359
|
+
|
|
360
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
361
|
+
const ca_size_t __li_TILE = 512;
|
|
362
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
363
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
364
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
365
|
+
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
366
|
+
|
|
367
|
+
ca_size_t __li_idx[CA_RANK_MAX];
|
|
368
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) __li_idx[__pd] = 0;
|
|
369
|
+
|
|
370
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
371
|
+
ca_size_t __li_obase = __li_k_base;
|
|
372
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
373
|
+
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
374
|
+
}
|
|
375
|
+
boolean8_t *__li_op_k = op + __li_obase;
|
|
376
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
377
|
+
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
378
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
379
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
380
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
381
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
382
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
383
|
+
}
|
|
384
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
385
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
386
|
+
_Pragma("omp simd")
|
|
387
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
388
|
+
boolean8_t v = __li_row[__j];
|
|
389
|
+
__li_buf_acc[__j] &= v;
|
|
390
|
+
(void) v;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
395
|
+
__li_op_k[__li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
399
|
+
for ( int8_t __pd = (int8_t)(__li_pa - 1); __pd >= 0; __pd-- ) {
|
|
400
|
+
if ( ++__li_idx[__pd] < __li_pre_dim[__pd] ) break;
|
|
401
|
+
__li_idx[__pd] = 0;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
ca_detach(__li_st->parents[__kk]);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
return vout;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/* M.2.0b (PROPOSAL_MKKERNEL_TIER2_K_AXIS_GEN, 2026-06-20):
|
|
412
|
+
Case M.2b (ax < k_axis - 1): parent INNER axes are split by K
|
|
413
|
+
in the output. Strategy C (effective OUTER expansion): treat
|
|
414
|
+
INNER_pre_K dims (parent dims [pa+1..k_axis-1]) as part of an
|
|
415
|
+
extended effective OUTER, and use effective INNER = parent dims
|
|
416
|
+
[k_axis..pndim-1] (= INNER_post_K, contig at output tail).
|
|
417
|
+
|
|
418
|
+
L.7 core variant: M-step stride in parent = full parent INNER
|
|
419
|
+
total (= Π pdim[pa+1..pndim-1]), separate from the tile loop
|
|
420
|
+
bound (= eff_INNER). Inner read stride 1 in parent + output
|
|
421
|
+
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
422
|
+
tile reduce + write maintained.
|
|
423
|
+
|
|
424
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
425
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
426
|
+
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
427
|
+
= 2.38x slow. Root cause is the multi-parent data layout
|
|
428
|
+
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
429
|
+
which costs DRAM/TLB/prefetcher efficiency irrespective of
|
|
430
|
+
strategy. Strategy A (temp buf + scatter, explored rev1
|
|
431
|
+
sparring round 1) was implemented and benched at 3783 us
|
|
432
|
+
= within noise of Strategy C, confirming the bottleneck is
|
|
433
|
+
structural, NOT cache-stride from this branch. Strategy C
|
|
434
|
+
is shipped here (= simpler emit, no malloc) per round 2
|
|
435
|
+
sparring decision; the 3.94x win vs generic fallback
|
|
436
|
+
(.copy.sum at 14872 us) justifies landing despite missing
|
|
437
|
+
eager-entity parity. Strategy A retained in PROPOSAL as
|
|
438
|
+
future-optimization substrate (= multi-level tile-M + tile-
|
|
439
|
+
INNER could reclaim DRAM efficiency if revisited).
|
|
440
|
+
|
|
441
|
+
Threshold eff_INNER >= 32 auto-bypasses at = -1 (k_axis =
|
|
442
|
+
pndim -> eff_INNER = 1, fast path skipped, generic path
|
|
443
|
+
handles correctness). */
|
|
444
|
+
if ( naxes == 1
|
|
445
|
+
&& slab_axes[0] >= 1
|
|
446
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
447
|
+
&& ! ca_has_mask(ca)
|
|
448
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
449
|
+
&& ((CAStack *) ca)->k_axis > 0
|
|
450
|
+
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
451
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
452
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
453
|
+
int8_t __li_ax = slab_axes[0];
|
|
454
|
+
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
455
|
+
CArray *__li_p0 = __li_st->parents[0];
|
|
456
|
+
int8_t __li_pndim = __li_p0->ndim;
|
|
457
|
+
int8_t __li_pa = __li_ax; /* M.2b: ax < kax -> pa = ax */
|
|
458
|
+
int8_t __li_kax_out = (int8_t)(__li_kax - 1);
|
|
459
|
+
|
|
460
|
+
ca_size_t __li_M = __li_p0->dim[__li_pa];
|
|
461
|
+
ca_size_t __li_eff_INNER = 1; /* Π pdim[kax..pndim-1] = INNER_post_K */
|
|
462
|
+
for ( int8_t __k = __li_kax; __k < __li_pndim; __k++ ) __li_eff_INNER *= __li_p0->dim[__k];
|
|
463
|
+
ca_size_t __li_M_stride = 1; /* Π pdim[pa+1..pndim-1] = full parent INNER */
|
|
464
|
+
for ( int8_t __k = (int8_t)(__li_pa + 1); __k < __li_pndim; __k++ ) __li_M_stride *= __li_p0->dim[__k];
|
|
465
|
+
|
|
466
|
+
/* Bail to the generic path if any parent carries a mask. */
|
|
467
|
+
int __li_ok = 1;
|
|
468
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
469
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
if ( __li_ok && __li_eff_INNER >= 32 && __li_M * __li_eff_INNER >= 1024 ) {
|
|
473
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
474
|
+
CArray *co;
|
|
475
|
+
GetCArray(vout, co);
|
|
476
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
477
|
+
|
|
478
|
+
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
479
|
+
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
480
|
+
{
|
|
481
|
+
ca_size_t __li_s = 1;
|
|
482
|
+
for ( int8_t __vo = (int8_t)(ca->ndim - 2); __vo >= 0; __vo-- ) {
|
|
483
|
+
int8_t __vv = (__vo < __li_ax) ? __vo : (int8_t)(__vo + 1);
|
|
484
|
+
__li_out_stride[__vo] = __li_s;
|
|
485
|
+
__li_s *= ca->dim[__vv];
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
ca_size_t __li_k_stride = __li_out_stride[__li_kax_out];
|
|
489
|
+
|
|
490
|
+
/* Parent stride table (row-major). */
|
|
491
|
+
ca_size_t __li_pstride[CA_RANK_MAX];
|
|
492
|
+
{
|
|
493
|
+
ca_size_t __li_pss = 1;
|
|
494
|
+
for ( int8_t __k = (int8_t)(__li_pndim - 1); __k >= 0; __k-- ) {
|
|
495
|
+
__li_pstride[__k] = __li_pss;
|
|
496
|
+
__li_pss *= __li_p0->dim[__k];
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/* Effective OUTER dim list: parent dims [0..kax-1] excluding pa.
|
|
501
|
+
PRE = [0..pa-1] + INNER_pre_K = [pa+1..kax-1]. For each, store
|
|
502
|
+
parent dim, parent stride, output stride. */
|
|
503
|
+
int8_t __li_eo_n = 0;
|
|
504
|
+
ca_size_t __li_eo_dim[CA_RANK_MAX];
|
|
505
|
+
ca_size_t __li_eo_pstride[CA_RANK_MAX];
|
|
506
|
+
ca_size_t __li_eo_ostride[CA_RANK_MAX];
|
|
507
|
+
for ( int8_t __d = 0; __d < __li_kax; __d++ ) {
|
|
508
|
+
if ( __d == __li_pa ) continue;
|
|
509
|
+
int8_t __vp = __d; /* M.2b: all eff OUTER d < kax -> vp = d */
|
|
510
|
+
int8_t __vo = (__vp < __li_ax) ? __vp : (int8_t)(__vp - 1);
|
|
511
|
+
__li_eo_dim[__li_eo_n] = __li_p0->dim[__d];
|
|
512
|
+
__li_eo_pstride[__li_eo_n] = __li_pstride[__d];
|
|
513
|
+
__li_eo_ostride[__li_eo_n] = __li_out_stride[__vo];
|
|
514
|
+
__li_eo_n++;
|
|
515
|
+
}
|
|
516
|
+
ca_size_t __li_OUTER = 1;
|
|
517
|
+
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
518
|
+
|
|
519
|
+
boolean8_t __li_buf_acc[512];
|
|
520
|
+
|
|
521
|
+
const ca_size_t __li_TILE = 512;
|
|
522
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
523
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
524
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
525
|
+
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
526
|
+
|
|
527
|
+
ca_size_t __li_idx[CA_RANK_MAX];
|
|
528
|
+
for ( int8_t __i = 0; __i < __li_eo_n; __i++ ) __li_idx[__i] = 0;
|
|
529
|
+
|
|
530
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
531
|
+
ca_size_t __li_pbase = 0;
|
|
532
|
+
ca_size_t __li_obase = __li_k_base;
|
|
533
|
+
for ( int8_t __i = 0; __i < __li_eo_n; __i++ ) {
|
|
534
|
+
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
535
|
+
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
536
|
+
}
|
|
537
|
+
boolean8_t *__li_op_k = op + __li_obase;
|
|
538
|
+
const boolean8_t *__li_plane = p_root + __li_pbase;
|
|
539
|
+
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
540
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
541
|
+
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
542
|
+
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
543
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
544
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
545
|
+
}
|
|
546
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
547
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
548
|
+
_Pragma("omp simd")
|
|
549
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
550
|
+
boolean8_t v = __li_row[__j];
|
|
551
|
+
__li_buf_acc[__j] &= v;
|
|
552
|
+
(void) v;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
557
|
+
__li_op_k[__li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
/* Carry-increment effective OUTER multi-index. */
|
|
561
|
+
for ( int8_t __i = (int8_t)(__li_eo_n - 1); __i >= 0; __i-- ) {
|
|
562
|
+
if ( ++__li_idx[__i] < __li_eo_dim[__i] ) break;
|
|
563
|
+
__li_idx[__i] = 0;
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
ca_detach(__li_st->parents[__kk]);
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
return vout;
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
if ( naxes == 1
|
|
573
|
+
&& ca->ndim >= 2
|
|
574
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
575
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
576
|
+
&& ca->dim[ca->ndim - 1] <= 16
|
|
577
|
+
&& ! ca_has_mask(ca)
|
|
578
|
+
&& ca_attach_is_alias(ca)
|
|
579
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
580
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
581
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
582
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
583
|
+
CArray *co;
|
|
584
|
+
GetCArray(vout, co);
|
|
585
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
586
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
587
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
588
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
589
|
+
const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
590
|
+
boolean8_t acc = (boolean8_t) (1);
|
|
591
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
592
|
+
boolean8_t v = __sf_row[__j];
|
|
593
|
+
acc &= v;
|
|
594
|
+
(void) v;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
op[__sf_o] = (boolean8_t) (acc);
|
|
598
|
+
}
|
|
599
|
+
ca_detach(ca);
|
|
600
|
+
return vout;
|
|
601
|
+
}
|
|
109
602
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
110
603
|
CArray *co;
|
|
111
604
|
GetCArray(vout, co);
|
|
112
605
|
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
113
|
-
ca_iter_state st;
|
|
114
606
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
115
607
|
slab_axes, naxes, 0);
|
|
116
608
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -122,13 +614,27 @@ boolean8_t *m;
|
|
|
122
614
|
ca_size_t out_i = 0;
|
|
123
615
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
124
616
|
boolean8_t acc;
|
|
125
|
-
|
|
617
|
+
CA_SLAB_REDUCE_T_SETTLE(boolean8_t, st, p, m, acc, 1, acc &= v, !acc);
|
|
126
618
|
op[out_i++] = (boolean8_t) (acc);
|
|
127
619
|
}
|
|
128
620
|
ca_iter_state_finish(&st);
|
|
129
621
|
if ( naxes == ca->ndim && !keep_axis ) return BOOL2VAL(op[0]);
|
|
130
622
|
return vout;
|
|
131
|
-
|
|
623
|
+
#undef st
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
static VALUE
|
|
627
|
+
all_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis)
|
|
628
|
+
{
|
|
629
|
+
all_ki_native_bool_frame fr;
|
|
630
|
+
fr.self = self;
|
|
631
|
+
fr.ca = ca;
|
|
632
|
+
fr.slab_axes = slab_axes;
|
|
633
|
+
fr.naxes = naxes;
|
|
634
|
+
fr.keep_axis = keep_axis;
|
|
635
|
+
return ca_iter_ensure(1, fr.states, all_ki_native_bool_body, (VALUE) &fr);
|
|
636
|
+
}
|
|
637
|
+
|
|
132
638
|
|
|
133
639
|
VALUE
|
|
134
640
|
rb_ca_all_ki (int argc, VALUE *argv, VALUE self)
|
|
@@ -210,24 +716,516 @@ any_ki_stream_bool (VALUE arg)
|
|
|
210
716
|
return BOOL2VAL((boolean8_t) (acc));
|
|
211
717
|
}
|
|
212
718
|
|
|
719
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
720
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
721
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
722
|
+
typedef struct {
|
|
723
|
+
ca_iter_state states[1];
|
|
724
|
+
VALUE self;
|
|
725
|
+
CArray *ca;
|
|
726
|
+
int8_t *slab_axes;
|
|
727
|
+
int8_t naxes;
|
|
728
|
+
int keep_axis;
|
|
729
|
+
} any_ki_native_bool_frame;
|
|
730
|
+
|
|
213
731
|
static VALUE
|
|
214
|
-
|
|
732
|
+
any_ki_native_bool_body (VALUE arg)
|
|
215
733
|
{
|
|
734
|
+
any_ki_native_bool_frame *fr = (any_ki_native_bool_frame *) arg;
|
|
735
|
+
VALUE self = fr->self;
|
|
736
|
+
(void) self;
|
|
737
|
+
CArray *ca = fr->ca;
|
|
738
|
+
(void) ca;
|
|
739
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
740
|
+
(void) slab_axes;
|
|
741
|
+
int8_t naxes = fr->naxes;
|
|
742
|
+
(void) naxes;
|
|
743
|
+
int keep_axis = fr->keep_axis;
|
|
744
|
+
(void) keep_axis;
|
|
745
|
+
#define st (fr->states[0])
|
|
216
746
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
217
747
|
ca->elements > 0 &&
|
|
218
748
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
219
|
-
|
|
749
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
220
750
|
any_ki_stream_bool_t __ctx;
|
|
221
751
|
__ctx.chunk = NULL;
|
|
222
752
|
__ctx.ca = ca;
|
|
223
753
|
return ca_lazy_arena_protect(any_ki_stream_bool, ca_lazy_arena_release_held,
|
|
224
754
|
(VALUE) &__ctx);
|
|
225
755
|
}
|
|
756
|
+
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
757
|
+
(PROPOSAL_REDUCTION_LOOP_INTERCHANGE).
|
|
758
|
+
MEMO_REDUCTION_FASTPATH_ENTITY_ONLY_GUARD Tier 1: extend from
|
|
759
|
+
entity-only to any contig row-major buffer. ca_attach_is_alias
|
|
760
|
+
is true for CA_OBJ_ARRAY and for CAStride-family views whose
|
|
761
|
+
composed strides are row-major contiguous (CARefer reshape,
|
|
762
|
+
contig CABlock, CAFarray, alias CAStride, ...); ca_attach then
|
|
763
|
+
exposes ca->ptr as that buffer at O(1) cost (no materialise). */
|
|
764
|
+
if ( naxes == 1
|
|
765
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
766
|
+
&& ! ca_has_mask(ca)
|
|
767
|
+
&& ca_attach_is_alias(ca)
|
|
768
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
769
|
+
int8_t __li_ax = slab_axes[0];
|
|
770
|
+
ca_size_t __li_M = ca->dim[__li_ax];
|
|
771
|
+
ca_size_t __li_INNER = 1;
|
|
772
|
+
for ( int8_t __k = (int8_t)(__li_ax + 1); __k < ca->ndim; __k++ ) __li_INNER *= ca->dim[__k];
|
|
773
|
+
ca_size_t __li_OUTER = 1;
|
|
774
|
+
for ( int8_t __k = 0; __k < __li_ax; __k++ ) __li_OUTER *= ca->dim[__k];
|
|
775
|
+
|
|
776
|
+
/* Threshold gate: skip when too small to amortise alloc + 2-pass overhead. */
|
|
777
|
+
if ( __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
778
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
779
|
+
CArray *co;
|
|
780
|
+
GetCArray(vout, co);
|
|
781
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
782
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
783
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
784
|
+
|
|
785
|
+
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
786
|
+
to L7 TILE = 512 cells regardless of INNER). */
|
|
787
|
+
boolean8_t __li_buf_acc[512];
|
|
788
|
+
|
|
789
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
790
|
+
const ca_size_t __li_TILE = 512;
|
|
791
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
792
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
793
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
794
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
795
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
796
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
797
|
+
__li_buf_acc[__j] = (boolean8_t) (0);
|
|
798
|
+
}
|
|
799
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
800
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
801
|
+
_Pragma("omp simd")
|
|
802
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
803
|
+
boolean8_t v = __li_row[__j];
|
|
804
|
+
__li_buf_acc[__j] |= v;
|
|
805
|
+
(void) v;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
810
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
ca_detach(ca);
|
|
815
|
+
|
|
816
|
+
if ( naxes == ca->ndim && !keep_axis ) return BOOL2VAL(op[0]); /* unreachable: ax != innermost && naxes == 1 < ndim */
|
|
817
|
+
return vout;
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
/* MEMO_REDUCTION_FASTPATH_ENTITY_ONLY_GUARD Tier 2: CAStack source.
|
|
822
|
+
view.reduce(axis: ax>=1) is exactly K independent per-parent
|
|
823
|
+
reduces over (ax - 1) of each contiguous parent; output row k
|
|
824
|
+
= parent k's reduced plane. Run the L.7 core per parent (each
|
|
825
|
+
attach is O(1) for entity parents), no whole-view materialise.
|
|
826
|
+
|
|
827
|
+
K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): gated to k_axis == 0
|
|
828
|
+
because the per-parent output layout in dst assumes K is the
|
|
829
|
+
outermost output axis (= each parent contributes a contiguous
|
|
830
|
+
block of parent_OUTER * parent_INNER cells). For k_axis != 0
|
|
831
|
+
the parents' outputs are interleaved in dst -- generalising
|
|
832
|
+
the output stride is doable but out of K.3 scope. The
|
|
833
|
+
dispatch falls through to ca_iter_state_init_l2 SLAB_AXES
|
|
834
|
+
which engages the K.3-generalised tile cache. */
|
|
835
|
+
if ( naxes == 1
|
|
836
|
+
&& slab_axes[0] >= 1
|
|
837
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
838
|
+
&& ! ca_has_mask(ca)
|
|
839
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
840
|
+
&& ((CAStack *) ca)->k_axis == 0
|
|
841
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
842
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
843
|
+
int8_t __li_ax = slab_axes[0];
|
|
844
|
+
ca_size_t __li_M = ca->dim[__li_ax];
|
|
845
|
+
ca_size_t __li_INNER = 1;
|
|
846
|
+
for ( int8_t __k = (int8_t)(__li_ax + 1); __k < ca->ndim; __k++ ) __li_INNER *= ca->dim[__k];
|
|
847
|
+
ca_size_t __li_OUTER = 1; /* parent outer = dims 1..ax-1 (excludes K = axis 0) */
|
|
848
|
+
for ( int8_t __k = 1; __k < __li_ax; __k++ ) __li_OUTER *= ca->dim[__k];
|
|
849
|
+
|
|
850
|
+
/* Bail to the generic path if any parent carries a mask (the
|
|
851
|
+
core reads parent->ptr without mask awareness). */
|
|
852
|
+
int __li_ok = 1;
|
|
853
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
854
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
if ( __li_ok && __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
858
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
859
|
+
CArray *co;
|
|
860
|
+
GetCArray(vout, co);
|
|
861
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
862
|
+
|
|
863
|
+
boolean8_t __li_buf_acc[512];
|
|
864
|
+
|
|
865
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
866
|
+
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
867
|
+
const ca_size_t __li_TILE = 512;
|
|
868
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
869
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
870
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
871
|
+
boolean8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
872
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
873
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
874
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
875
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
876
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
877
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
878
|
+
__li_buf_acc[__j] = (boolean8_t) (0);
|
|
879
|
+
}
|
|
880
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
881
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
882
|
+
_Pragma("omp simd")
|
|
883
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
884
|
+
boolean8_t v = __li_row[__j];
|
|
885
|
+
__li_buf_acc[__j] |= v;
|
|
886
|
+
(void) v;
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
891
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
ca_detach(__li_st->parents[__kk]);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
return vout;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
/* M.1.0 + M.2.0a (PROPOSAL_MKKERNEL_TIER2_K_AXIS_GEN, 2026-06-20):
|
|
903
|
+
Tier 2 fast path for CAStack with k_axis > 0, covering two
|
|
904
|
+
structurally-equivalent cases:
|
|
905
|
+
|
|
906
|
+
M.1 ax > k_axis parent INNER all post-K in output
|
|
907
|
+
M.2a ax < k_axis = ax + 1 parent INNER all post-K in output
|
|
908
|
+
(K directly after reduce axis)
|
|
909
|
+
|
|
910
|
+
In both cases parent INNER axes remain contig at the output
|
|
911
|
+
tail (j-stride 1 preserved -> SIMD tile write maintained),
|
|
912
|
+
parent PRE axes form a scattered OUTER (per-OUTER output base
|
|
913
|
+
computed via output stride table + carry-increment multi-index).
|
|
914
|
+
The k_axis == 0 branch above stays as the tightest fast path
|
|
915
|
+
(PRE contig in output, no scatter).
|
|
916
|
+
|
|
917
|
+
Out of this branch: M.2b (ax < k_axis - 1) where parent INNER
|
|
918
|
+
is split by K in output (= INNER tile write becomes scatter,
|
|
919
|
+
SIMD-impacting). Falls through to the generic init_l2
|
|
920
|
+
SLAB_AXES path. */
|
|
921
|
+
if ( naxes == 1
|
|
922
|
+
&& slab_axes[0] >= 1
|
|
923
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
924
|
+
&& ! ca_has_mask(ca)
|
|
925
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
926
|
+
&& ((CAStack *) ca)->k_axis > 0
|
|
927
|
+
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
928
|
+
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
929
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
930
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
931
|
+
int8_t __li_ax = slab_axes[0];
|
|
932
|
+
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
933
|
+
CArray *__li_p0 = __li_st->parents[0]; /* shape representative */
|
|
934
|
+
int8_t __li_pa = (__li_ax < __li_kax) ? __li_ax : (int8_t)(__li_ax - 1);
|
|
935
|
+
int8_t __li_kax_out = (__li_ax < __li_kax) ? (int8_t)(__li_kax - 1) : __li_kax;
|
|
936
|
+
ca_size_t __li_M = __li_p0->dim[__li_pa]; /* = ca->dim[__li_ax] for ax != kax */
|
|
937
|
+
ca_size_t __li_INNER = 1;
|
|
938
|
+
for ( int8_t __k = (int8_t)(__li_pa + 1); __k < __li_p0->ndim; __k++ ) __li_INNER *= __li_p0->dim[__k];
|
|
939
|
+
ca_size_t __li_OUTER = 1;
|
|
940
|
+
for ( int8_t __k = 0; __k < __li_pa; __k++ ) __li_OUTER *= __li_p0->dim[__k];
|
|
941
|
+
|
|
942
|
+
/* Bail to the generic path if any parent carries a mask. */
|
|
943
|
+
int __li_ok = 1;
|
|
944
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
945
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
if ( __li_ok && __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
949
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
950
|
+
CArray *co;
|
|
951
|
+
GetCArray(vout, co);
|
|
952
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
953
|
+
|
|
954
|
+
/* Output row-major stride table (length = ca->ndim - 1).
|
|
955
|
+
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
956
|
+
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
957
|
+
{
|
|
958
|
+
ca_size_t __li_s = 1;
|
|
959
|
+
for ( int8_t __vo = (int8_t)(ca->ndim - 2); __vo >= 0; __vo-- ) {
|
|
960
|
+
int8_t __vv = (__vo < __li_ax) ? __vo : (int8_t)(__vo + 1);
|
|
961
|
+
__li_out_stride[__vo] = __li_s;
|
|
962
|
+
__li_s *= ca->dim[__vv];
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
ca_size_t __li_k_stride = __li_out_stride[__li_kax_out];
|
|
966
|
+
|
|
967
|
+
/* Per-parent-PRE output strides + dims. PRE view position
|
|
968
|
+
vp = (pd < kax) ? pd : pd + 1. PRE output position
|
|
969
|
+
vo = (vp < ax) ? vp : vp - 1. For M.1 (ax > kax) all PRE
|
|
970
|
+
view pos < ax so vo = vp. For M.2a (ax < kax = ax+1) all
|
|
971
|
+
PRE pd < ax < kax so vp = pd and vo = pd. */
|
|
972
|
+
ca_size_t __li_pre_out_stride[CA_RANK_MAX];
|
|
973
|
+
ca_size_t __li_pre_dim[CA_RANK_MAX];
|
|
974
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
975
|
+
int8_t __vp = (__pd < __li_kax) ? __pd : (int8_t)(__pd + 1);
|
|
976
|
+
int8_t __vo = (__vp < __li_ax) ? __vp : (int8_t)(__vp - 1);
|
|
977
|
+
__li_pre_out_stride[__pd] = __li_out_stride[__vo];
|
|
978
|
+
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
boolean8_t __li_buf_acc[512];
|
|
982
|
+
|
|
983
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
984
|
+
const ca_size_t __li_TILE = 512;
|
|
985
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
986
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
987
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
988
|
+
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
989
|
+
|
|
990
|
+
ca_size_t __li_idx[CA_RANK_MAX];
|
|
991
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) __li_idx[__pd] = 0;
|
|
992
|
+
|
|
993
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
994
|
+
ca_size_t __li_obase = __li_k_base;
|
|
995
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
996
|
+
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
997
|
+
}
|
|
998
|
+
boolean8_t *__li_op_k = op + __li_obase;
|
|
999
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1000
|
+
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
1001
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1002
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1003
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1004
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1005
|
+
__li_buf_acc[__j] = (boolean8_t) (0);
|
|
1006
|
+
}
|
|
1007
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1008
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1009
|
+
_Pragma("omp simd")
|
|
1010
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1011
|
+
boolean8_t v = __li_row[__j];
|
|
1012
|
+
__li_buf_acc[__j] |= v;
|
|
1013
|
+
(void) v;
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1018
|
+
__li_op_k[__li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
1022
|
+
for ( int8_t __pd = (int8_t)(__li_pa - 1); __pd >= 0; __pd-- ) {
|
|
1023
|
+
if ( ++__li_idx[__pd] < __li_pre_dim[__pd] ) break;
|
|
1024
|
+
__li_idx[__pd] = 0;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
ca_detach(__li_st->parents[__kk]);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
return vout;
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/* M.2.0b (PROPOSAL_MKKERNEL_TIER2_K_AXIS_GEN, 2026-06-20):
|
|
1035
|
+
Case M.2b (ax < k_axis - 1): parent INNER axes are split by K
|
|
1036
|
+
in the output. Strategy C (effective OUTER expansion): treat
|
|
1037
|
+
INNER_pre_K dims (parent dims [pa+1..k_axis-1]) as part of an
|
|
1038
|
+
extended effective OUTER, and use effective INNER = parent dims
|
|
1039
|
+
[k_axis..pndim-1] (= INNER_post_K, contig at output tail).
|
|
1040
|
+
|
|
1041
|
+
L.7 core variant: M-step stride in parent = full parent INNER
|
|
1042
|
+
total (= Π pdim[pa+1..pndim-1]), separate from the tile loop
|
|
1043
|
+
bound (= eff_INNER). Inner read stride 1 in parent + output
|
|
1044
|
+
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
1045
|
+
tile reduce + write maintained.
|
|
1046
|
+
|
|
1047
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
1048
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
1049
|
+
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
1050
|
+
= 2.38x slow. Root cause is the multi-parent data layout
|
|
1051
|
+
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
1052
|
+
which costs DRAM/TLB/prefetcher efficiency irrespective of
|
|
1053
|
+
strategy. Strategy A (temp buf + scatter, explored rev1
|
|
1054
|
+
sparring round 1) was implemented and benched at 3783 us
|
|
1055
|
+
= within noise of Strategy C, confirming the bottleneck is
|
|
1056
|
+
structural, NOT cache-stride from this branch. Strategy C
|
|
1057
|
+
is shipped here (= simpler emit, no malloc) per round 2
|
|
1058
|
+
sparring decision; the 3.94x win vs generic fallback
|
|
1059
|
+
(.copy.sum at 14872 us) justifies landing despite missing
|
|
1060
|
+
eager-entity parity. Strategy A retained in PROPOSAL as
|
|
1061
|
+
future-optimization substrate (= multi-level tile-M + tile-
|
|
1062
|
+
INNER could reclaim DRAM efficiency if revisited).
|
|
1063
|
+
|
|
1064
|
+
Threshold eff_INNER >= 32 auto-bypasses at = -1 (k_axis =
|
|
1065
|
+
pndim -> eff_INNER = 1, fast path skipped, generic path
|
|
1066
|
+
handles correctness). */
|
|
1067
|
+
if ( naxes == 1
|
|
1068
|
+
&& slab_axes[0] >= 1
|
|
1069
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1070
|
+
&& ! ca_has_mask(ca)
|
|
1071
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1072
|
+
&& ((CAStack *) ca)->k_axis > 0
|
|
1073
|
+
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
1074
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1075
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
1076
|
+
int8_t __li_ax = slab_axes[0];
|
|
1077
|
+
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
1078
|
+
CArray *__li_p0 = __li_st->parents[0];
|
|
1079
|
+
int8_t __li_pndim = __li_p0->ndim;
|
|
1080
|
+
int8_t __li_pa = __li_ax; /* M.2b: ax < kax -> pa = ax */
|
|
1081
|
+
int8_t __li_kax_out = (int8_t)(__li_kax - 1);
|
|
1082
|
+
|
|
1083
|
+
ca_size_t __li_M = __li_p0->dim[__li_pa];
|
|
1084
|
+
ca_size_t __li_eff_INNER = 1; /* Π pdim[kax..pndim-1] = INNER_post_K */
|
|
1085
|
+
for ( int8_t __k = __li_kax; __k < __li_pndim; __k++ ) __li_eff_INNER *= __li_p0->dim[__k];
|
|
1086
|
+
ca_size_t __li_M_stride = 1; /* Π pdim[pa+1..pndim-1] = full parent INNER */
|
|
1087
|
+
for ( int8_t __k = (int8_t)(__li_pa + 1); __k < __li_pndim; __k++ ) __li_M_stride *= __li_p0->dim[__k];
|
|
1088
|
+
|
|
1089
|
+
/* Bail to the generic path if any parent carries a mask. */
|
|
1090
|
+
int __li_ok = 1;
|
|
1091
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1092
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
if ( __li_ok && __li_eff_INNER >= 32 && __li_M * __li_eff_INNER >= 1024 ) {
|
|
1096
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1097
|
+
CArray *co;
|
|
1098
|
+
GetCArray(vout, co);
|
|
1099
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1100
|
+
|
|
1101
|
+
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
1102
|
+
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
1103
|
+
{
|
|
1104
|
+
ca_size_t __li_s = 1;
|
|
1105
|
+
for ( int8_t __vo = (int8_t)(ca->ndim - 2); __vo >= 0; __vo-- ) {
|
|
1106
|
+
int8_t __vv = (__vo < __li_ax) ? __vo : (int8_t)(__vo + 1);
|
|
1107
|
+
__li_out_stride[__vo] = __li_s;
|
|
1108
|
+
__li_s *= ca->dim[__vv];
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
ca_size_t __li_k_stride = __li_out_stride[__li_kax_out];
|
|
1112
|
+
|
|
1113
|
+
/* Parent stride table (row-major). */
|
|
1114
|
+
ca_size_t __li_pstride[CA_RANK_MAX];
|
|
1115
|
+
{
|
|
1116
|
+
ca_size_t __li_pss = 1;
|
|
1117
|
+
for ( int8_t __k = (int8_t)(__li_pndim - 1); __k >= 0; __k-- ) {
|
|
1118
|
+
__li_pstride[__k] = __li_pss;
|
|
1119
|
+
__li_pss *= __li_p0->dim[__k];
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
/* Effective OUTER dim list: parent dims [0..kax-1] excluding pa.
|
|
1124
|
+
PRE = [0..pa-1] + INNER_pre_K = [pa+1..kax-1]. For each, store
|
|
1125
|
+
parent dim, parent stride, output stride. */
|
|
1126
|
+
int8_t __li_eo_n = 0;
|
|
1127
|
+
ca_size_t __li_eo_dim[CA_RANK_MAX];
|
|
1128
|
+
ca_size_t __li_eo_pstride[CA_RANK_MAX];
|
|
1129
|
+
ca_size_t __li_eo_ostride[CA_RANK_MAX];
|
|
1130
|
+
for ( int8_t __d = 0; __d < __li_kax; __d++ ) {
|
|
1131
|
+
if ( __d == __li_pa ) continue;
|
|
1132
|
+
int8_t __vp = __d; /* M.2b: all eff OUTER d < kax -> vp = d */
|
|
1133
|
+
int8_t __vo = (__vp < __li_ax) ? __vp : (int8_t)(__vp - 1);
|
|
1134
|
+
__li_eo_dim[__li_eo_n] = __li_p0->dim[__d];
|
|
1135
|
+
__li_eo_pstride[__li_eo_n] = __li_pstride[__d];
|
|
1136
|
+
__li_eo_ostride[__li_eo_n] = __li_out_stride[__vo];
|
|
1137
|
+
__li_eo_n++;
|
|
1138
|
+
}
|
|
1139
|
+
ca_size_t __li_OUTER = 1;
|
|
1140
|
+
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
1141
|
+
|
|
1142
|
+
boolean8_t __li_buf_acc[512];
|
|
1143
|
+
|
|
1144
|
+
const ca_size_t __li_TILE = 512;
|
|
1145
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1146
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1147
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
1148
|
+
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1149
|
+
|
|
1150
|
+
ca_size_t __li_idx[CA_RANK_MAX];
|
|
1151
|
+
for ( int8_t __i = 0; __i < __li_eo_n; __i++ ) __li_idx[__i] = 0;
|
|
1152
|
+
|
|
1153
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1154
|
+
ca_size_t __li_pbase = 0;
|
|
1155
|
+
ca_size_t __li_obase = __li_k_base;
|
|
1156
|
+
for ( int8_t __i = 0; __i < __li_eo_n; __i++ ) {
|
|
1157
|
+
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
1158
|
+
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
1159
|
+
}
|
|
1160
|
+
boolean8_t *__li_op_k = op + __li_obase;
|
|
1161
|
+
const boolean8_t *__li_plane = p_root + __li_pbase;
|
|
1162
|
+
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
1163
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
1164
|
+
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
1165
|
+
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
1166
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1167
|
+
__li_buf_acc[__j] = (boolean8_t) (0);
|
|
1168
|
+
}
|
|
1169
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1170
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
1171
|
+
_Pragma("omp simd")
|
|
1172
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1173
|
+
boolean8_t v = __li_row[__j];
|
|
1174
|
+
__li_buf_acc[__j] |= v;
|
|
1175
|
+
(void) v;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1180
|
+
__li_op_k[__li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
/* Carry-increment effective OUTER multi-index. */
|
|
1184
|
+
for ( int8_t __i = (int8_t)(__li_eo_n - 1); __i >= 0; __i-- ) {
|
|
1185
|
+
if ( ++__li_idx[__i] < __li_eo_dim[__i] ) break;
|
|
1186
|
+
__li_idx[__i] = 0;
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
ca_detach(__li_st->parents[__kk]);
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
return vout;
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
if ( naxes == 1
|
|
1196
|
+
&& ca->ndim >= 2
|
|
1197
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
1198
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
1199
|
+
&& ca->dim[ca->ndim - 1] <= 16
|
|
1200
|
+
&& ! ca_has_mask(ca)
|
|
1201
|
+
&& ca_attach_is_alias(ca)
|
|
1202
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
1203
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
1204
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
1205
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1206
|
+
CArray *co;
|
|
1207
|
+
GetCArray(vout, co);
|
|
1208
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1209
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1210
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
1211
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
1212
|
+
const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
1213
|
+
boolean8_t acc = (boolean8_t) (0);
|
|
1214
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
1215
|
+
boolean8_t v = __sf_row[__j];
|
|
1216
|
+
acc |= v;
|
|
1217
|
+
(void) v;
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
op[__sf_o] = (boolean8_t) (acc);
|
|
1221
|
+
}
|
|
1222
|
+
ca_detach(ca);
|
|
1223
|
+
return vout;
|
|
1224
|
+
}
|
|
226
1225
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
227
1226
|
CArray *co;
|
|
228
1227
|
GetCArray(vout, co);
|
|
229
1228
|
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
230
|
-
ca_iter_state st;
|
|
231
1229
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
232
1230
|
slab_axes, naxes, 0);
|
|
233
1231
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -239,13 +1237,27 @@ boolean8_t *m;
|
|
|
239
1237
|
ca_size_t out_i = 0;
|
|
240
1238
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
241
1239
|
boolean8_t acc;
|
|
242
|
-
|
|
1240
|
+
CA_SLAB_REDUCE_T_SETTLE(boolean8_t, st, p, m, acc, 0, acc |= v, acc);
|
|
243
1241
|
op[out_i++] = (boolean8_t) (acc);
|
|
244
1242
|
}
|
|
245
1243
|
ca_iter_state_finish(&st);
|
|
246
1244
|
if ( naxes == ca->ndim && !keep_axis ) return BOOL2VAL(op[0]);
|
|
247
1245
|
return vout;
|
|
248
|
-
|
|
1246
|
+
#undef st
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
static VALUE
|
|
1250
|
+
any_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis)
|
|
1251
|
+
{
|
|
1252
|
+
any_ki_native_bool_frame fr;
|
|
1253
|
+
fr.self = self;
|
|
1254
|
+
fr.ca = ca;
|
|
1255
|
+
fr.slab_axes = slab_axes;
|
|
1256
|
+
fr.naxes = naxes;
|
|
1257
|
+
fr.keep_axis = keep_axis;
|
|
1258
|
+
return ca_iter_ensure(1, fr.states, any_ki_native_bool_body, (VALUE) &fr);
|
|
1259
|
+
}
|
|
1260
|
+
|
|
249
1261
|
|
|
250
1262
|
VALUE
|
|
251
1263
|
rb_ca_any_ki (int argc, VALUE *argv, VALUE self)
|
|
@@ -327,24 +1339,516 @@ none_ki_stream_bool (VALUE arg)
|
|
|
327
1339
|
return BOOL2VAL((boolean8_t) (acc));
|
|
328
1340
|
}
|
|
329
1341
|
|
|
1342
|
+
/* The walks can raise part way (a Ruby call per cell in the object
|
|
1343
|
+
lane, an allocation in any lane). Their states live in the caller's
|
|
1344
|
+
frame, and ca_iter_ensure finishes them however the body leaves. */
|
|
1345
|
+
typedef struct {
|
|
1346
|
+
ca_iter_state states[1];
|
|
1347
|
+
VALUE self;
|
|
1348
|
+
CArray *ca;
|
|
1349
|
+
int8_t *slab_axes;
|
|
1350
|
+
int8_t naxes;
|
|
1351
|
+
int keep_axis;
|
|
1352
|
+
} none_ki_native_bool_frame;
|
|
1353
|
+
|
|
330
1354
|
static VALUE
|
|
331
|
-
|
|
1355
|
+
none_ki_native_bool_body (VALUE arg)
|
|
332
1356
|
{
|
|
1357
|
+
none_ki_native_bool_frame *fr = (none_ki_native_bool_frame *) arg;
|
|
1358
|
+
VALUE self = fr->self;
|
|
1359
|
+
(void) self;
|
|
1360
|
+
CArray *ca = fr->ca;
|
|
1361
|
+
(void) ca;
|
|
1362
|
+
int8_t *slab_axes = fr->slab_axes;
|
|
1363
|
+
(void) slab_axes;
|
|
1364
|
+
int8_t naxes = fr->naxes;
|
|
1365
|
+
(void) naxes;
|
|
1366
|
+
int keep_axis = fr->keep_axis;
|
|
1367
|
+
(void) keep_axis;
|
|
1368
|
+
#define st (fr->states[0])
|
|
333
1369
|
if ( naxes == ca->ndim && ca->ndim >= 1 && !keep_axis &&
|
|
334
1370
|
ca->elements > 0 &&
|
|
335
1371
|
ca_is_lazy_view(ca) && ! ca_has_mask(ca) &&
|
|
336
|
-
|
|
1372
|
+
ca_fast_path(CA_SC_LAZY_STREAMING_REDUCE) ) {
|
|
337
1373
|
none_ki_stream_bool_t __ctx;
|
|
338
1374
|
__ctx.chunk = NULL;
|
|
339
1375
|
__ctx.ca = ca;
|
|
340
1376
|
return ca_lazy_arena_protect(none_ki_stream_bool, ca_lazy_arena_release_held,
|
|
341
1377
|
(VALUE) &__ctx);
|
|
342
1378
|
}
|
|
1379
|
+
/* L.1 / L.7: loop-interchange fast path with inner-tiling
|
|
1380
|
+
(PROPOSAL_REDUCTION_LOOP_INTERCHANGE).
|
|
1381
|
+
MEMO_REDUCTION_FASTPATH_ENTITY_ONLY_GUARD Tier 1: extend from
|
|
1382
|
+
entity-only to any contig row-major buffer. ca_attach_is_alias
|
|
1383
|
+
is true for CA_OBJ_ARRAY and for CAStride-family views whose
|
|
1384
|
+
composed strides are row-major contiguous (CARefer reshape,
|
|
1385
|
+
contig CABlock, CAFarray, alias CAStride, ...); ca_attach then
|
|
1386
|
+
exposes ca->ptr as that buffer at O(1) cost (no materialise). */
|
|
1387
|
+
if ( naxes == 1
|
|
1388
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1389
|
+
&& ! ca_has_mask(ca)
|
|
1390
|
+
&& ca_attach_is_alias(ca)
|
|
1391
|
+
&& ca_fast_path(CA_SC_LOOP_INTERCHANGE) ) {
|
|
1392
|
+
int8_t __li_ax = slab_axes[0];
|
|
1393
|
+
ca_size_t __li_M = ca->dim[__li_ax];
|
|
1394
|
+
ca_size_t __li_INNER = 1;
|
|
1395
|
+
for ( int8_t __k = (int8_t)(__li_ax + 1); __k < ca->ndim; __k++ ) __li_INNER *= ca->dim[__k];
|
|
1396
|
+
ca_size_t __li_OUTER = 1;
|
|
1397
|
+
for ( int8_t __k = 0; __k < __li_ax; __k++ ) __li_OUTER *= ca->dim[__k];
|
|
1398
|
+
|
|
1399
|
+
/* Threshold gate: skip when too small to amortise alloc + 2-pass overhead. */
|
|
1400
|
+
if ( __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
1401
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1402
|
+
CArray *co;
|
|
1403
|
+
GetCArray(vout, co);
|
|
1404
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1405
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1406
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
1407
|
+
|
|
1408
|
+
/* Per-buffer stack accumulators (one per :plus state, sized
|
|
1409
|
+
to L7 TILE = 512 cells regardless of INNER). */
|
|
1410
|
+
boolean8_t __li_buf_acc[512];
|
|
1411
|
+
|
|
1412
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1413
|
+
const ca_size_t __li_TILE = 512;
|
|
1414
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1415
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1416
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1417
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1418
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1419
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1420
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
1421
|
+
}
|
|
1422
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1423
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1424
|
+
_Pragma("omp simd")
|
|
1425
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1426
|
+
boolean8_t v = __li_row[__j];
|
|
1427
|
+
__li_buf_acc[__j] &= !v;
|
|
1428
|
+
(void) v;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1433
|
+
op[__li_o * __li_INNER + __li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
ca_detach(ca);
|
|
1438
|
+
|
|
1439
|
+
if ( naxes == ca->ndim && !keep_axis ) return BOOL2VAL(op[0]); /* unreachable: ax != innermost && naxes == 1 < ndim */
|
|
1440
|
+
return vout;
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/* MEMO_REDUCTION_FASTPATH_ENTITY_ONLY_GUARD Tier 2: CAStack source.
|
|
1445
|
+
view.reduce(axis: ax>=1) is exactly K independent per-parent
|
|
1446
|
+
reduces over (ax - 1) of each contiguous parent; output row k
|
|
1447
|
+
= parent k's reduced plane. Run the L.7 core per parent (each
|
|
1448
|
+
attach is O(1) for entity parents), no whole-view materialise.
|
|
1449
|
+
|
|
1450
|
+
K.3 (PROPOSAL_CASTACK_K_AXIS, 2026-06-20): gated to k_axis == 0
|
|
1451
|
+
because the per-parent output layout in dst assumes K is the
|
|
1452
|
+
outermost output axis (= each parent contributes a contiguous
|
|
1453
|
+
block of parent_OUTER * parent_INNER cells). For k_axis != 0
|
|
1454
|
+
the parents' outputs are interleaved in dst -- generalising
|
|
1455
|
+
the output stride is doable but out of K.3 scope. The
|
|
1456
|
+
dispatch falls through to ca_iter_state_init_l2 SLAB_AXES
|
|
1457
|
+
which engages the K.3-generalised tile cache. */
|
|
1458
|
+
if ( naxes == 1
|
|
1459
|
+
&& slab_axes[0] >= 1
|
|
1460
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1461
|
+
&& ! ca_has_mask(ca)
|
|
1462
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1463
|
+
&& ((CAStack *) ca)->k_axis == 0
|
|
1464
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1465
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
1466
|
+
int8_t __li_ax = slab_axes[0];
|
|
1467
|
+
ca_size_t __li_M = ca->dim[__li_ax];
|
|
1468
|
+
ca_size_t __li_INNER = 1;
|
|
1469
|
+
for ( int8_t __k = (int8_t)(__li_ax + 1); __k < ca->ndim; __k++ ) __li_INNER *= ca->dim[__k];
|
|
1470
|
+
ca_size_t __li_OUTER = 1; /* parent outer = dims 1..ax-1 (excludes K = axis 0) */
|
|
1471
|
+
for ( int8_t __k = 1; __k < __li_ax; __k++ ) __li_OUTER *= ca->dim[__k];
|
|
1472
|
+
|
|
1473
|
+
/* Bail to the generic path if any parent carries a mask (the
|
|
1474
|
+
core reads parent->ptr without mask awareness). */
|
|
1475
|
+
int __li_ok = 1;
|
|
1476
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1477
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
if ( __li_ok && __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
1481
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1482
|
+
CArray *co;
|
|
1483
|
+
GetCArray(vout, co);
|
|
1484
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1485
|
+
|
|
1486
|
+
boolean8_t __li_buf_acc[512];
|
|
1487
|
+
|
|
1488
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1489
|
+
ca_size_t __li_parent_out = __li_OUTER * __li_INNER;
|
|
1490
|
+
const ca_size_t __li_TILE = 512;
|
|
1491
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1492
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1493
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
1494
|
+
boolean8_t *__li_op_k = op + (ca_size_t) __kk * __li_parent_out;
|
|
1495
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1496
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1497
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1498
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1499
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1500
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1501
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
1502
|
+
}
|
|
1503
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1504
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1505
|
+
_Pragma("omp simd")
|
|
1506
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1507
|
+
boolean8_t v = __li_row[__j];
|
|
1508
|
+
__li_buf_acc[__j] &= !v;
|
|
1509
|
+
(void) v;
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1514
|
+
__li_op_k[__li_o * __li_INNER + __li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
ca_detach(__li_st->parents[__kk]);
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
return vout;
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
/* M.1.0 + M.2.0a (PROPOSAL_MKKERNEL_TIER2_K_AXIS_GEN, 2026-06-20):
|
|
1526
|
+
Tier 2 fast path for CAStack with k_axis > 0, covering two
|
|
1527
|
+
structurally-equivalent cases:
|
|
1528
|
+
|
|
1529
|
+
M.1 ax > k_axis parent INNER all post-K in output
|
|
1530
|
+
M.2a ax < k_axis = ax + 1 parent INNER all post-K in output
|
|
1531
|
+
(K directly after reduce axis)
|
|
1532
|
+
|
|
1533
|
+
In both cases parent INNER axes remain contig at the output
|
|
1534
|
+
tail (j-stride 1 preserved -> SIMD tile write maintained),
|
|
1535
|
+
parent PRE axes form a scattered OUTER (per-OUTER output base
|
|
1536
|
+
computed via output stride table + carry-increment multi-index).
|
|
1537
|
+
The k_axis == 0 branch above stays as the tightest fast path
|
|
1538
|
+
(PRE contig in output, no scatter).
|
|
1539
|
+
|
|
1540
|
+
Out of this branch: M.2b (ax < k_axis - 1) where parent INNER
|
|
1541
|
+
is split by K in output (= INNER tile write becomes scatter,
|
|
1542
|
+
SIMD-impacting). Falls through to the generic init_l2
|
|
1543
|
+
SLAB_AXES path. */
|
|
1544
|
+
if ( naxes == 1
|
|
1545
|
+
&& slab_axes[0] >= 1
|
|
1546
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1547
|
+
&& ! ca_has_mask(ca)
|
|
1548
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1549
|
+
&& ((CAStack *) ca)->k_axis > 0
|
|
1550
|
+
&& ( slab_axes[0] > ((CAStack *) ca)->k_axis
|
|
1551
|
+
|| slab_axes[0] + 1 == ((CAStack *) ca)->k_axis )
|
|
1552
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1553
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
1554
|
+
int8_t __li_ax = slab_axes[0];
|
|
1555
|
+
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
1556
|
+
CArray *__li_p0 = __li_st->parents[0]; /* shape representative */
|
|
1557
|
+
int8_t __li_pa = (__li_ax < __li_kax) ? __li_ax : (int8_t)(__li_ax - 1);
|
|
1558
|
+
int8_t __li_kax_out = (__li_ax < __li_kax) ? (int8_t)(__li_kax - 1) : __li_kax;
|
|
1559
|
+
ca_size_t __li_M = __li_p0->dim[__li_pa]; /* = ca->dim[__li_ax] for ax != kax */
|
|
1560
|
+
ca_size_t __li_INNER = 1;
|
|
1561
|
+
for ( int8_t __k = (int8_t)(__li_pa + 1); __k < __li_p0->ndim; __k++ ) __li_INNER *= __li_p0->dim[__k];
|
|
1562
|
+
ca_size_t __li_OUTER = 1;
|
|
1563
|
+
for ( int8_t __k = 0; __k < __li_pa; __k++ ) __li_OUTER *= __li_p0->dim[__k];
|
|
1564
|
+
|
|
1565
|
+
/* Bail to the generic path if any parent carries a mask. */
|
|
1566
|
+
int __li_ok = 1;
|
|
1567
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1568
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
if ( __li_ok && __li_INNER >= 64 && __li_M * __li_INNER >= 1024 ) {
|
|
1572
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1573
|
+
CArray *co;
|
|
1574
|
+
GetCArray(vout, co);
|
|
1575
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1576
|
+
|
|
1577
|
+
/* Output row-major stride table (length = ca->ndim - 1).
|
|
1578
|
+
Output position vo maps to view position vv = (vo < ax) ? vo : vo + 1. */
|
|
1579
|
+
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
1580
|
+
{
|
|
1581
|
+
ca_size_t __li_s = 1;
|
|
1582
|
+
for ( int8_t __vo = (int8_t)(ca->ndim - 2); __vo >= 0; __vo-- ) {
|
|
1583
|
+
int8_t __vv = (__vo < __li_ax) ? __vo : (int8_t)(__vo + 1);
|
|
1584
|
+
__li_out_stride[__vo] = __li_s;
|
|
1585
|
+
__li_s *= ca->dim[__vv];
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
ca_size_t __li_k_stride = __li_out_stride[__li_kax_out];
|
|
1589
|
+
|
|
1590
|
+
/* Per-parent-PRE output strides + dims. PRE view position
|
|
1591
|
+
vp = (pd < kax) ? pd : pd + 1. PRE output position
|
|
1592
|
+
vo = (vp < ax) ? vp : vp - 1. For M.1 (ax > kax) all PRE
|
|
1593
|
+
view pos < ax so vo = vp. For M.2a (ax < kax = ax+1) all
|
|
1594
|
+
PRE pd < ax < kax so vp = pd and vo = pd. */
|
|
1595
|
+
ca_size_t __li_pre_out_stride[CA_RANK_MAX];
|
|
1596
|
+
ca_size_t __li_pre_dim[CA_RANK_MAX];
|
|
1597
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
1598
|
+
int8_t __vp = (__pd < __li_kax) ? __pd : (int8_t)(__pd + 1);
|
|
1599
|
+
int8_t __vo = (__vp < __li_ax) ? __vp : (int8_t)(__vp - 1);
|
|
1600
|
+
__li_pre_out_stride[__pd] = __li_out_stride[__vo];
|
|
1601
|
+
__li_pre_dim[__pd] = __li_p0->dim[__pd];
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
boolean8_t __li_buf_acc[512];
|
|
1605
|
+
|
|
1606
|
+
ca_size_t __li_plane_elems = __li_M * __li_INNER;
|
|
1607
|
+
const ca_size_t __li_TILE = 512;
|
|
1608
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1609
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1610
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
1611
|
+
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1612
|
+
|
|
1613
|
+
ca_size_t __li_idx[CA_RANK_MAX];
|
|
1614
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) __li_idx[__pd] = 0;
|
|
1615
|
+
|
|
1616
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1617
|
+
ca_size_t __li_obase = __li_k_base;
|
|
1618
|
+
for ( int8_t __pd = 0; __pd < __li_pa; __pd++ ) {
|
|
1619
|
+
__li_obase += __li_idx[__pd] * __li_pre_out_stride[__pd];
|
|
1620
|
+
}
|
|
1621
|
+
boolean8_t *__li_op_k = op + __li_obase;
|
|
1622
|
+
const boolean8_t *__li_plane = p_root + __li_o * __li_plane_elems;
|
|
1623
|
+
/* Inline single-OUTER core: j-stride 1 in __li_op_k preserved. */
|
|
1624
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_INNER; __li_tile += __li_TILE ) {
|
|
1625
|
+
ca_size_t __li_tile_len = (__li_INNER - __li_tile < __li_TILE)
|
|
1626
|
+
? (__li_INNER - __li_tile) : __li_TILE;
|
|
1627
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1628
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
1629
|
+
}
|
|
1630
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1631
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_INNER + __li_tile;
|
|
1632
|
+
_Pragma("omp simd")
|
|
1633
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1634
|
+
boolean8_t v = __li_row[__j];
|
|
1635
|
+
__li_buf_acc[__j] &= !v;
|
|
1636
|
+
(void) v;
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1641
|
+
__li_op_k[__li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
/* Carry-increment OUTER multi-index (row-major over parent PRE dims). */
|
|
1645
|
+
for ( int8_t __pd = (int8_t)(__li_pa - 1); __pd >= 0; __pd-- ) {
|
|
1646
|
+
if ( ++__li_idx[__pd] < __li_pre_dim[__pd] ) break;
|
|
1647
|
+
__li_idx[__pd] = 0;
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
ca_detach(__li_st->parents[__kk]);
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
return vout;
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
/* M.2.0b (PROPOSAL_MKKERNEL_TIER2_K_AXIS_GEN, 2026-06-20):
|
|
1658
|
+
Case M.2b (ax < k_axis - 1): parent INNER axes are split by K
|
|
1659
|
+
in the output. Strategy C (effective OUTER expansion): treat
|
|
1660
|
+
INNER_pre_K dims (parent dims [pa+1..k_axis-1]) as part of an
|
|
1661
|
+
extended effective OUTER, and use effective INNER = parent dims
|
|
1662
|
+
[k_axis..pndim-1] (= INNER_post_K, contig at output tail).
|
|
1663
|
+
|
|
1664
|
+
L.7 core variant: M-step stride in parent = full parent INNER
|
|
1665
|
+
total (= Π pdim[pa+1..pndim-1]), separate from the tile loop
|
|
1666
|
+
bound (= eff_INNER). Inner read stride 1 in parent + output
|
|
1667
|
+
write stride 1 (= contig output tail) both preserved -> SIMD
|
|
1668
|
+
tile reduce + write maintained.
|
|
1669
|
+
|
|
1670
|
+
Performance characteristic (deliver-via-view -- the surface
|
|
1671
|
+
prioritises delivering the cells over avoiding a copy): bench (b2 pattern, M=200, K=5, eff_INNER
|
|
1672
|
+
=360, INNER_pre_K=16) yields 3674 us vs eager-entity 1545 us
|
|
1673
|
+
= 2.38x slow. Root cause is the multi-parent data layout
|
|
1674
|
+
(= 5 separate 9 MB regions instead of one contig 46 MB),
|
|
1675
|
+
which costs DRAM/TLB/prefetcher efficiency irrespective of
|
|
1676
|
+
strategy. Strategy A (temp buf + scatter, explored rev1
|
|
1677
|
+
sparring round 1) was implemented and benched at 3783 us
|
|
1678
|
+
= within noise of Strategy C, confirming the bottleneck is
|
|
1679
|
+
structural, NOT cache-stride from this branch. Strategy C
|
|
1680
|
+
is shipped here (= simpler emit, no malloc) per round 2
|
|
1681
|
+
sparring decision; the 3.94x win vs generic fallback
|
|
1682
|
+
(.copy.sum at 14872 us) justifies landing despite missing
|
|
1683
|
+
eager-entity parity. Strategy A retained in PROPOSAL as
|
|
1684
|
+
future-optimization substrate (= multi-level tile-M + tile-
|
|
1685
|
+
INNER could reclaim DRAM efficiency if revisited).
|
|
1686
|
+
|
|
1687
|
+
Threshold eff_INNER >= 32 auto-bypasses at = -1 (k_axis =
|
|
1688
|
+
pndim -> eff_INNER = 1, fast path skipped, generic path
|
|
1689
|
+
handles correctness). */
|
|
1690
|
+
if ( naxes == 1
|
|
1691
|
+
&& slab_axes[0] >= 1
|
|
1692
|
+
&& slab_axes[0] != (int8_t)(ca->ndim - 1)
|
|
1693
|
+
&& ! ca_has_mask(ca)
|
|
1694
|
+
&& ca_func[ca->obj_type].attach == ca_stack_func.attach
|
|
1695
|
+
&& ((CAStack *) ca)->k_axis > 0
|
|
1696
|
+
&& slab_axes[0] + 1 < ((CAStack *) ca)->k_axis
|
|
1697
|
+
&& ca_fast_path(CA_SC_STACK_LOOP_INTERCHANGE) ) {
|
|
1698
|
+
CAStack *__li_st = (CAStack *) ca;
|
|
1699
|
+
int8_t __li_ax = slab_axes[0];
|
|
1700
|
+
int8_t __li_kax = (int8_t) __li_st->k_axis;
|
|
1701
|
+
CArray *__li_p0 = __li_st->parents[0];
|
|
1702
|
+
int8_t __li_pndim = __li_p0->ndim;
|
|
1703
|
+
int8_t __li_pa = __li_ax; /* M.2b: ax < kax -> pa = ax */
|
|
1704
|
+
int8_t __li_kax_out = (int8_t)(__li_kax - 1);
|
|
1705
|
+
|
|
1706
|
+
ca_size_t __li_M = __li_p0->dim[__li_pa];
|
|
1707
|
+
ca_size_t __li_eff_INNER = 1; /* Π pdim[kax..pndim-1] = INNER_post_K */
|
|
1708
|
+
for ( int8_t __k = __li_kax; __k < __li_pndim; __k++ ) __li_eff_INNER *= __li_p0->dim[__k];
|
|
1709
|
+
ca_size_t __li_M_stride = 1; /* Π pdim[pa+1..pndim-1] = full parent INNER */
|
|
1710
|
+
for ( int8_t __k = (int8_t)(__li_pa + 1); __k < __li_pndim; __k++ ) __li_M_stride *= __li_p0->dim[__k];
|
|
1711
|
+
|
|
1712
|
+
/* Bail to the generic path if any parent carries a mask. */
|
|
1713
|
+
int __li_ok = 1;
|
|
1714
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1715
|
+
if ( ca_has_mask(__li_st->parents[__kk]) ) { __li_ok = 0; break; }
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
if ( __li_ok && __li_eff_INNER >= 32 && __li_M * __li_eff_INNER >= 1024 ) {
|
|
1719
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1720
|
+
CArray *co;
|
|
1721
|
+
GetCArray(vout, co);
|
|
1722
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1723
|
+
|
|
1724
|
+
/* Output row-major stride table (length = ca->ndim - 1). */
|
|
1725
|
+
ca_size_t __li_out_stride[CA_RANK_MAX];
|
|
1726
|
+
{
|
|
1727
|
+
ca_size_t __li_s = 1;
|
|
1728
|
+
for ( int8_t __vo = (int8_t)(ca->ndim - 2); __vo >= 0; __vo-- ) {
|
|
1729
|
+
int8_t __vv = (__vo < __li_ax) ? __vo : (int8_t)(__vo + 1);
|
|
1730
|
+
__li_out_stride[__vo] = __li_s;
|
|
1731
|
+
__li_s *= ca->dim[__vv];
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
ca_size_t __li_k_stride = __li_out_stride[__li_kax_out];
|
|
1735
|
+
|
|
1736
|
+
/* Parent stride table (row-major). */
|
|
1737
|
+
ca_size_t __li_pstride[CA_RANK_MAX];
|
|
1738
|
+
{
|
|
1739
|
+
ca_size_t __li_pss = 1;
|
|
1740
|
+
for ( int8_t __k = (int8_t)(__li_pndim - 1); __k >= 0; __k-- ) {
|
|
1741
|
+
__li_pstride[__k] = __li_pss;
|
|
1742
|
+
__li_pss *= __li_p0->dim[__k];
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
/* Effective OUTER dim list: parent dims [0..kax-1] excluding pa.
|
|
1747
|
+
PRE = [0..pa-1] + INNER_pre_K = [pa+1..kax-1]. For each, store
|
|
1748
|
+
parent dim, parent stride, output stride. */
|
|
1749
|
+
int8_t __li_eo_n = 0;
|
|
1750
|
+
ca_size_t __li_eo_dim[CA_RANK_MAX];
|
|
1751
|
+
ca_size_t __li_eo_pstride[CA_RANK_MAX];
|
|
1752
|
+
ca_size_t __li_eo_ostride[CA_RANK_MAX];
|
|
1753
|
+
for ( int8_t __d = 0; __d < __li_kax; __d++ ) {
|
|
1754
|
+
if ( __d == __li_pa ) continue;
|
|
1755
|
+
int8_t __vp = __d; /* M.2b: all eff OUTER d < kax -> vp = d */
|
|
1756
|
+
int8_t __vo = (__vp < __li_ax) ? __vp : (int8_t)(__vp - 1);
|
|
1757
|
+
__li_eo_dim[__li_eo_n] = __li_p0->dim[__d];
|
|
1758
|
+
__li_eo_pstride[__li_eo_n] = __li_pstride[__d];
|
|
1759
|
+
__li_eo_ostride[__li_eo_n] = __li_out_stride[__vo];
|
|
1760
|
+
__li_eo_n++;
|
|
1761
|
+
}
|
|
1762
|
+
ca_size_t __li_OUTER = 1;
|
|
1763
|
+
for ( int8_t __k = 0; __k < __li_eo_n; __k++ ) __li_OUTER *= __li_eo_dim[__k];
|
|
1764
|
+
|
|
1765
|
+
boolean8_t __li_buf_acc[512];
|
|
1766
|
+
|
|
1767
|
+
const ca_size_t __li_TILE = 512;
|
|
1768
|
+
for ( int32_t __kk = 0; __kk < __li_st->n_parents; __kk++ ) {
|
|
1769
|
+
ca_attach(__li_st->parents[__kk]); /* window: a parent may run Ruby to fill itself (a CAObject's copy_data); a raising attach holds nothing and the earlier parents are already detached, and the numeric loop after it does not raise */
|
|
1770
|
+
const boolean8_t *p_root = (const boolean8_t *) __li_st->parents[__kk]->ptr;
|
|
1771
|
+
ca_size_t __li_k_base = (ca_size_t) __kk * __li_k_stride;
|
|
1772
|
+
|
|
1773
|
+
ca_size_t __li_idx[CA_RANK_MAX];
|
|
1774
|
+
for ( int8_t __i = 0; __i < __li_eo_n; __i++ ) __li_idx[__i] = 0;
|
|
1775
|
+
|
|
1776
|
+
for ( ca_size_t __li_o = 0; __li_o < __li_OUTER; __li_o++ ) {
|
|
1777
|
+
ca_size_t __li_pbase = 0;
|
|
1778
|
+
ca_size_t __li_obase = __li_k_base;
|
|
1779
|
+
for ( int8_t __i = 0; __i < __li_eo_n; __i++ ) {
|
|
1780
|
+
__li_pbase += __li_idx[__i] * __li_eo_pstride[__i];
|
|
1781
|
+
__li_obase += __li_idx[__i] * __li_eo_ostride[__i];
|
|
1782
|
+
}
|
|
1783
|
+
boolean8_t *__li_op_k = op + __li_obase;
|
|
1784
|
+
const boolean8_t *__li_plane = p_root + __li_pbase;
|
|
1785
|
+
/* L.7 core variant: M-step uses __li_M_stride, tile loop uses __li_eff_INNER. */
|
|
1786
|
+
for ( ca_size_t __li_tile = 0; __li_tile < __li_eff_INNER; __li_tile += __li_TILE ) {
|
|
1787
|
+
ca_size_t __li_tile_len = (__li_eff_INNER - __li_tile < __li_TILE)
|
|
1788
|
+
? (__li_eff_INNER - __li_tile) : __li_TILE;
|
|
1789
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1790
|
+
__li_buf_acc[__j] = (boolean8_t) (1);
|
|
1791
|
+
}
|
|
1792
|
+
for ( ca_size_t __li_i = 0; __li_i < __li_M; __li_i++ ) {
|
|
1793
|
+
const boolean8_t *__li_row = __li_plane + __li_i * __li_M_stride + __li_tile;
|
|
1794
|
+
_Pragma("omp simd")
|
|
1795
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1796
|
+
boolean8_t v = __li_row[__j];
|
|
1797
|
+
__li_buf_acc[__j] &= !v;
|
|
1798
|
+
(void) v;
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
for ( ca_size_t __j = 0; __j < __li_tile_len; __j++ ) {
|
|
1803
|
+
__li_op_k[__li_tile + __j] = (boolean8_t) (__li_buf_acc[__j]);
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
/* Carry-increment effective OUTER multi-index. */
|
|
1807
|
+
for ( int8_t __i = (int8_t)(__li_eo_n - 1); __i >= 0; __i-- ) {
|
|
1808
|
+
if ( ++__li_idx[__i] < __li_eo_dim[__i] ) break;
|
|
1809
|
+
__li_idx[__i] = 0;
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
ca_detach(__li_st->parents[__kk]);
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
return vout;
|
|
1816
|
+
}
|
|
1817
|
+
}
|
|
1818
|
+
if ( naxes == 1
|
|
1819
|
+
&& ca->ndim >= 2
|
|
1820
|
+
&& slab_axes[0] == (int8_t)(ca->ndim - 1)
|
|
1821
|
+
&& ca->dim[ca->ndim - 1] >= 1
|
|
1822
|
+
&& ca->dim[ca->ndim - 1] <= 16
|
|
1823
|
+
&& ! ca_has_mask(ca)
|
|
1824
|
+
&& ca_attach_is_alias(ca)
|
|
1825
|
+
&& ca_fast_path(CA_SC_SHORT_FIBER_REDUCE) ) {
|
|
1826
|
+
ca_size_t __sf_M = ca->dim[ca->ndim - 1];
|
|
1827
|
+
ca_size_t __sf_OUTER = ca->elements / __sf_M;
|
|
1828
|
+
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
1829
|
+
CArray *co;
|
|
1830
|
+
GetCArray(vout, co);
|
|
1831
|
+
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
1832
|
+
ca_attach(ca); /* window: numeric lanes only, nothing raises inside; O(1) alias attach */
|
|
1833
|
+
const boolean8_t *p_root = (const boolean8_t *) ca->ptr;
|
|
1834
|
+
for ( ca_size_t __sf_o = 0; __sf_o < __sf_OUTER; __sf_o++ ) {
|
|
1835
|
+
const boolean8_t *__sf_row = p_root + __sf_o * __sf_M;
|
|
1836
|
+
boolean8_t acc = (boolean8_t) (1);
|
|
1837
|
+
for ( ca_size_t __j = 0; __j < __sf_M; __j++ ) {
|
|
1838
|
+
boolean8_t v = __sf_row[__j];
|
|
1839
|
+
acc &= !v;
|
|
1840
|
+
(void) v;
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
op[__sf_o] = (boolean8_t) (acc);
|
|
1844
|
+
}
|
|
1845
|
+
ca_detach(ca);
|
|
1846
|
+
return vout;
|
|
1847
|
+
}
|
|
343
1848
|
VALUE vout = rb_ca_new_reduced(self, slab_axes, naxes, CA_BOOLEAN, keep_axis);
|
|
344
1849
|
CArray *co;
|
|
345
1850
|
GetCArray(vout, co);
|
|
346
1851
|
boolean8_t *op = (boolean8_t *) co->ptr;
|
|
347
|
-
ca_iter_state st;
|
|
348
1852
|
int rc = ca_iter_state_init_l2(&st, ca, CA_SLAB_AXES,
|
|
349
1853
|
slab_axes, naxes, 0);
|
|
350
1854
|
if ( rc != CA_ITER_OK ) {
|
|
@@ -356,13 +1860,27 @@ boolean8_t *m;
|
|
|
356
1860
|
ca_size_t out_i = 0;
|
|
357
1861
|
while ( ca_iter_state_next_slab_axes(&st, &p, &m) ) {
|
|
358
1862
|
boolean8_t acc;
|
|
359
|
-
|
|
1863
|
+
CA_SLAB_REDUCE_T_SETTLE(boolean8_t, st, p, m, acc, 1, acc &= !v, !acc);
|
|
360
1864
|
op[out_i++] = (boolean8_t) (acc);
|
|
361
1865
|
}
|
|
362
1866
|
ca_iter_state_finish(&st);
|
|
363
1867
|
if ( naxes == ca->ndim && !keep_axis ) return BOOL2VAL(op[0]);
|
|
364
1868
|
return vout;
|
|
365
|
-
|
|
1869
|
+
#undef st
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
static VALUE
|
|
1873
|
+
none_ki_native_bool (VALUE self, CArray *ca, int8_t *slab_axes, int8_t naxes, int keep_axis)
|
|
1874
|
+
{
|
|
1875
|
+
none_ki_native_bool_frame fr;
|
|
1876
|
+
fr.self = self;
|
|
1877
|
+
fr.ca = ca;
|
|
1878
|
+
fr.slab_axes = slab_axes;
|
|
1879
|
+
fr.naxes = naxes;
|
|
1880
|
+
fr.keep_axis = keep_axis;
|
|
1881
|
+
return ca_iter_ensure(1, fr.states, none_ki_native_bool_body, (VALUE) &fr);
|
|
1882
|
+
}
|
|
1883
|
+
|
|
366
1884
|
|
|
367
1885
|
VALUE
|
|
368
1886
|
rb_ca_none_ki (int argc, VALUE *argv, VALUE self)
|