carray 3.0.5 → 3.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +233 -0
- data/README.md +1 -1
- data/carray.gemspec +3 -2
- data/ext/ca_axis_dispatch.c +6 -0
- data/ext/ca_axis_group.c +1 -1
- data/ext/ca_categorical_iterator.c +7 -7
- data/ext/ca_kernel_iterator.c +116 -231
- data/ext/ca_kernel_iterator.h +410 -254
- data/ext/ca_obj_bincmp.c +9 -1
- data/ext/ca_obj_binop.c +10 -1
- data/ext/ca_obj_bitfield.c +2 -0
- data/ext/ca_obj_byte_swap.c +2 -0
- data/ext/ca_obj_face.c +2 -0
- data/ext/ca_obj_fake.c +3 -1
- data/ext/ca_obj_grid.c +167 -12
- data/ext/ca_obj_meld.c +69 -113
- data/ext/ca_obj_moncmp.c +9 -1
- data/ext/ca_obj_monop.c +40 -2
- data/ext/ca_obj_object.c +54 -20
- data/ext/ca_obj_reduce.c +15 -1
- data/ext/ca_obj_refer.c +65 -20
- data/ext/ca_obj_remap.c +69 -10
- data/ext/ca_obj_roll.c +1 -0
- data/ext/ca_obj_select.c +4 -1
- data/ext/ca_obj_select_axis.c +1 -0
- data/ext/ca_obj_shift.c +1 -0
- data/ext/ca_obj_stack.c +310 -111
- data/ext/ca_obj_stride.c +131 -45
- data/ext/ca_obj_string.c +18 -2
- data/ext/ca_obj_tile.c +1 -0
- data/ext/ca_obj_triop.c +9 -1
- data/ext/ca_obj_window.c +1 -0
- data/ext/ca_op_ipower.c +8 -8
- data/ext/caframe_csv_reader.c +1 -1
- data/ext/carray.h +2 -1
- data/ext/carray_access.c +27 -64
- data/ext/carray_address_basis.c +134 -0
- data/ext/carray_attribute.c +8 -0
- data/ext/carray_broadcast.c +3 -4
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_cast.c +8 -11
- data/ext/carray_conversion.c +9 -5
- data/ext/carray_core.c +811 -72
- data/ext/carray_element.c +13 -9
- data/ext/carray_histogram.c +25 -3
- data/ext/carray_internal.h +147 -12
- data/ext/carray_kernels_bincmp.c +2007 -3
- data/ext/carray_kernels_binop.c +25365 -5199
- data/ext/carray_kernels_init.c +8 -8
- data/ext/carray_kernels_monop.c +10696 -3678
- data/ext/carray_kernels_reduce_aggregate.c +4429 -702
- data/ext/carray_kernels_reduce_boolean.c +1533 -15
- data/ext/carray_kernels_reduce_cumulative.c +1808 -248
- data/ext/carray_kernels_reduce_extreme.c +4508 -511
- data/ext/carray_kernels_reduce_variance.c +2284 -152
- data/ext/carray_kernels_scan.c +240 -240
- data/ext/carray_kernels_search.c +4211 -264
- data/ext/carray_kernels_sort.c +367 -450
- data/ext/carray_lazy.c +47 -0
- data/ext/carray_loop.c +1 -1
- data/ext/carray_mask.c +130 -139
- data/ext/carray_median_percentile.c +4 -4
- data/ext/carray_memory_view.c +13 -7
- data/ext/carray_operator.c +147 -44
- data/ext/carray_order.c +3 -2
- data/ext/carray_random.c +4 -1
- data/ext/carray_slab.c +97 -21
- data/ext/carray_sort.c +1 -1
- data/ext/carray_test.c +74 -19
- data/ext/carray_time_text.c +2 -2
- data/ext/extconf.rb +11 -0
- data/ext/mkkernel.rb +346 -94
- data/ext/ruby_carray.c +4 -1
- data/ext/version.h +4 -4
- data/lib/carray/attribute.rb +25 -1
- data/lib/carray/autoload_carray.rb +3 -3
- data/lib/carray/autoload_method_extension.rb +5 -3
- data/lib/carray/axis_group.rb +13 -7
- data/lib/carray/basics.rb +1 -1
- data/lib/carray/block_iterator.rb +17 -3
- data/lib/carray/categorical.rb +6 -8
- data/lib/carray/categorical_iterator.rb +2 -2
- data/lib/carray/complex.rb +2 -2
- data/lib/carray/conditional.rb +3 -2
- data/lib/carray/frame/column_verbs.rb +45 -14
- data/lib/carray/frame/convert.rb +19 -0
- data/lib/carray/frame/csv_parser.rb +32 -32
- data/lib/carray/frame/describe.rb +4 -2
- data/lib/carray/frame/frame.rb +399 -78
- data/lib/carray/frame/group.rb +679 -53
- data/lib/carray/frame/io.rb +209 -71
- data/lib/carray/frame/join.rb +146 -37
- data/lib/carray/frame/records.rb +3 -3
- data/lib/carray/frame/reshape.rb +89 -11
- data/lib/carray/frame/sort.rb +57 -9
- data/lib/carray/frame/verbs.rb +384 -49
- data/lib/carray/frame/windows.rb +44 -6
- data/lib/carray/inspect.rb +7 -2
- data/lib/carray/iterator.rb +32 -0
- data/lib/carray/lazy.rb +17 -8
- data/lib/carray/mask_gap_fill.rb +37 -17
- data/lib/carray/meld_reduce.rb +1 -1
- data/lib/carray/methods/choose.rb +3 -2
- data/lib/carray/methods/meshgrid.rb +3 -1
- data/lib/carray/runtime.rb +47 -8
- data/lib/carray/segment_iterator.rb +23 -8
- data/lib/carray/serialize.rb +34 -17
- data/lib/carray/stack.rb +18 -6
- data/lib/carray/time.rb +5 -0
- data/lib/carray/window_iterator.rb +16 -4
- data/yard-stubs/ca_obj_array.rb +26 -1
- data/yard-stubs/carray_cast.rb +8 -5
- data/yard-stubs/carray_conversion.rb +8 -2
- data/yard-stubs/carray_mask.rb +20 -1
- data/yard-stubs/carray_math.rb +31 -16
- data/yard-stubs/carray_memory_view.rb +7 -0
- data/yard-stubs/carray_test.rb +7 -4
- metadata +18 -4
data/ext/ca_kernel_iterator.h
CHANGED
|
@@ -49,14 +49,16 @@
|
|
|
49
49
|
v, r, w, acc, idx, first
|
|
50
50
|
|
|
51
51
|
INTERNAL (free to refactor across 3.x — engine implementation):
|
|
52
|
-
- the state-machine functions' bodies +
|
|
53
|
-
ca_iter_can_alias
|
|
52
|
+
- the state-machine functions' bodies + ca_iter_can_alias
|
|
54
53
|
- the macros only the kernel generator (ext/mkkernel.rb) emits:
|
|
55
54
|
CA_SLAB_REDUCE_SUM8_EX / MIN8_EX / MAX8_EX / STAR8_EX (8-way
|
|
56
55
|
accumulator split; they take EXPR(x) rather than exposing v and
|
|
57
56
|
idx), CA_SLAB_REDUCE_T_MIN_EX / _MAX_EX / _STAR_EX (what it emits
|
|
58
57
|
when a :min / :max / :prod kernel does not take the 8-way form),
|
|
59
|
-
|
|
58
|
+
CA_SLAB_REDUCE_SUM8_NARROW_EX / M2_NARROW_EX (8- and 16-bit
|
|
59
|
+
integer sum and centred sum of squares), CA_SLAB_REDUCE_T_SETTLE
|
|
60
|
+
with CA_SLAB_SETTLE_BLOCK (all / any / none), CA_SLAB_SCAN_T_GATED,
|
|
61
|
+
and CA_SLAB_REDUCE_T_VAR_EX (no caller)
|
|
60
62
|
- alias_mode / src_kind routing (CA_ITER_ALIAS_* / CA_ITER_SRC_* /
|
|
61
63
|
CA_KERNEL_FIBER_CONTIG / CA_SLAB_FREE)
|
|
62
64
|
- every ca_iter_state field NOT marked frozen (scratch / stack /
|
|
@@ -97,7 +99,9 @@ typedef enum {
|
|
|
97
99
|
there, even for a kernel that
|
|
98
100
|
writes every cell */
|
|
99
101
|
#define CA_KERNEL_NO_MASK 0x2 /* the kernel cannot take a mask:
|
|
100
|
-
a
|
|
102
|
+
a source that has a mask is
|
|
103
|
+
refused, even with no cell
|
|
104
|
+
masked (no cell is scanned) */
|
|
101
105
|
#define CA_KERNEL_CHUNK_HINT 0x4 /* reserved: init refuses it
|
|
102
106
|
with CA_ITER_ERR_FLAGS */
|
|
103
107
|
#define CA_KERNEL_FIBER_CONTIG 0x8 /* with policy=CA_SLAB_AXES and
|
|
@@ -136,13 +140,12 @@ typedef enum {
|
|
|
136
140
|
non-innermost slab offsets are
|
|
137
141
|
computed once per slab row and
|
|
138
142
|
the inner run is a memcpy. */
|
|
139
|
-
#define CA_ITER_ALIAS_PER_FIBER_FUSED 6 /* each fiber is fetched
|
|
140
|
-
|
|
141
|
-
ca_xfer_stride, without
|
|
143
|
+
#define CA_ITER_ALIAS_PER_FIBER_FUSED 6 /* each fiber is fetched with the
|
|
144
|
+
view's ca_xfer_stride, without
|
|
142
145
|
gathering the whole view: a
|
|
143
|
-
CAWindow / CAShift fiber, or a
|
|
144
146
|
read of the innermost fiber of
|
|
145
|
-
a conversion view
|
|
147
|
+
a conversion view (a write takes
|
|
148
|
+
the transfer walk). Chosen by
|
|
146
149
|
ca_iter_should_per_fiber_fused.
|
|
147
150
|
The fiber is handed out from
|
|
148
151
|
fiber_data_scratch, its mask
|
|
@@ -208,7 +211,8 @@ typedef enum {
|
|
|
208
211
|
#define CA_ITER_SRC_ATTACH 3 /* every other view the classifier
|
|
209
212
|
knows (conversion views, CATile,
|
|
210
213
|
CARoll, CAStack, CAMeld, CARemap,
|
|
211
|
-
CAObject, the lazy
|
|
214
|
+
CAReduce, CAObject, the lazy
|
|
215
|
+
views) and any
|
|
212
216
|
registered external obj_type: the
|
|
213
217
|
view's own transfer gathers it */
|
|
214
218
|
#define CA_ITER_SRC_DESCRIPTOR_L2_ALIASABLE 4
|
|
@@ -233,16 +237,19 @@ typedef enum {
|
|
|
233
237
|
CA_ITER_SRC_ATTACH is the only kind that may be registered. It is the
|
|
234
238
|
one whose contract an external class can meet on its own: the engine
|
|
235
239
|
reads the cells with ca_xfer_all GET and, on a WRITE walk, sends back
|
|
236
|
-
the changed cells with ca_xfer_addrs
|
|
237
|
-
|
|
238
|
-
slots
|
|
239
|
-
|
|
240
|
+
the changed cells with ca_xfer_addrs, so the view's transfer slots
|
|
241
|
+
are all the iterator needs. It does not call the view's attach / sync
|
|
242
|
+
slots, which ca_install_obj_type requires all the same: the paths
|
|
243
|
+
that take a view's buffer, such as a bang operator, call them. The
|
|
244
|
+
other kinds are not open to registration: SRC_CASTRIDE
|
|
240
245
|
asserts the struct *is* a CAStride (the engine reads its strides
|
|
241
246
|
directly, and a class that really is one is already classified by its
|
|
242
247
|
inherited operation table), and SRC_DESCRIPTOR requires a
|
|
243
248
|
describe_axes function the engine looks up in its own table, which an
|
|
244
249
|
external type has no way to supply. Passing anything else raises
|
|
245
|
-
rather than accepting a routing the iterator cannot honour.
|
|
250
|
+
rather than accepting a routing the iterator cannot honour. An
|
|
251
|
+
obj_type carray installs for itself is refused too: its routing comes
|
|
252
|
+
from its operation table.
|
|
246
253
|
|
|
247
254
|
Registering is additive and does not override the two structural
|
|
248
255
|
cases: the classifier still decides entity and CAStride-family sources
|
|
@@ -272,14 +279,24 @@ typedef struct {
|
|
|
272
279
|
/* --- CAStride compose-fold cache (L2 alias path) ---
|
|
273
280
|
For CAStride-family sources at L2, init_l2 runs
|
|
274
281
|
ca_stride_compose_to_root once to get the root + per-axis
|
|
275
|
-
byte strides + base offset,
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
282
|
+
byte strides + base offset, recorded here. These are engine
|
|
283
|
+
bookkeeping, not part of the author contract (a kernel reads the
|
|
284
|
+
slab_* / outer_* fields). The walk itself steps from composed_base
|
|
285
|
+
by outer_strides (below), which init builds from the composed
|
|
286
|
+
strides. Inline CA_RANK_MAX array (no heap alloc).
|
|
287
|
+
|
|
288
|
+
root is the array init attaches and finish detaches:
|
|
289
|
+
- the compose-fold root of an L2 CAStride walk (an entity, or a
|
|
290
|
+
view that lends memory), whose memory the slabs are cut from
|
|
291
|
+
- the parent of a descriptor view, on every descriptor route. On
|
|
292
|
+
the strided route (DESC_STRIDED) the slabs are cut from its
|
|
293
|
+
memory; on the L1 descriptor walk, PER_SLAB / PER_SLAB_HOIST
|
|
294
|
+
and DESC_WHOLE (SHIFT_WHOLE_VIEW included) the descriptor
|
|
295
|
+
engine gathers from it into a scratch of the walk's own (once,
|
|
296
|
+
or per slab for PER_SLAB) and the kernel reads the scratch, but
|
|
297
|
+
root stays attached until finish.
|
|
298
|
+
NULL when the walk reads src itself, a scratch filled through the
|
|
299
|
+
view's own transfer, or a CAStack's parents. */
|
|
283
300
|
struct _CArray *root; /* attached at init, detached (or
|
|
284
301
|
synced and detached, on WRITE)
|
|
285
302
|
by finish */
|
|
@@ -289,7 +306,7 @@ typedef struct {
|
|
|
289
306
|
/* --- descriptor framework cache (descriptor sources) ---
|
|
290
307
|
For CASelectAxis / CAGrid / CASelect / CAWindow / CAShift, init runs
|
|
291
308
|
the view's *_describe_axes once then
|
|
292
|
-
reuses the
|
|
309
|
+
reuses the descriptor engine's substrate (ca_axis_dispatch_prepare / _layout /
|
|
293
310
|
_classify_prefix) to derive the slab layout. Cached inline so
|
|
294
311
|
next_slab walks the prefix axes without re-doing the analysis. */
|
|
295
312
|
ca_axis_desc_t descs[CA_RANK_MAX]; /* post-merge axes */
|
|
@@ -406,9 +423,8 @@ typedef struct {
|
|
|
406
423
|
over src->dim used in ca_xfer_stride strides[] argument. */
|
|
407
424
|
int8_t fiber_axis;
|
|
408
425
|
ca_size_t fiber_native_strides[CA_RANK_MAX];
|
|
409
|
-
/* fiber_region_starts[]
|
|
410
|
-
|
|
411
|
-
region for WRITE PUT. The same hazard as last_data_off. */
|
|
426
|
+
/* fiber_region_starts[]: not used; kept so the fields after it stay
|
|
427
|
+
where ext gems built against this header expect them. */
|
|
412
428
|
ca_size_t fiber_region_starts[CA_RANK_MAX];
|
|
413
429
|
|
|
414
430
|
/* --- CAStack read from its parents (STACK / STACK_OUTER_K) --- */
|
|
@@ -526,15 +542,18 @@ int ca_iter_state_init_l1 (ca_iter_state *st,
|
|
|
526
542
|
- entity / CAStride contig: alias_mode = CONTIG, the slab reads
|
|
527
543
|
the source's own memory
|
|
528
544
|
- CAStride non-contig: alias_mode = STRIDED, the slab reads the
|
|
529
|
-
root through the composed strides
|
|
545
|
+
root through the composed strides, without a scratch, when the
|
|
546
|
+
root lends memory; over a root that does not (a conversion view,
|
|
547
|
+
a CAObject, a CAStack, ...) the view is gathered into scratch
|
|
530
548
|
- descriptor views: aliased when the innermost axis is a STRIDE
|
|
531
549
|
axis, otherwise gathered into scratch (per slab when a slab axis
|
|
532
550
|
is an INDEX or SHIFT axis)
|
|
533
551
|
- SRC_ATTACH views: iterator-owned scratch filled through
|
|
534
552
|
ca_xfer_all, written back with ki_scratch_push_changed (only the
|
|
535
553
|
changed cells). A CAStack may instead be walked through its
|
|
536
|
-
parents, and a single FIBER_CONTIG fiber
|
|
537
|
-
|
|
554
|
+
parents, and on a READ walk a single FIBER_CONTIG fiber along the
|
|
555
|
+
innermost axis of a cast / byte-swap / single-cast lazy view
|
|
556
|
+
through ca_xfer_stride (PER_FIBER_FUSED)
|
|
538
557
|
- a source with no attach slot: CA_ITER_ERR_NOT_CHEAP
|
|
539
558
|
|
|
540
559
|
policy: CA_SLAB_AXES only. CA_SLAB_WHOLE and CA_SLAB_FREE return
|
|
@@ -632,14 +651,14 @@ int ca_iter_state_next_slab_axes (ca_iter_state *st,
|
|
|
632
651
|
- per-fiber gather (CA_KERNEL_FIBER_CONTIG on a strided fiber):
|
|
633
652
|
the fiber is scattered back from its scratch first
|
|
634
653
|
- CAStack walked through its parents: READ only, no-op
|
|
635
|
-
- per-fiber fused (CA_ITER_ALIAS_PER_FIBER_FUSED):
|
|
636
|
-
|
|
654
|
+
- per-fiber fused (CA_ITER_ALIAS_PER_FIBER_FUSED): READ only, so a
|
|
655
|
+
WRITE walk never has it
|
|
637
656
|
- alias path (scratch_ptr == NULL): no-op — kernel wrote through
|
|
638
657
|
alias_ptr. When the alias is into a view root's own buffer, that
|
|
639
658
|
buffer is written back once, by finish (ca_sync_detach)
|
|
640
659
|
- scratch path, at L1 or L2: the cells the kernel changed are sent
|
|
641
|
-
back through the source (ki_scratch_push_changed: ca_xfer_addrs
|
|
642
|
-
|
|
660
|
+
back through the source (ki_scratch_push_changed: ca_xfer_addrs);
|
|
661
|
+
a write over a
|
|
643
662
|
descriptor view takes this path, opened as a transfer walk
|
|
644
663
|
|
|
645
664
|
A view that shows one root cell more than once (a repeated index, a
|
|
@@ -700,7 +719,8 @@ void ca_iter_state_close (ca_iter_state *st);
|
|
|
700
719
|
int ca_iter_check_init (int rc);
|
|
701
720
|
|
|
702
721
|
/* Raise ArgumentError unless the two arrays have the same shape. The
|
|
703
|
-
INOUT and PAIR
|
|
722
|
+
INOUT, SLAB_INOUT and PAIR macros call this before opening either
|
|
723
|
+
walk. */
|
|
704
724
|
int ca_iter_check_same_shape (struct _CArray *a, struct _CArray *b);
|
|
705
725
|
|
|
706
726
|
/* Run body(arg) so that the walks in states[0..n) are finished however the
|
|
@@ -991,8 +1011,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
991
1011
|
The INOUT forms cover input + output. These cover input + input, which
|
|
992
1012
|
is what a routine taking two vectors of the same length wants -- a
|
|
993
1013
|
correlation, a dot product, a distance. Neither state gets
|
|
994
|
-
CA_KERNEL_WRITE, and `flags` must not carry it
|
|
995
|
-
write.
|
|
1014
|
+
CA_KERNEL_WRITE, and `flags` must not carry it (it is refused with
|
|
1015
|
+
CA_ITER_ERR_FLAGS): use the INOUT forms to write.
|
|
996
1016
|
|
|
997
1017
|
Both fibers are contig-delivered, so the pair may be handed straight to
|
|
998
1018
|
a C routine that walks it itself. The MASKED form yields BOTH mask
|
|
@@ -1008,6 +1028,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1008
1028
|
#define CA_FOR_EACH_FIBER_PAIR(st_a, st_b, ca_a, ca_b, axis, \
|
|
1009
1029
|
flags, p_a, p_b, n) \
|
|
1010
1030
|
for ( int __cffp_init = ( \
|
|
1031
|
+
ca_iter_check_init(((flags) & CA_KERNEL_WRITE) \
|
|
1032
|
+
? CA_ITER_ERR_FLAGS : CA_ITER_OK), \
|
|
1011
1033
|
ca_iter_check_same_shape((ca_a), (ca_b)), \
|
|
1012
1034
|
ca_iter_check_init( \
|
|
1013
1035
|
ca_iter_state_init_l2(&(st_a), (ca_a), CA_SLAB_AXES, \
|
|
@@ -1032,6 +1054,8 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1032
1054
|
#define CA_FOR_EACH_FIBER_PAIR_MASKED(st_a, st_b, ca_a, ca_b, axis, \
|
|
1033
1055
|
flags, p_a, p_b, n, m_a, m_b) \
|
|
1034
1056
|
for ( int __cffpm_init = ( \
|
|
1057
|
+
ca_iter_check_init(((flags) & CA_KERNEL_WRITE) \
|
|
1058
|
+
? CA_ITER_ERR_FLAGS : CA_ITER_OK), \
|
|
1035
1059
|
ca_iter_check_same_shape((ca_a), (ca_b)), \
|
|
1036
1060
|
ca_iter_check_init( \
|
|
1037
1061
|
ca_iter_state_init_l2(&(st_a), (ca_a), CA_SLAB_AXES, \
|
|
@@ -1115,6 +1139,38 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1115
1139
|
|
|
1116
1140
|
/* ---- per-data_type reduction macro suite ------------------------------- */
|
|
1117
1141
|
|
|
1142
|
+
/* Whether the stride table `strides` walks the current slab of `st` in
|
|
1143
|
+
row-major order with no gap: the innermost axis steps `elem_bytes`, and
|
|
1144
|
+
each outer axis steps the whole of the axis inside it. A slab that does
|
|
1145
|
+
can be read as one flat run of slab_elements cells. A length-1 axis whose
|
|
1146
|
+
stride does not fit is answered "no", which only costs the flat loop. */
|
|
1147
|
+
static inline int
|
|
1148
|
+
ca_slab_strides_are_flat (const ca_iter_state *st, const ca_size_t *strides,
|
|
1149
|
+
ca_size_t elem_bytes)
|
|
1150
|
+
{
|
|
1151
|
+
int8_t k = (int8_t) (st->slab_ndim - 1);
|
|
1152
|
+
if ( strides[k] != elem_bytes ) {
|
|
1153
|
+
return 0;
|
|
1154
|
+
}
|
|
1155
|
+
for ( k = (int8_t) (k - 1); k >= 0; k-- ) {
|
|
1156
|
+
if ( strides[k] != st->slab_dims[k + 1] * strides[k + 1] ) {
|
|
1157
|
+
return 0;
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
return 1;
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
/* Whether the slab reductions may fold all slab axes of `st` into one flat
|
|
1164
|
+
loop: the data strides and, when the slab has a mask, the mask strides
|
|
1165
|
+
(one byte per cell) are row-major with no gap. */
|
|
1166
|
+
static inline int
|
|
1167
|
+
ca_slab_is_flat (const ca_iter_state *st, const ca_size_t *strides,
|
|
1168
|
+
ca_size_t elem_bytes, int has_mask)
|
|
1169
|
+
{
|
|
1170
|
+
return ca_slab_strides_are_flat(st, strides, elem_bytes)
|
|
1171
|
+
&& ( ! has_mask || ca_slab_strides_are_flat(st, st->slab_mask_strides, 1) );
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1118
1174
|
/* CA_SLAB_REDUCE_T(T, ...): generic per-data_type slab reduction. T is the
|
|
1119
1175
|
element load type (`double`, `float`, `int32_t`, `int64_t`, ...).
|
|
1120
1176
|
The accumulator `acc` is supplied by the caller and may be a wider
|
|
@@ -1170,27 +1226,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1170
1226
|
ca_size_t __srOC = 1; \
|
|
1171
1227
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1172
1228
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1173
|
-
/*
|
|
1174
|
-
(
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
contig multi-axis reduce). Platform-general: structural, not SIMD. \
|
|
1179
|
-
No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
|
|
1180
|
-
{ \
|
|
1181
|
-
int __srFlat = __srContig; \
|
|
1182
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1183
|
-
if ( (st).slab_strides[__sk] != \
|
|
1184
|
-
(st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
|
|
1185
|
-
__srFlat = 0; \
|
|
1186
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1187
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1188
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1189
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1190
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1191
|
-
__srFlat = 0; \
|
|
1192
|
-
} \
|
|
1193
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
1229
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1230
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1231
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1232
|
+
__srInnerN = (st).slab_elements; \
|
|
1233
|
+
__srOC = 1; \
|
|
1194
1234
|
} \
|
|
1195
1235
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1196
1236
|
ca_size_t idx = 0; \
|
|
@@ -1259,6 +1299,84 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1259
1299
|
(void) __sr_throwaway_mc; \
|
|
1260
1300
|
} while (0)
|
|
1261
1301
|
|
|
1302
|
+
/* CA_SLAB_REDUCE_T_SETTLE: a reduction that may stop before the end of
|
|
1303
|
+
the slab. SETTLED is an expression over `acc` that is true once no
|
|
1304
|
+
further cell can change the result (`acc` for any, `!acc` for all).
|
|
1305
|
+
The slab is walked in blocks of CA_SLAB_SETTLE_BLOCK cells; each block
|
|
1306
|
+
is the same branch-free loop as CA_SLAB_REDUCE_T_EX, and SETTLED is
|
|
1307
|
+
tested between blocks, so a slab that never settles costs one extra
|
|
1308
|
+
test per block. Masked cells are skipped as in CA_SLAB_REDUCE_T.
|
|
1309
|
+
`idx` and the masked count are not provided. */
|
|
1310
|
+
#define CA_SLAB_SETTLE_BLOCK 16384
|
|
1311
|
+
#define CA_SLAB_REDUCE_T_SETTLE(T, st, p, m, acc, INIT, REDUCE, SETTLED) do { \
|
|
1312
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1313
|
+
(acc) = (INIT); \
|
|
1314
|
+
int8_t __srK = (st).slab_ndim; \
|
|
1315
|
+
int8_t __srOuterK = __srK - 1; \
|
|
1316
|
+
ca_size_t __srInnerN = (st).slab_dims[__srK - 1]; \
|
|
1317
|
+
ca_size_t __srInnerS = (st).slab_strides[__srK - 1]; \
|
|
1318
|
+
ca_size_t __srInnerMS = (st).slab_mask_strides[__srK - 1]; \
|
|
1319
|
+
int __srContig = (__srInnerS == (ca_size_t) sizeof(T)); \
|
|
1320
|
+
int __srMaskU = (__srInnerMS == 1); \
|
|
1321
|
+
ca_size_t __srOC = 1; \
|
|
1322
|
+
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1323
|
+
__srOC *= (st).slab_dims[__sk]; \
|
|
1324
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1325
|
+
__srInnerN = (st).slab_elements; \
|
|
1326
|
+
__srOC = 1; \
|
|
1327
|
+
} \
|
|
1328
|
+
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1329
|
+
int __srDone = 0; \
|
|
1330
|
+
for ( ca_size_t __so = 0; __so < __srOC && ! __srDone; __so++ ) { \
|
|
1331
|
+
ca_size_t __srDoff = 0, __srMoff = 0; \
|
|
1332
|
+
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) { \
|
|
1333
|
+
__srDoff += __srIdx[__sk] * (st).slab_strides[__sk]; \
|
|
1334
|
+
__srMoff += __srIdx[__sk] * (st).slab_mask_strides[__sk]; \
|
|
1335
|
+
} \
|
|
1336
|
+
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1337
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1338
|
+
const boolean8_t *__srMM = \
|
|
1339
|
+
( (m) == NULL ) ? NULL : (const boolean8_t *)(m) + __srMoff; \
|
|
1340
|
+
for ( ca_size_t __sb = 0; __sb < __srInnerN; \
|
|
1341
|
+
__sb += CA_SLAB_SETTLE_BLOCK ) { \
|
|
1342
|
+
ca_size_t __se = ( __srInnerN - __sb > CA_SLAB_SETTLE_BLOCK ) \
|
|
1343
|
+
? __sb + CA_SLAB_SETTLE_BLOCK : __srInnerN; \
|
|
1344
|
+
if ( __srMM == NULL ) { \
|
|
1345
|
+
if ( __srContig ) { \
|
|
1346
|
+
for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
|
|
1347
|
+
T v = __srSrc[__sj]; \
|
|
1348
|
+
REDUCE; \
|
|
1349
|
+
} \
|
|
1350
|
+
} else { \
|
|
1351
|
+
for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
|
|
1352
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1353
|
+
REDUCE; \
|
|
1354
|
+
} \
|
|
1355
|
+
} \
|
|
1356
|
+
} else if ( __srContig && __srMaskU ) { \
|
|
1357
|
+
for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
|
|
1358
|
+
if ( ! __srMM[__sj] ) { \
|
|
1359
|
+
T v = __srSrc[__sj]; \
|
|
1360
|
+
REDUCE; \
|
|
1361
|
+
} \
|
|
1362
|
+
} \
|
|
1363
|
+
} else { \
|
|
1364
|
+
for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
|
|
1365
|
+
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1366
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1367
|
+
REDUCE; \
|
|
1368
|
+
} \
|
|
1369
|
+
} \
|
|
1370
|
+
} \
|
|
1371
|
+
if ( SETTLED ) { __srDone = 1; break; } \
|
|
1372
|
+
} \
|
|
1373
|
+
for ( int8_t __sk = __srOuterK - 1; __sk >= 0; __sk-- ) { \
|
|
1374
|
+
if ( ++__srIdx[__sk] < (st).slab_dims[__sk] ) break; \
|
|
1375
|
+
__srIdx[__sk] = 0; \
|
|
1376
|
+
} \
|
|
1377
|
+
} \
|
|
1378
|
+
} while (0)
|
|
1379
|
+
|
|
1262
1380
|
/* ----------------------------------------------------------------------
|
|
1263
1381
|
* SIMD-licensed reduction variants
|
|
1264
1382
|
*
|
|
@@ -1310,27 +1428,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1310
1428
|
ca_size_t __srOC = 1; \
|
|
1311
1429
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1312
1430
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1313
|
-
/*
|
|
1314
|
-
(
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
contig multi-axis reduce). Platform-general: structural, not SIMD. \
|
|
1319
|
-
No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
|
|
1320
|
-
{ \
|
|
1321
|
-
int __srFlat = __srContig; \
|
|
1322
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1323
|
-
if ( (st).slab_strides[__sk] != \
|
|
1324
|
-
(st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
|
|
1325
|
-
__srFlat = 0; \
|
|
1326
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1327
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1328
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1329
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1330
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1331
|
-
__srFlat = 0; \
|
|
1332
|
-
} \
|
|
1333
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
1431
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1432
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1433
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1434
|
+
__srInnerN = (st).slab_elements; \
|
|
1435
|
+
__srOC = 1; \
|
|
1334
1436
|
} \
|
|
1335
1437
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1336
1438
|
ca_size_t idx = 0; \
|
|
@@ -1431,20 +1533,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1431
1533
|
ca_size_t __srOC = 1; \
|
|
1432
1534
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1433
1535
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
__srFlat = 0; \
|
|
1440
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1441
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1442
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1443
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1444
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1445
|
-
__srFlat = 0; \
|
|
1446
|
-
} \
|
|
1447
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
1536
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1537
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1538
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1539
|
+
__srInnerN = (st).slab_elements; \
|
|
1540
|
+
__srOC = 1; \
|
|
1448
1541
|
} \
|
|
1449
1542
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1450
1543
|
for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
|
|
@@ -1503,6 +1596,185 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1503
1596
|
} \
|
|
1504
1597
|
} while (0)
|
|
1505
1598
|
|
|
1599
|
+
/* CA_SLAB_REDUCE_SUM8_NARROW_EX:
|
|
1600
|
+
* CA_SLAB_REDUCE_SUM8_EX for an 8- or 16-bit integer source summed into
|
|
1601
|
+
* a floating accumulator (sum and mean). Converting each cell to double
|
|
1602
|
+
* does not vectorize for these widths, so the contig run is summed in
|
|
1603
|
+
* int32 over blocks of 32768 cells (|cell| < 2^16, so a block sum stays
|
|
1604
|
+
* below 2^31) and only the block sums meet the accumulator. The block
|
|
1605
|
+
* sums are exact, so the answer is at least as close as the per-cell
|
|
1606
|
+
* form. EXPR(x) converts a block sum to ACC_T. The masked and
|
|
1607
|
+
* non-contig branches are the same as CA_SLAB_REDUCE_SUM8_EX.
|
|
1608
|
+
*
|
|
1609
|
+
* The block loop has no omp simd pragma: an integer sum vectorizes on
|
|
1610
|
+
* its own, and with the pragma GCC left a scalar copy of the loop in
|
|
1611
|
+
* the large generated kernels.
|
|
1612
|
+
*/
|
|
1613
|
+
#define CA_SLAB_REDUCE_SUM8_NARROW_EX(T, ACC_T, st, p, m, acc, INIT, EXPR, masked_cnt) do { \
|
|
1614
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1615
|
+
(acc) = (INIT); \
|
|
1616
|
+
int8_t __srK = (st).slab_ndim; \
|
|
1617
|
+
int8_t __srOuterK = __srK - 1; \
|
|
1618
|
+
ca_size_t __srInnerN = (st).slab_dims[__srK - 1]; \
|
|
1619
|
+
ca_size_t __srInnerS = (st).slab_strides[__srK - 1]; \
|
|
1620
|
+
ca_size_t __srInnerMS = (st).slab_mask_strides[__srK - 1]; \
|
|
1621
|
+
int __srContig = (__srInnerS == (ca_size_t) sizeof(T)); \
|
|
1622
|
+
int __srMaskU = (__srInnerMS == 1); \
|
|
1623
|
+
ca_size_t __srOC = 1; \
|
|
1624
|
+
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1625
|
+
__srOC *= (st).slab_dims[__sk]; \
|
|
1626
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1627
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1628
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1629
|
+
__srInnerN = (st).slab_elements; \
|
|
1630
|
+
__srOC = 1; \
|
|
1631
|
+
} \
|
|
1632
|
+
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1633
|
+
for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
|
|
1634
|
+
ca_size_t __srDoff = 0, __srMoff = 0; \
|
|
1635
|
+
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) { \
|
|
1636
|
+
__srDoff += __srIdx[__sk] * (st).slab_strides[__sk]; \
|
|
1637
|
+
__srMoff += __srIdx[__sk] * (st).slab_mask_strides[__sk]; \
|
|
1638
|
+
} \
|
|
1639
|
+
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1640
|
+
if ( (m) == NULL ) { \
|
|
1641
|
+
if ( __srContig ) { \
|
|
1642
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1643
|
+
for ( ca_size_t __sb = 0; __sb < __srInnerN; __sb += 32768 ) { \
|
|
1644
|
+
ca_size_t __se = ( __srInnerN - __sb < 32768 ) \
|
|
1645
|
+
? __srInnerN : __sb + 32768; \
|
|
1646
|
+
int32_t __sp = 0; \
|
|
1647
|
+
for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) \
|
|
1648
|
+
__sp += __srSrc[__sj]; \
|
|
1649
|
+
(acc) += EXPR(__sp); \
|
|
1650
|
+
} \
|
|
1651
|
+
} else { \
|
|
1652
|
+
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1653
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1654
|
+
(acc) += EXPR(v); \
|
|
1655
|
+
} \
|
|
1656
|
+
} \
|
|
1657
|
+
} else { \
|
|
1658
|
+
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1659
|
+
if ( __srContig && __srMaskU ) { \
|
|
1660
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1661
|
+
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1662
|
+
if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) += EXPR(v); } \
|
|
1663
|
+
else { (masked_cnt)++; } \
|
|
1664
|
+
} \
|
|
1665
|
+
} else { \
|
|
1666
|
+
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1667
|
+
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1668
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1669
|
+
(acc) += EXPR(v); \
|
|
1670
|
+
} else { (masked_cnt)++; } \
|
|
1671
|
+
} \
|
|
1672
|
+
} \
|
|
1673
|
+
} \
|
|
1674
|
+
for ( int8_t __sk = __srOuterK - 1; __sk >= 0; __sk-- ) { \
|
|
1675
|
+
if ( ++__srIdx[__sk] < (st).slab_dims[__sk] ) break; \
|
|
1676
|
+
__srIdx[__sk] = 0; \
|
|
1677
|
+
} \
|
|
1678
|
+
} \
|
|
1679
|
+
} while (0)
|
|
1680
|
+
|
|
1681
|
+
/* CA_SLAB_REDUCE_M2_NARROW_EX:
|
|
1682
|
+
* The centred sum of squares, sum (x - MEAN)^2, of an 8- or 16-bit
|
|
1683
|
+
* integer source (the second pass of variance and stddev). Converting
|
|
1684
|
+
* each cell to double does not vectorize for these widths, so the
|
|
1685
|
+
* contig run is centred on the integer c nearest MEAN (held within the
|
|
1686
|
+
* 8- and 16-bit range), and D = sum (x - c) and sum (x - c)^2 are added
|
|
1687
|
+
* in integers over blocks of 32768 cells; then
|
|
1688
|
+
* sum (x - MEAN)^2 = sum (x - c)^2 - 2 (MEAN - c) D + n (MEAN - c)^2.
|
|
1689
|
+
* The identity holds for any MEAN, and when MEAN is the mean of the
|
|
1690
|
+
* cells its rounding error enters only squared. Q_T is int32_t for
|
|
1691
|
+
* 8-bit cells (a block is at most 32768 * 255^2 < 2^31) and int64_t for
|
|
1692
|
+
* 16-bit cells. The block sums are exact. The masked and non-contig
|
|
1693
|
+
* branches add EXPR(x) per cell as CA_SLAB_REDUCE_SUM8_EX does; acc is
|
|
1694
|
+
* double.
|
|
1695
|
+
*/
|
|
1696
|
+
#define CA_SLAB_REDUCE_M2_NARROW_EX(T, Q_T, st, p, m, acc, INIT, MEAN, EXPR, masked_cnt) do { \
|
|
1697
|
+
typedef T __ca_u_T CA_UNALIGNED; \
|
|
1698
|
+
(acc) = (INIT); \
|
|
1699
|
+
int8_t __srK = (st).slab_ndim; \
|
|
1700
|
+
int8_t __srOuterK = __srK - 1; \
|
|
1701
|
+
ca_size_t __srInnerN = (st).slab_dims[__srK - 1]; \
|
|
1702
|
+
ca_size_t __srInnerS = (st).slab_strides[__srK - 1]; \
|
|
1703
|
+
ca_size_t __srInnerMS = (st).slab_mask_strides[__srK - 1]; \
|
|
1704
|
+
int __srContig = (__srInnerS == (ca_size_t) sizeof(T)); \
|
|
1705
|
+
int __srMaskU = (__srInnerMS == 1); \
|
|
1706
|
+
ca_size_t __srOC = 1; \
|
|
1707
|
+
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1708
|
+
__srOC *= (st).slab_dims[__sk]; \
|
|
1709
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1710
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1711
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1712
|
+
__srInnerN = (st).slab_elements; \
|
|
1713
|
+
__srOC = 1; \
|
|
1714
|
+
} \
|
|
1715
|
+
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1716
|
+
for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
|
|
1717
|
+
ca_size_t __srDoff = 0, __srMoff = 0; \
|
|
1718
|
+
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) { \
|
|
1719
|
+
__srDoff += __srIdx[__sk] * (st).slab_strides[__sk]; \
|
|
1720
|
+
__srMoff += __srIdx[__sk] * (st).slab_mask_strides[__sk]; \
|
|
1721
|
+
} \
|
|
1722
|
+
const char *__srQ = (const char *)(p) + __srDoff; \
|
|
1723
|
+
if ( (m) == NULL ) { \
|
|
1724
|
+
if ( __srContig ) { \
|
|
1725
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1726
|
+
double __scm = (MEAN); \
|
|
1727
|
+
int32_t __sc = ( __scm > 65535.0 ) ? 65535 \
|
|
1728
|
+
: ( __scm < -32768.0 ) ? -32768 \
|
|
1729
|
+
: (int32_t) lrint(__scm); \
|
|
1730
|
+
double __sdm = __scm - (double) __sc; \
|
|
1731
|
+
int64_t __sdt = 0; \
|
|
1732
|
+
for ( ca_size_t __sb = 0; __sb < __srInnerN; __sb += 32768 ) { \
|
|
1733
|
+
ca_size_t __se = ( __srInnerN - __sb < 32768 ) \
|
|
1734
|
+
? __srInnerN : __sb + 32768; \
|
|
1735
|
+
Q_T __sq = 0; \
|
|
1736
|
+
int32_t __sl = 0; \
|
|
1737
|
+
for ( ca_size_t __sj = __sb; __sj < __se; __sj++ ) { \
|
|
1738
|
+
int32_t __sd = (int32_t) __srSrc[__sj] - __sc; \
|
|
1739
|
+
__sl += __sd; \
|
|
1740
|
+
__sq += (Q_T) __sd * __sd; \
|
|
1741
|
+
} \
|
|
1742
|
+
__sdt += __sl; \
|
|
1743
|
+
(acc) += (double) __sq; \
|
|
1744
|
+
} \
|
|
1745
|
+
if ( __srInnerN > 0 ) \
|
|
1746
|
+
(acc) += __sdm * ( (double) __srInnerN * __sdm \
|
|
1747
|
+
- 2.0 * (double) __sdt ); \
|
|
1748
|
+
} else { \
|
|
1749
|
+
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1750
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1751
|
+
(acc) += EXPR(v); \
|
|
1752
|
+
} \
|
|
1753
|
+
} \
|
|
1754
|
+
} else { \
|
|
1755
|
+
const boolean8_t *__srMM = (const boolean8_t *)(m) + __srMoff; \
|
|
1756
|
+
if ( __srContig && __srMaskU ) { \
|
|
1757
|
+
const __ca_u_T *__srSrc = (const __ca_u_T *) __srQ; \
|
|
1758
|
+
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1759
|
+
if ( ! __srMM[__sj] ) { T v = __srSrc[__sj]; (acc) += EXPR(v); } \
|
|
1760
|
+
else { (masked_cnt)++; } \
|
|
1761
|
+
} \
|
|
1762
|
+
} else { \
|
|
1763
|
+
for ( ca_size_t __sj = 0; __sj < __srInnerN; __sj++ ) { \
|
|
1764
|
+
if ( ! __srMM[__sj * __srInnerMS] ) { \
|
|
1765
|
+
T v = *(const __ca_u_T *)(__srQ + __sj * __srInnerS); \
|
|
1766
|
+
(acc) += EXPR(v); \
|
|
1767
|
+
} else { (masked_cnt)++; } \
|
|
1768
|
+
} \
|
|
1769
|
+
} \
|
|
1770
|
+
} \
|
|
1771
|
+
for ( int8_t __sk = __srOuterK - 1; __sk >= 0; __sk-- ) { \
|
|
1772
|
+
if ( ++__srIdx[__sk] < (st).slab_dims[__sk] ) break; \
|
|
1773
|
+
__srIdx[__sk] = 0; \
|
|
1774
|
+
} \
|
|
1775
|
+
} \
|
|
1776
|
+
} while (0)
|
|
1777
|
+
|
|
1506
1778
|
/* CA_SLAB_REDUCE_MIN8_EX / _MAX8_EX / _STAR8_EX (the SUM8_EX pattern for
|
|
1507
1779
|
* the other standard reducers):
|
|
1508
1780
|
*
|
|
@@ -1538,20 +1810,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1538
1810
|
ca_size_t __srOC = 1; \
|
|
1539
1811
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1540
1812
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
__srFlat = 0; \
|
|
1547
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1548
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1549
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1550
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1551
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1552
|
-
__srFlat = 0; \
|
|
1553
|
-
} \
|
|
1554
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
1813
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1814
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1815
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1816
|
+
__srInnerN = (st).slab_elements; \
|
|
1817
|
+
__srOC = 1; \
|
|
1555
1818
|
} \
|
|
1556
1819
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1557
1820
|
for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
|
|
@@ -1632,20 +1895,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1632
1895
|
ca_size_t __srOC = 1; \
|
|
1633
1896
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1634
1897
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
__srFlat = 0; \
|
|
1641
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1642
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1643
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1644
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1645
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1646
|
-
__srFlat = 0; \
|
|
1647
|
-
} \
|
|
1648
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
1898
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1899
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1900
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1901
|
+
__srInnerN = (st).slab_elements; \
|
|
1902
|
+
__srOC = 1; \
|
|
1649
1903
|
} \
|
|
1650
1904
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1651
1905
|
for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
|
|
@@ -1726,20 +1980,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1726
1980
|
ca_size_t __srOC = 1; \
|
|
1727
1981
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1728
1982
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
__srFlat = 0; \
|
|
1735
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1736
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1737
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1738
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1739
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1740
|
-
__srFlat = 0; \
|
|
1741
|
-
} \
|
|
1742
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
1983
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
1984
|
+
contiguous (ca_slab_is_flat) */ \
|
|
1985
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
1986
|
+
__srInnerN = (st).slab_elements; \
|
|
1987
|
+
__srOC = 1; \
|
|
1743
1988
|
} \
|
|
1744
1989
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1745
1990
|
for ( ca_size_t __so = 0; __so < __srOC; __so++ ) { \
|
|
@@ -1823,27 +2068,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1823
2068
|
ca_size_t __srOC = 1; \
|
|
1824
2069
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1825
2070
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1826
|
-
/*
|
|
1827
|
-
(
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
contig multi-axis reduce). Platform-general: structural, not SIMD. \
|
|
1832
|
-
No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
|
|
1833
|
-
{ \
|
|
1834
|
-
int __srFlat = __srContig; \
|
|
1835
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1836
|
-
if ( (st).slab_strides[__sk] != \
|
|
1837
|
-
(st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
|
|
1838
|
-
__srFlat = 0; \
|
|
1839
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1840
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1841
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1842
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1843
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1844
|
-
__srFlat = 0; \
|
|
1845
|
-
} \
|
|
1846
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
2071
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
2072
|
+
contiguous (ca_slab_is_flat) */ \
|
|
2073
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
2074
|
+
__srInnerN = (st).slab_elements; \
|
|
2075
|
+
__srOC = 1; \
|
|
1847
2076
|
} \
|
|
1848
2077
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1849
2078
|
ca_size_t idx = 0; \
|
|
@@ -1917,27 +2146,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
1917
2146
|
ca_size_t __srOC = 1; \
|
|
1918
2147
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
1919
2148
|
__srOC *= (st).slab_dims[__sk]; \
|
|
1920
|
-
/*
|
|
1921
|
-
(
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
contig multi-axis reduce). Platform-general: structural, not SIMD. \
|
|
1926
|
-
No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
|
|
1927
|
-
{ \
|
|
1928
|
-
int __srFlat = __srContig; \
|
|
1929
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1930
|
-
if ( (st).slab_strides[__sk] != \
|
|
1931
|
-
(st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
|
|
1932
|
-
__srFlat = 0; \
|
|
1933
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
1934
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
1935
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
1936
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
1937
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
1938
|
-
__srFlat = 0; \
|
|
1939
|
-
} \
|
|
1940
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
2149
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
2150
|
+
contiguous (ca_slab_is_flat) */ \
|
|
2151
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
2152
|
+
__srInnerN = (st).slab_elements; \
|
|
2153
|
+
__srOC = 1; \
|
|
1941
2154
|
} \
|
|
1942
2155
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
1943
2156
|
ca_size_t idx = 0; \
|
|
@@ -2018,27 +2231,11 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2018
2231
|
ca_size_t __srOC = 1; \
|
|
2019
2232
|
for ( int8_t __sk = 0; __sk < __srOuterK; __sk++ ) \
|
|
2020
2233
|
__srOC *= (st).slab_dims[__sk]; \
|
|
2021
|
-
/*
|
|
2022
|
-
(
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
contig multi-axis reduce). Platform-general: structural, not SIMD. \
|
|
2027
|
-
No-op for 1-D slabs (__srOuterK == 0 leaves __srOC == 1). */ \
|
|
2028
|
-
{ \
|
|
2029
|
-
int __srFlat = __srContig; \
|
|
2030
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
2031
|
-
if ( (st).slab_strides[__sk] != \
|
|
2032
|
-
(st).slab_dims[__sk + 1] * (st).slab_strides[__sk + 1] ) \
|
|
2033
|
-
__srFlat = 0; \
|
|
2034
|
-
if ( __srFlat && (m) != NULL ) { \
|
|
2035
|
-
if ( ! __srMaskU ) __srFlat = 0; \
|
|
2036
|
-
for ( int8_t __sk = (int8_t)(__srK - 2); __sk >= 0 && __srFlat; __sk-- ) \
|
|
2037
|
-
if ( (st).slab_mask_strides[__sk] != \
|
|
2038
|
-
(st).slab_dims[__sk + 1] * (st).slab_mask_strides[__sk + 1] ) \
|
|
2039
|
-
__srFlat = 0; \
|
|
2040
|
-
} \
|
|
2041
|
-
if ( __srFlat ) { __srInnerN = (st).slab_elements; __srOC = 1; } \
|
|
2234
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
2235
|
+
contiguous (ca_slab_is_flat) */ \
|
|
2236
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) ) { \
|
|
2237
|
+
__srInnerN = (st).slab_elements; \
|
|
2238
|
+
__srOC = 1; \
|
|
2042
2239
|
} \
|
|
2043
2240
|
ca_size_t __srIdx[CA_RANK_MAX] = { 0 }; \
|
|
2044
2241
|
ca_size_t idx = 0; \
|
|
@@ -2161,32 +2358,12 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2161
2358
|
ca_size_t __sraOC = 1; \
|
|
2162
2359
|
for ( int8_t __sak = 0; __sak < __sraOuterK; __sak++ ) \
|
|
2163
2360
|
__sraOC *= (st).slab_dims[__sak]; \
|
|
2164
|
-
/*
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
Weights share slab_dims with source (shape invariant), so the dim \
|
|
2171
|
-
cascade uses (st).slab_dims for both stride tables. */ \
|
|
2172
|
-
{ \
|
|
2173
|
-
int __sraFlat = __sraContig && __sraContigW; \
|
|
2174
|
-
for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) { \
|
|
2175
|
-
if ( (st).slab_strides[__sak] != \
|
|
2176
|
-
(st).slab_dims[__sak + 1] * (st).slab_strides[__sak + 1] ) \
|
|
2177
|
-
__sraFlat = 0; \
|
|
2178
|
-
if ( (st_w).slab_strides[__sak] != \
|
|
2179
|
-
(st).slab_dims[__sak + 1] * (st_w).slab_strides[__sak + 1] ) \
|
|
2180
|
-
__sraFlat = 0; \
|
|
2181
|
-
} \
|
|
2182
|
-
if ( __sraFlat && (m) != NULL ) { \
|
|
2183
|
-
if ( ! __sraMaskU ) __sraFlat = 0; \
|
|
2184
|
-
for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) \
|
|
2185
|
-
if ( (st).slab_mask_strides[__sak] != \
|
|
2186
|
-
(st).slab_dims[__sak + 1] * (st).slab_mask_strides[__sak + 1] ) \
|
|
2187
|
-
__sraFlat = 0; \
|
|
2188
|
-
} \
|
|
2189
|
-
if ( __sraFlat ) { __sraInnerN = (st).slab_elements; __sraOC = 1; } \
|
|
2361
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
2362
|
+
contiguous (ca_slab_is_flat) */ \
|
|
2363
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) \
|
|
2364
|
+
&& ca_slab_strides_are_flat(&(st), (st_w).slab_strides, sizeof(T_W)) ) { \
|
|
2365
|
+
__sraInnerN = (st).slab_elements; \
|
|
2366
|
+
__sraOC = 1; \
|
|
2190
2367
|
} \
|
|
2191
2368
|
ca_size_t __sraIdx[CA_RANK_MAX] = { 0 }; \
|
|
2192
2369
|
ca_size_t idx = 0; \
|
|
@@ -2283,32 +2460,12 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2283
2460
|
ca_size_t __sraOC = 1; \
|
|
2284
2461
|
for ( int8_t __sak = 0; __sak < __sraOuterK; __sak++ ) \
|
|
2285
2462
|
__sraOC *= (st).slab_dims[__sak]; \
|
|
2286
|
-
/*
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
Weights share slab_dims with source (shape invariant), so the dim \
|
|
2293
|
-
cascade uses (st).slab_dims for both stride tables. */ \
|
|
2294
|
-
{ \
|
|
2295
|
-
int __sraFlat = __sraContig && __sraContigW; \
|
|
2296
|
-
for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) { \
|
|
2297
|
-
if ( (st).slab_strides[__sak] != \
|
|
2298
|
-
(st).slab_dims[__sak + 1] * (st).slab_strides[__sak + 1] ) \
|
|
2299
|
-
__sraFlat = 0; \
|
|
2300
|
-
if ( (st_w).slab_strides[__sak] != \
|
|
2301
|
-
(st).slab_dims[__sak + 1] * (st_w).slab_strides[__sak + 1] ) \
|
|
2302
|
-
__sraFlat = 0; \
|
|
2303
|
-
} \
|
|
2304
|
-
if ( __sraFlat && (m) != NULL ) { \
|
|
2305
|
-
if ( ! __sraMaskU ) __sraFlat = 0; \
|
|
2306
|
-
for ( int8_t __sak = (int8_t)(__sraK - 2); __sak >= 0 && __sraFlat; __sak-- ) \
|
|
2307
|
-
if ( (st).slab_mask_strides[__sak] != \
|
|
2308
|
-
(st).slab_dims[__sak + 1] * (st).slab_mask_strides[__sak + 1] ) \
|
|
2309
|
-
__sraFlat = 0; \
|
|
2310
|
-
} \
|
|
2311
|
-
if ( __sraFlat ) { __sraInnerN = (st).slab_elements; __sraOC = 1; } \
|
|
2463
|
+
/* fold all slab axes into one flat loop when the slab is row-major \
|
|
2464
|
+
contiguous (ca_slab_is_flat) */ \
|
|
2465
|
+
if ( ca_slab_is_flat(&(st), (st).slab_strides, sizeof(T), (m) != NULL) \
|
|
2466
|
+
&& ca_slab_strides_are_flat(&(st), (st_w).slab_strides, sizeof(T_W)) ) { \
|
|
2467
|
+
__sraInnerN = (st).slab_elements; \
|
|
2468
|
+
__sraOC = 1; \
|
|
2312
2469
|
} \
|
|
2313
2470
|
ca_size_t __sraIdx[CA_RANK_MAX] = { 0 }; \
|
|
2314
2471
|
ca_size_t idx = 0; \
|
|
@@ -2741,10 +2898,9 @@ VALUE ca_iter_ensure (int32_t n, ca_iter_state *states,
|
|
|
2741
2898
|
} while (0)
|
|
2742
2899
|
|
|
2743
2900
|
/* ---- Ruby surface ---------------------------------------------------- */
|
|
2744
|
-
/* Called from Init_carray_ext (ruby_carray.c).
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
the spec_ai kernel iterator tests use. */
|
|
2901
|
+
/* Called from Init_carray_ext (ruby_carray.c). In a development build
|
|
2902
|
+
(CARRAY_DEV_BUILD) it registers the CArray.t1_* smoke entry points the
|
|
2903
|
+
spec_ai kernel iterator tests use; in a release build it does nothing. */
|
|
2748
2904
|
void Init_ca_kernel_iterator (void);
|
|
2749
2905
|
|
|
2750
2906
|
#endif /* CA_KERNEL_ITERATOR_H */
|