carray 3.0.2 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +568 -0
- data/README.md +6 -6
- data/carray.gemspec +1 -1
- data/ext/ca_axis_descriptor.h +27 -24
- data/ext/ca_axis_dispatch.c +220 -0
- data/ext/ca_axis_group.c +201 -123
- data/ext/ca_binop_dispatch.c +46 -52
- data/ext/ca_binop_dispatch.h +1 -1
- data/ext/ca_categorical_iterator.c +249 -183
- data/ext/ca_compare.c +7 -12
- data/ext/ca_for_buffer.h +21 -68
- data/ext/ca_group_iter.c +38 -23
- data/ext/ca_kernel_iterator.c +1765 -2076
- data/ext/ca_kernel_iterator.h +584 -566
- data/ext/ca_monop_dispatch.c +86 -63
- data/ext/ca_monop_dispatch.h +5 -0
- data/ext/ca_obj_array.c +223 -56
- data/ext/ca_obj_bincmp.c +172 -138
- data/ext/ca_obj_binop.c +116 -230
- data/ext/ca_obj_bitarray.c +40 -26
- data/ext/ca_obj_bitfield.c +100 -68
- data/ext/ca_obj_block.c +19 -9
- data/ext/ca_obj_byte_swap.c +45 -46
- data/ext/ca_obj_const_string.c +269 -44
- data/ext/ca_obj_face.c +612 -89
- data/ext/ca_obj_face.h +111 -47
- data/ext/ca_obj_fake.c +36 -79
- data/ext/ca_obj_farray.c +7 -3
- data/ext/ca_obj_field.c +33 -8
- data/ext/ca_obj_fixlen_string.c +23 -24
- data/ext/ca_obj_grid.c +541 -210
- data/ext/ca_obj_meld.c +331 -111
- data/ext/ca_obj_moncmp.c +80 -65
- data/ext/ca_obj_monop.c +177 -149
- data/ext/ca_obj_object.c +664 -248
- data/ext/ca_obj_record.c +63 -23
- data/ext/ca_obj_reduce.c +18 -16
- data/ext/ca_obj_refer.c +140 -58
- data/ext/ca_obj_remap.c +85 -24
- data/ext/ca_obj_repeat.c +24 -9
- data/ext/ca_obj_roll.c +121 -16
- data/ext/ca_obj_select.c +61 -58
- data/ext/ca_obj_select_axis.c +73 -107
- data/ext/ca_obj_shift.c +32 -102
- data/ext/ca_obj_stack.c +372 -122
- data/ext/ca_obj_stride.c +348 -140
- data/ext/ca_obj_string.c +88 -22
- data/ext/ca_obj_tile.c +38 -21
- data/ext/ca_obj_time.c +45 -31
- data/ext/ca_obj_timedelta.c +43 -29
- data/ext/ca_obj_transpose.c +17 -6
- data/ext/ca_obj_triop.c +125 -141
- data/ext/ca_obj_window.c +330 -44
- data/ext/ca_op_ipower.c +24 -11
- data/ext/ca_op_powi.h +57 -30
- data/ext/ca_pow5_table.h +656 -0
- data/ext/ca_sort_kernels.h +70 -0
- data/ext/ca_sweep_engine.c +104 -81
- data/ext/ca_sweep_engine.h +7 -20
- data/ext/ca_triop_dispatch.c +4 -1
- data/ext/ca_triop_dispatch.h +5 -2
- data/ext/caframe_csv_reader.c +542 -0
- data/ext/carray.h +241 -92
- data/ext/carray_access.c +608 -276
- data/ext/carray_address_basis.c +494 -97
- data/ext/carray_attribute.c +129 -4
- data/ext/carray_bincount.c +39 -7
- data/ext/carray_broadcast.c +56 -19
- data/ext/carray_build_flags.h +1 -1
- data/ext/carray_call_cfunc.c +108 -28
- data/ext/carray_cast.c +1576 -223
- data/ext/carray_cast_func.rb +77 -24
- data/ext/carray_class.c +4 -3
- data/ext/carray_conversion.c +208 -75
- data/ext/carray_copy.c +82 -24
- data/ext/carray_core.c +646 -162
- data/ext/carray_count.c +171 -88
- data/ext/carray_element.c +94 -60
- data/ext/carray_factorize.c +670 -324
- data/ext/carray_generate.c +260 -83
- data/ext/carray_histogram.c +209 -106
- data/ext/carray_hold.c +14 -16
- data/ext/carray_index_classifier.c +422 -243
- data/ext/carray_internal.h +692 -3
- data/ext/carray_kernels_bincmp.c +7 -0
- data/ext/carray_kernels_binop.c +487 -120
- data/ext/carray_kernels_init.c +653 -50
- data/ext/carray_kernels_moncmp.c +15 -8
- data/ext/carray_kernels_monop.c +99 -43
- data/ext/carray_kernels_reduce_aggregate.c +3925 -6112
- data/ext/carray_kernels_reduce_boolean.c +112 -36
- data/ext/carray_kernels_reduce_cumulative.c +2127 -6903
- data/ext/carray_kernels_reduce_extreme.c +3032 -2892
- data/ext/carray_kernels_reduce_variance.c +439 -500
- data/ext/carray_kernels_scan.c +1097 -488
- data/ext/carray_kernels_search.c +3787 -2176
- data/ext/carray_kernels_sort.c +1200 -920
- data/ext/carray_kernels_triop.c +479 -7
- data/ext/carray_key_windows.c +216 -0
- data/ext/carray_lazy.c +967 -153
- data/ext/carray_loop.c +81 -65
- data/ext/carray_mask.c +910 -324
- data/ext/carray_math_kernel.h +1 -0
- data/ext/carray_median_percentile.c +299 -100
- data/ext/carray_memory_view.c +872 -758
- data/ext/carray_operator.c +1171 -364
- data/ext/carray_order.c +505 -133
- data/ext/carray_partition.c +112 -65
- data/ext/carray_random.c +327 -239
- data/ext/carray_round.c +273 -0
- data/ext/carray_scatter.c +346 -78
- data/ext/carray_segment.c +257 -0
- data/ext/carray_slab.c +249 -86
- data/ext/carray_sort.c +224 -227
- data/ext/carray_sort_kernel.c +8 -1
- data/ext/carray_struct.c +118 -39
- data/ext/carray_test.c +193 -69
- data/ext/carray_time_text.c +1182 -0
- data/ext/carray_undef.c +4 -0
- data/ext/carray_utils.c +579 -113
- data/ext/extconf.rb +27 -2
- data/ext/mk_call_cfunc.rb +19 -4
- data/ext/mk_pow5_table.rb +35 -0
- data/ext/mkkernel.rb +1202 -1014
- data/ext/ruby_carray.c +17 -5
- data/ext/version.h +4 -4
- data/ext/version.rb +4 -2
- data/lib/carray/attribute.rb +160 -17
- data/lib/carray/autoload_carray.rb +23 -27
- data/lib/carray/autoload_method_extension.rb +20 -5
- data/lib/carray/axis_group.rb +360 -63
- data/lib/carray/basics.rb +162 -28
- data/lib/carray/bincount_nd.rb +18 -2
- data/lib/carray/block_iterator.rb +231 -58
- data/lib/carray/boolean_reduce.rb +34 -42
- data/lib/carray/categorical.rb +242 -52
- data/lib/carray/categorical_iterator.rb +437 -827
- data/lib/carray/complex.rb +41 -34
- data/lib/carray/conditional.rb +150 -14
- data/lib/carray/const_string.rb +59 -4
- data/lib/carray/construct.rb +15 -11
- data/lib/carray/data_type_extension.rb +1 -0
- data/lib/carray/distance_limit.rb +77 -0
- data/lib/carray/fixlen_string.rb +7 -1
- data/lib/carray/frame/column_verbs.rb +70 -0
- data/lib/carray/frame/concat.rb +157 -6
- data/lib/carray/frame/convert.rb +82 -7
- data/lib/carray/frame/csv_parser.rb +688 -87
- data/lib/carray/frame/describe.rb +80 -0
- data/lib/carray/frame/frame.rb +369 -45
- data/lib/carray/frame/group.rb +631 -90
- data/lib/carray/frame/io.rb +506 -79
- data/lib/carray/frame/join.rb +411 -62
- data/lib/carray/frame/records.rb +116 -19
- data/lib/carray/frame/reshape.rb +455 -0
- data/lib/carray/frame/sort.rb +30 -18
- data/lib/carray/frame/verbs.rb +868 -61
- data/lib/carray/frame/windows.rb +117 -0
- data/lib/carray/frame.rb +4 -0
- data/lib/carray/fuse_source.rb +118 -2
- data/lib/carray/fusion.rb +257 -42
- data/lib/carray/histogram.rb +6 -1
- data/lib/carray/inspect.rb +54 -13
- data/lib/carray/iterator.rb +99 -9
- data/lib/carray/key_windows.rb +620 -0
- data/lib/carray/lazy.rb +216 -193
- data/lib/carray/mask_gap_fill.rb +112 -92
- data/lib/carray/math.rb +32 -19
- data/lib/carray/meld_reduce.rb +52 -21
- data/lib/carray/methods/align_addr.rb +8 -4
- data/lib/carray/methods/bin.rb +26 -18
- data/lib/carray/methods/bincount.rb +18 -2
- data/lib/carray/methods/bit_string.rb +15 -4
- data/lib/carray/methods/choose.rb +16 -3
- data/lib/carray/methods/composition.rb +18 -10
- data/lib/carray/methods/factorize.rb +2 -2
- data/lib/carray/methods/gather_nd.rb +25 -5
- data/lib/carray/methods/insert_block.rb +9 -14
- data/lib/carray/methods/is_between.rb +59 -0
- data/lib/carray/methods/is_in.rb +140 -10
- data/lib/carray/methods/join.rb +2 -7
- data/lib/carray/methods/lag.rb +49 -0
- data/lib/carray/methods/locate_addr.rb +182 -45
- data/lib/carray/methods/mode.rb +18 -1
- data/lib/carray/methods/nunique.rb +11 -6
- data/lib/carray/methods/repeat.rb +1 -20
- data/lib/carray/methods/resize.rb +21 -23
- data/lib/carray/methods/snap.rb +34 -12
- data/lib/carray/methods/unique.rb +11 -2
- data/lib/carray/methods/value_counts.rb +4 -11
- data/lib/carray/record_order.rb +208 -0
- data/lib/carray/rng.rb +3 -0
- data/lib/carray/runtime.rb +65 -17
- data/lib/carray/segment_iterator.rb +1221 -0
- data/lib/carray/serialize.rb +214 -30
- data/lib/carray/slab_iterator.rb +101 -27
- data/lib/carray/stack.rb +10 -4
- data/lib/carray/string.rb +105 -5
- data/lib/carray/string_operation_extension.rb +12 -4
- data/lib/carray/struct.rb +137 -58
- data/lib/carray/struct_builder.rb +124 -17
- data/lib/carray/time.rb +1222 -186
- data/lib/carray/window_iterator.rb +242 -81
- data/lib/carray.rb +9 -2
- data/yard-stubs/ca_obj_array.rb +4 -4
- data/yard-stubs/ca_obj_bitarray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +23 -12
- data/yard-stubs/ca_obj_fake.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +5 -1
- data/yard-stubs/ca_obj_grid.rb +21 -10
- data/yard-stubs/ca_obj_refer.rb +5 -4
- data/yard-stubs/ca_obj_roll.rb +2 -3
- data/yard-stubs/ca_obj_shift.rb +5 -1
- data/yard-stubs/ca_obj_stride.rb +10 -7
- data/yard-stubs/ca_obj_tile.rb +1 -1
- data/yard-stubs/ca_obj_transpose.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +9 -5
- data/yard-stubs/carray_access.rb +11 -5
- data/yard-stubs/carray_attribute.rb +0 -5
- data/yard-stubs/carray_broadcast.rb +1 -1
- data/yard-stubs/carray_cast.rb +88 -43
- data/yard-stubs/carray_conversion.rb +15 -2
- data/yard-stubs/carray_copy.rb +12 -5
- data/yard-stubs/carray_core.rb +3 -3
- data/yard-stubs/carray_count.rb +9 -0
- data/yard-stubs/carray_element.rb +10 -4
- data/yard-stubs/carray_generate.rb +21 -1
- data/yard-stubs/carray_lazy.rb +5 -1
- data/yard-stubs/carray_loop.rb +4 -1
- data/yard-stubs/carray_mask.rb +60 -12
- data/yard-stubs/carray_math.rb +52 -14
- data/yard-stubs/carray_median_percentile.rb +15 -8
- data/yard-stubs/carray_memory_view.rb +40 -23
- data/yard-stubs/carray_order.rb +61 -29
- data/yard-stubs/carray_random.rb +13 -4
- data/yard-stubs/carray_scatter.rb +21 -9
- data/yard-stubs/carray_segment.rb +72 -0
- data/yard-stubs/carray_slab.rb +10 -2
- data/yard-stubs/carray_sort.rb +49 -8
- data/yard-stubs/carray_test.rb +15 -3
- data/yard-stubs/carray_utils.rb +0 -26
- metadata +19 -5
- data/ext/ca_for_each_element.h +0 -239
- data/ext/carray_kernels_map.c +0 -3467
- data/lib/carray/arrow_tensor.rb +0 -401
- data/lib/carray/table.rb +0 -108
data/ext/carray_operator.c
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
|
|
13
13
|
#include "carray.h"
|
|
14
14
|
#include "carray_internal.h" /* ca_lazy_arena_* */
|
|
15
|
+
#include "ca_binop_dispatch.h" /* ca_binop_op_id_of_table */
|
|
16
|
+
#include "ca_monop_dispatch.h" /* ca_monop_op_id_of_table */
|
|
15
17
|
|
|
16
18
|
/* Operand scratch for a chunked / gathered kernel run.
|
|
17
19
|
|
|
@@ -30,6 +32,19 @@ ca_op_acquire_operand_scratch (CArray *ca, ca_size_t n_elements)
|
|
|
30
32
|
|
|
31
33
|
#include "ca_obj_face.h" /* ca_face_reconcile_comparison (comparison Face gate) */
|
|
32
34
|
|
|
35
|
+
/* A kernel table with no entry for `data_type`, the type the operator
|
|
36
|
+
would run in. Name the method and the type, or the Face, rather than
|
|
37
|
+
leave it to the table's stub, which knows neither. The eager operators
|
|
38
|
+
and the lazy builders refuse through here, so the two say the same. */
|
|
39
|
+
void
|
|
40
|
+
ca_op_refuse_unimplemented (VALUE operand, int8_t data_type, VALUE eclass)
|
|
41
|
+
{
|
|
42
|
+
const char *name = ca_calling_method_name();
|
|
43
|
+
ca_face_refuse_numeric(operand, name, eclass);
|
|
44
|
+
rb_raise(eclass, "%s: not defined for data_type %s",
|
|
45
|
+
name, ca_type_name[data_type]);
|
|
46
|
+
}
|
|
47
|
+
|
|
33
48
|
VALUE rb_mCAMath;
|
|
34
49
|
|
|
35
50
|
extern ca_binop_func_t ca_binop_mul[CA_NTYPE];
|
|
@@ -52,8 +67,8 @@ ca_zerodiv (void)
|
|
|
52
67
|
|
|
53
68
|
These three helpers (ca_chunk_inner_size / ca_chunk_compute_n /
|
|
54
69
|
ca_chunked_gather) are extern so ca_sweep_engine.c can reuse the same
|
|
55
|
-
chunk policy + region-gather mechanism for the
|
|
56
|
-
|
|
70
|
+
chunk policy + region-gather mechanism for the chunked slab family
|
|
71
|
+
(ca_call_cslab_*) without duplicating the implementation. */
|
|
57
72
|
|
|
58
73
|
#define CA_CHUNK_TARGET_ELEMENTS 4096
|
|
59
74
|
|
|
@@ -133,6 +148,397 @@ ca_chunked_gather (CArray *ca, ca_size_t off, ca_size_t n, void *dest)
|
|
|
133
148
|
ca_xfer_stride(ca, starts, counts, strides, dest, CA_XFER_GET);
|
|
134
149
|
}
|
|
135
150
|
|
|
151
|
+
/* The chunked kernel run shared by the binop / triop / bincmp drivers.
|
|
152
|
+
|
|
153
|
+
Each input operand is acquired once and read chunk by chunk:
|
|
154
|
+
- a scalar (step 0) is gathered once, or aliased
|
|
155
|
+
- an alias array is read through its own ptr at the chunk offset
|
|
156
|
+
- any other array is gathered into an arena scratch every chunk
|
|
157
|
+
|
|
158
|
+
The acquires, the gathers and the kernel itself can all raise (a view
|
|
159
|
+
that refuses the transfer, an object lane calling Ruby), so the run
|
|
160
|
+
goes through ca_lazy_arena_protect and the release -- scratches back to
|
|
161
|
+
the arena, aliases detached -- happens however it ends. */
|
|
162
|
+
|
|
163
|
+
#define CA_CHUNKED_MAX_OPERANDS 3
|
|
164
|
+
|
|
165
|
+
typedef struct ca_chunked_run {
|
|
166
|
+
int n_op;
|
|
167
|
+
CArray *ca[CA_CHUNKED_MAX_OPERANDS];
|
|
168
|
+
ca_size_t step[CA_CHUNKED_MAX_OPERANDS];
|
|
169
|
+
char *src[CA_CHUNKED_MAX_OPERANDS];
|
|
170
|
+
void *scratch[CA_CHUNKED_MAX_OPERANDS];
|
|
171
|
+
int per_chunk[CA_CHUNKED_MAX_OPERANDS];
|
|
172
|
+
int attached[CA_CHUNKED_MAX_OPERANDS];
|
|
173
|
+
ca_size_t n_total;
|
|
174
|
+
ca_size_t chunk_n;
|
|
175
|
+
/* Calls the kernel on cells [off, off + n) with p[k] the chunk of
|
|
176
|
+
operand k. */
|
|
177
|
+
void (*kernel)(struct ca_chunked_run *run, ca_size_t off, ca_size_t n,
|
|
178
|
+
char **p);
|
|
179
|
+
void *data; /* the driver's kernel table and output */
|
|
180
|
+
} ca_chunked_run_t;
|
|
181
|
+
|
|
182
|
+
static void
|
|
183
|
+
ca_chunked_run_init (ca_chunked_run_t *run, ca_size_t n_total,
|
|
184
|
+
ca_size_t chunk_n)
|
|
185
|
+
{
|
|
186
|
+
MEMZERO(run, ca_chunked_run_t, 1);
|
|
187
|
+
run->n_total = n_total;
|
|
188
|
+
run->chunk_n = chunk_n;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
static void
|
|
192
|
+
ca_chunked_run_add (ca_chunked_run_t *run, CArray *ca, ca_size_t step)
|
|
193
|
+
{
|
|
194
|
+
run->ca[run->n_op] = ca;
|
|
195
|
+
run->step[run->n_op] = step;
|
|
196
|
+
run->n_op++;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
static void
|
|
200
|
+
ca_chunked_run_acquire (ca_chunked_run_t *run, int k)
|
|
201
|
+
{
|
|
202
|
+
CArray *ca = run->ca[k];
|
|
203
|
+
if ( ca_attach_is_alias(ca) ) {
|
|
204
|
+
ca_attach(ca); /* window: released by ca_chunked_run on the way out */
|
|
205
|
+
run->attached[k] = 1;
|
|
206
|
+
run->src[k] = (char *) ca->ptr;
|
|
207
|
+
}
|
|
208
|
+
else if ( run->step[k] == 0 ) {
|
|
209
|
+
run->scratch[k] = ca_op_acquire_operand_scratch(ca, 1);
|
|
210
|
+
ca_xfer_all(ca, run->scratch[k], CA_XFER_GET);
|
|
211
|
+
run->src[k] = (char *) run->scratch[k];
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
run->scratch[k] = ca_op_acquire_operand_scratch(ca, run->chunk_n);
|
|
215
|
+
run->src[k] = (char *) run->scratch[k];
|
|
216
|
+
run->per_chunk[k] = 1;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
static VALUE
|
|
221
|
+
ca_chunked_run_body (VALUE arg)
|
|
222
|
+
{
|
|
223
|
+
ca_chunked_run_t *run = (ca_chunked_run_t *) arg;
|
|
224
|
+
char *p[CA_CHUNKED_MAX_OPERANDS];
|
|
225
|
+
ca_size_t off;
|
|
226
|
+
int k;
|
|
227
|
+
|
|
228
|
+
for ( k = 0; k < run->n_op; k++ ) {
|
|
229
|
+
ca_chunked_run_acquire(run, k);
|
|
230
|
+
}
|
|
231
|
+
for ( off = 0; off < run->n_total; off += run->chunk_n ) {
|
|
232
|
+
ca_size_t n = ( off + run->chunk_n > run->n_total )
|
|
233
|
+
? run->n_total - off : run->chunk_n;
|
|
234
|
+
for ( k = 0; k < run->n_op; k++ ) {
|
|
235
|
+
if ( run->per_chunk[k] ) {
|
|
236
|
+
ca_chunked_gather(run->ca[k], off, n, run->scratch[k]);
|
|
237
|
+
p[k] = run->src[k];
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
p[k] = run->src[k] + ( run->step[k] ? off * run->ca[k]->bytes : 0 );
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
run->kernel(run, off, n, p);
|
|
244
|
+
}
|
|
245
|
+
return Qnil;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
static VALUE
|
|
249
|
+
ca_chunked_run_release (VALUE arg)
|
|
250
|
+
{
|
|
251
|
+
ca_chunked_run_t *run = (ca_chunked_run_t *) arg;
|
|
252
|
+
int k;
|
|
253
|
+
for ( k = run->n_op - 1; k >= 0; k-- ) {
|
|
254
|
+
if ( run->scratch[k] ) {
|
|
255
|
+
ca_lazy_arena_release(run->scratch[k]);
|
|
256
|
+
run->scratch[k] = NULL;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
for ( k = run->n_op - 1; k >= 0; k-- ) {
|
|
260
|
+
if ( run->attached[k] ) {
|
|
261
|
+
ca_detach(run->ca[k]);
|
|
262
|
+
run->attached[k] = 0;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return Qnil;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
static void
|
|
269
|
+
ca_chunked_run (ca_chunked_run_t *run)
|
|
270
|
+
{
|
|
271
|
+
ca_lazy_arena_protect(ca_chunked_run_body, ca_chunked_run_release,
|
|
272
|
+
(VALUE) run);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/* ---- Kernels that can raise -----------------------------------------
|
|
276
|
+
|
|
277
|
+
An operator kernel runs with its alias operands attached. Most kernels
|
|
278
|
+
cannot raise and are called directly. Two kinds can:
|
|
279
|
+
|
|
280
|
+
- a kernel that can raise on a cell (ca_op_raises_on_cell: an object
|
|
281
|
+
lane, which calls Ruby for every cell; an integer division, remainder
|
|
282
|
+
or reciprocal by zero; an integer power of zero to a negative power)
|
|
283
|
+
- a data type the operator has no kernel for (the not_implement slot)
|
|
284
|
+
|
|
285
|
+
Those run through ca_op_guarded, which releases the operands the call
|
|
286
|
+
holds if the kernel raises. An operand the kernel writes in place (the
|
|
287
|
+
receiver of a bang form) is synced first, so the cells written reach
|
|
288
|
+
its parent as they would for an entity. */
|
|
289
|
+
|
|
290
|
+
static int
|
|
291
|
+
ca_op_is_integer (int8_t dt)
|
|
292
|
+
{
|
|
293
|
+
return ( dt >= CA_INT8 && dt <= CA_UINT64 );
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
int
|
|
297
|
+
ca_op_raises_on_cell (int kind, int op_id, int8_t dt)
|
|
298
|
+
{
|
|
299
|
+
if ( dt == CA_OBJECT ) {
|
|
300
|
+
return 1;
|
|
301
|
+
}
|
|
302
|
+
if ( ! ca_op_is_integer(dt) ) {
|
|
303
|
+
return 0;
|
|
304
|
+
}
|
|
305
|
+
switch ( kind ) {
|
|
306
|
+
case CA_OP_KIND_MONOP:
|
|
307
|
+
return op_id == CA_MONOP_RCP;
|
|
308
|
+
case CA_OP_KIND_BINOP:
|
|
309
|
+
switch ( op_id ) {
|
|
310
|
+
case CA_BINOP_DIV:
|
|
311
|
+
case CA_BINOP_MOD:
|
|
312
|
+
case CA_BINOP_QUO:
|
|
313
|
+
case CA_BINOP_FMOD:
|
|
314
|
+
case CA_BINOP_RCP_MUL:
|
|
315
|
+
case CA_BINOP_POW:
|
|
316
|
+
case CA_BINOP_IPOWER:
|
|
317
|
+
return 1;
|
|
318
|
+
default:
|
|
319
|
+
return 0;
|
|
320
|
+
}
|
|
321
|
+
default:
|
|
322
|
+
return 0;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/* The eager forms: a kernel that can raise on a cell
|
|
327
|
+
(ca_op_raises_on_cell), or a data type the operator has no kernel for. */
|
|
328
|
+
static int
|
|
329
|
+
ca_monop_may_raise (ca_monop_func_t func[], int8_t dt)
|
|
330
|
+
{
|
|
331
|
+
return func[dt] == ca_monop_not_implement
|
|
332
|
+
|| ca_op_raises_on_cell(CA_OP_KIND_MONOP, ca_monop_op_id_of_table(func), dt);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
static int
|
|
336
|
+
ca_binop_may_raise (ca_binop_func_t func[], int8_t dt)
|
|
337
|
+
{
|
|
338
|
+
return func[dt] == ca_binop_not_implement
|
|
339
|
+
|| ca_op_raises_on_cell(CA_OP_KIND_BINOP, ca_binop_op_id_of_table(func), dt);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
static int
|
|
343
|
+
ca_triop_may_raise (ca_triop_func_t func[], int8_t dt)
|
|
344
|
+
{
|
|
345
|
+
return func[dt] == ca_triop_not_implement
|
|
346
|
+
|| ca_op_raises_on_cell(CA_OP_KIND_TRIOP, -1, dt);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
static int
|
|
350
|
+
ca_moncmp_may_raise (ca_moncmp_func_t func[], int8_t dt)
|
|
351
|
+
{
|
|
352
|
+
return func[dt] == ca_moncmp_not_implement
|
|
353
|
+
|| ca_op_raises_on_cell(CA_OP_KIND_MONCMP, -1, dt);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
static int
|
|
357
|
+
ca_bincmp_may_raise (ca_bincmp_func_t func[], int8_t dt)
|
|
358
|
+
{
|
|
359
|
+
return func[dt] == ca_bincmp_not_implement
|
|
360
|
+
|| ca_op_raises_on_cell(CA_OP_KIND_BINCMP, -1, dt);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/* One kernel call, packed so that it can run under rb_ensure. */
|
|
364
|
+
|
|
365
|
+
typedef struct {
|
|
366
|
+
ca_monop_func_t f;
|
|
367
|
+
ca_size_t n; boolean8_t *m;
|
|
368
|
+
char *p1; ca_size_t i1;
|
|
369
|
+
char *p2; ca_size_t i2;
|
|
370
|
+
} ca_monop_call_t;
|
|
371
|
+
|
|
372
|
+
static VALUE
|
|
373
|
+
ca_monop_call (VALUE arg)
|
|
374
|
+
{
|
|
375
|
+
ca_monop_call_t *k = (ca_monop_call_t *) arg;
|
|
376
|
+
k->f(k->n, k->m, k->p1, k->i1, k->p2, k->i2);
|
|
377
|
+
return Qnil;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
typedef struct {
|
|
381
|
+
ca_binop_func_t f;
|
|
382
|
+
ca_size_t n; boolean8_t *m;
|
|
383
|
+
char *p1; ca_size_t i1;
|
|
384
|
+
char *p2; ca_size_t i2;
|
|
385
|
+
char *p3; ca_size_t i3;
|
|
386
|
+
} ca_binop_call_t;
|
|
387
|
+
|
|
388
|
+
static VALUE
|
|
389
|
+
ca_binop_call (VALUE arg)
|
|
390
|
+
{
|
|
391
|
+
ca_binop_call_t *k = (ca_binop_call_t *) arg;
|
|
392
|
+
k->f(k->n, k->m, k->p1, k->i1, k->p2, k->i2, k->p3, k->i3);
|
|
393
|
+
return Qnil;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
typedef struct {
|
|
397
|
+
ca_triop_func_t f;
|
|
398
|
+
ca_size_t n; boolean8_t *m;
|
|
399
|
+
char *p1; ca_size_t i1;
|
|
400
|
+
char *p2; ca_size_t i2;
|
|
401
|
+
char *p3; ca_size_t i3;
|
|
402
|
+
char *p4; ca_size_t i4;
|
|
403
|
+
} ca_triop_call_t;
|
|
404
|
+
|
|
405
|
+
static VALUE
|
|
406
|
+
ca_triop_call (VALUE arg)
|
|
407
|
+
{
|
|
408
|
+
ca_triop_call_t *k = (ca_triop_call_t *) arg;
|
|
409
|
+
k->f(k->n, k->m, k->p1, k->i1, k->p2, k->i2, k->p3, k->i3, k->p4, k->i4);
|
|
410
|
+
return Qnil;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
typedef struct {
|
|
414
|
+
ca_moncmp_func_t f;
|
|
415
|
+
ca_size_t n; boolean8_t *m;
|
|
416
|
+
char *p1; ca_size_t i1;
|
|
417
|
+
boolean8_t *p2; ca_size_t i2;
|
|
418
|
+
} ca_moncmp_call_t;
|
|
419
|
+
|
|
420
|
+
static VALUE
|
|
421
|
+
ca_moncmp_call (VALUE arg)
|
|
422
|
+
{
|
|
423
|
+
ca_moncmp_call_t *k = (ca_moncmp_call_t *) arg;
|
|
424
|
+
k->f(k->n, k->m, k->p1, k->i1, k->p2, k->i2);
|
|
425
|
+
return Qnil;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
typedef struct {
|
|
429
|
+
ca_bincmp_func_t f;
|
|
430
|
+
ca_size_t n; boolean8_t *m;
|
|
431
|
+
char *p1; ca_size_t b1; ca_size_t i1;
|
|
432
|
+
char *p2; ca_size_t b2; ca_size_t i2;
|
|
433
|
+
char *p3; ca_size_t b3; ca_size_t i3;
|
|
434
|
+
double tol;
|
|
435
|
+
} ca_bincmp_call_t;
|
|
436
|
+
|
|
437
|
+
static VALUE
|
|
438
|
+
ca_bincmp_call (VALUE arg)
|
|
439
|
+
{
|
|
440
|
+
ca_bincmp_call_t *k = (ca_bincmp_call_t *) arg;
|
|
441
|
+
k->f(k->n, k->m, k->p1, k->b1, k->i1, k->p2, k->b2, k->i2,
|
|
442
|
+
k->p3, k->b3, k->i3, k->tol);
|
|
443
|
+
return Qnil;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
typedef struct {
|
|
447
|
+
VALUE (*run)(VALUE);
|
|
448
|
+
VALUE arg;
|
|
449
|
+
int done;
|
|
450
|
+
int n;
|
|
451
|
+
CArray **held;
|
|
452
|
+
const char *mode; /* per held array: 'r' read, 'w' written in place */
|
|
453
|
+
} ca_op_guard_t;
|
|
454
|
+
|
|
455
|
+
static VALUE
|
|
456
|
+
ca_op_guard_body (VALUE varg)
|
|
457
|
+
{
|
|
458
|
+
ca_op_guard_t *g = (ca_op_guard_t *) varg;
|
|
459
|
+
g->run(g->arg);
|
|
460
|
+
g->done = 1;
|
|
461
|
+
return Qnil;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
static VALUE
|
|
465
|
+
ca_op_guard_sync (VALUE varg)
|
|
466
|
+
{
|
|
467
|
+
ca_sync((void *) varg);
|
|
468
|
+
return Qnil;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/* Runs only after a raise: the caller closes the operands itself when the
|
|
472
|
+
kernel returns. rb_ensure keeps the kernel's exception across this, so
|
|
473
|
+
a sync failing in here is dropped rather than let replace it. */
|
|
474
|
+
static VALUE
|
|
475
|
+
ca_op_guard_release (VALUE varg)
|
|
476
|
+
{
|
|
477
|
+
ca_op_guard_t *g = (ca_op_guard_t *) varg;
|
|
478
|
+
int k;
|
|
479
|
+
if ( g->done ) {
|
|
480
|
+
return Qnil;
|
|
481
|
+
}
|
|
482
|
+
for ( k = g->n - 1; k >= 0; k-- ) {
|
|
483
|
+
CArray *ca = g->held[k];
|
|
484
|
+
if ( ! ca || ! ca->ptr ) {
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
if ( g->mode[k] == 'w' ) {
|
|
488
|
+
int tag = 0;
|
|
489
|
+
/* protect: a sync that raises is dropped so that every held array is detached */
|
|
490
|
+
rb_protect(ca_op_guard_sync, (VALUE) ca, &tag);
|
|
491
|
+
rb_set_errinfo(Qnil);
|
|
492
|
+
}
|
|
493
|
+
ca_detach(ca);
|
|
494
|
+
}
|
|
495
|
+
return Qnil;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/* Runs the kernel packed in `call` with `held` attached; see above. NULL
|
|
499
|
+
entries in held are skipped. */
|
|
500
|
+
static void
|
|
501
|
+
ca_op_guarded (VALUE (*run)(VALUE), void *call,
|
|
502
|
+
int n, CArray **held, const char *mode)
|
|
503
|
+
{
|
|
504
|
+
ca_op_guard_t g;
|
|
505
|
+
g.run = run;
|
|
506
|
+
g.arg = (VALUE) call;
|
|
507
|
+
g.done = 0;
|
|
508
|
+
g.n = n;
|
|
509
|
+
g.held = held;
|
|
510
|
+
g.mode = mode;
|
|
511
|
+
rb_ensure(ca_op_guard_body, (VALUE) &g, ca_op_guard_release, (VALUE) &g);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/* ORs one operand's mask into ma through an arena scratch. The scratch
|
|
515
|
+
is acquired inside the bracket: the transfer below opens a bracket of
|
|
516
|
+
its own, and an entry from depth 0 treats every slot as free. */
|
|
517
|
+
typedef struct {
|
|
518
|
+
void *scratch; /* first: released by ca_lazy_arena_release_held */
|
|
519
|
+
CArray *mask;
|
|
520
|
+
boolean8_t *ma;
|
|
521
|
+
ca_size_t elements;
|
|
522
|
+
} ca_mask_overlay_or_t;
|
|
523
|
+
|
|
524
|
+
static VALUE
|
|
525
|
+
ca_mask_overlay_or_body (VALUE arg)
|
|
526
|
+
{
|
|
527
|
+
ca_mask_overlay_or_t *o = (ca_mask_overlay_or_t *) arg;
|
|
528
|
+
boolean8_t *ms;
|
|
529
|
+
/* Locals, not o->ma / o->elements: a store through a boolean8_t may alias
|
|
530
|
+
*o, so reading them through o reloads both every iteration and keeps
|
|
531
|
+
the loop from vectorizing. */
|
|
532
|
+
boolean8_t *ma = o->ma;
|
|
533
|
+
ca_size_t elements = o->elements;
|
|
534
|
+
ca_size_t j;
|
|
535
|
+
o->scratch = ca_lazy_arena_acquire(elements);
|
|
536
|
+
ms = (boolean8_t *) o->scratch;
|
|
537
|
+
ca_xfer_all(o->mask, o->scratch, CA_XFER_GET);
|
|
538
|
+
for ( j = 0; j < elements; j++ ) ma[j] |= ms[j];
|
|
539
|
+
return Qnil;
|
|
540
|
+
}
|
|
541
|
+
|
|
136
542
|
/* Operand mask overlay without calling ca_attach on the operand masks.
|
|
137
543
|
|
|
138
544
|
`ca_copy_mask_overlay` (carray_mask.c) attaches each operand mask via
|
|
@@ -145,9 +551,10 @@ ca_chunked_gather (CArray *ca, ca_size_t off, ca_size_t n, void *dest)
|
|
|
145
551
|
Memory peak: one mask scratch at a time (= elements bytes, 1 B/cell),
|
|
146
552
|
released between operands.
|
|
147
553
|
|
|
148
|
-
CAREFUL:
|
|
149
|
-
|
|
150
|
-
|
|
554
|
+
CAREFUL: ca_out's mask is written through ca_out->mask->ptr, so ca_out
|
|
555
|
+
is an entity (a fresh one from ca_template_safe) or an attached view.
|
|
556
|
+
Each operand other than a scalar has ca_out's number of elements
|
|
557
|
+
(ArgumentError otherwise); at most 8 operands.
|
|
151
558
|
|
|
152
559
|
This gathers the full mask once; a per-chunk mask gather inside the
|
|
153
560
|
chunked branch is a possible future optimisation. */
|
|
@@ -164,10 +571,22 @@ ca_mask_overlay_safe (CArray *ca_out, int n, ...)
|
|
|
164
571
|
va_start(ap, n);
|
|
165
572
|
for ( i = 0; i < n; i++ ) {
|
|
166
573
|
slist[i] = va_arg(ap, CArray *);
|
|
167
|
-
if ( slist[i] && ca_has_mask(slist[i]) ) any_mask = 1;
|
|
168
574
|
}
|
|
169
575
|
va_end(ap);
|
|
170
576
|
|
|
577
|
+
/* An array operand's mask is gathered over ca_out's cells. */
|
|
578
|
+
for ( i = 0; i < n; i++ ) {
|
|
579
|
+
CArray *cs = slist[i];
|
|
580
|
+
if ( cs && ! ca_is_scalar(cs) && cs->elements != ca_out->elements ) {
|
|
581
|
+
rb_raise(rb_eArgError,
|
|
582
|
+
"mask operand has %lld elements, the output %lld",
|
|
583
|
+
(long long) cs->elements, (long long) ca_out->elements);
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
for ( i = 0; i < n; i++ ) {
|
|
587
|
+
if ( slist[i] && ca_has_mask(slist[i]) ) any_mask = 1;
|
|
588
|
+
}
|
|
589
|
+
|
|
171
590
|
if ( ! any_mask ) return;
|
|
172
591
|
|
|
173
592
|
ca_update_mask(ca_out);
|
|
@@ -180,7 +599,6 @@ ca_mask_overlay_safe (CArray *ca_out, int n, ...)
|
|
|
180
599
|
|
|
181
600
|
boolean8_t *ma = (boolean8_t *) ca_out->mask->ptr;
|
|
182
601
|
ca_size_t elements = ca_out->elements;
|
|
183
|
-
ca_size_t j;
|
|
184
602
|
|
|
185
603
|
/* Fresh mask → zero-init. Existing mask (= bang variant where
|
|
186
604
|
ca_out IS one of the operands) → preserve as initial accumulator
|
|
@@ -200,16 +618,54 @@ ca_mask_overlay_safe (CArray *ca_out, int n, ...)
|
|
|
200
618
|
ca_xfer_all(cs->mask, &bit, CA_XFER_GET);
|
|
201
619
|
if ( bit ) memset(ma, 1, elements);
|
|
202
620
|
} else {
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
621
|
+
ca_mask_overlay_or_t o;
|
|
622
|
+
o.scratch = NULL;
|
|
623
|
+
o.mask = cs->mask;
|
|
624
|
+
o.ma = ma;
|
|
625
|
+
o.elements = elements;
|
|
626
|
+
ca_lazy_arena_protect(ca_mask_overlay_or_body,
|
|
627
|
+
ca_lazy_arena_release_held, (VALUE) &o);
|
|
208
628
|
}
|
|
209
629
|
}
|
|
210
630
|
}
|
|
211
631
|
}
|
|
212
632
|
|
|
633
|
+
/* ca_mask_overlay_safe into a bang operation's self, which is attached:
|
|
634
|
+
reading an operand's mask can call Ruby (a CAObject's mask hooks) and
|
|
635
|
+
raise, and self is then detached before the raise goes on. Nothing has
|
|
636
|
+
been written to self's data yet, so there is nothing to sync. */
|
|
637
|
+
typedef struct {
|
|
638
|
+
CArray *out;
|
|
639
|
+
CArray *src[3];
|
|
640
|
+
int n;
|
|
641
|
+
} ca_overlay_args_t;
|
|
642
|
+
|
|
643
|
+
static VALUE
|
|
644
|
+
ca_overlay_attached_body (VALUE arg)
|
|
645
|
+
{
|
|
646
|
+
ca_overlay_args_t *a = (ca_overlay_args_t *) arg;
|
|
647
|
+
if ( a->n == 2 ) {
|
|
648
|
+
ca_mask_overlay_safe(a->out, 2, a->src[0], a->src[1]);
|
|
649
|
+
}
|
|
650
|
+
else {
|
|
651
|
+
ca_mask_overlay_safe(a->out, 3, a->src[0], a->src[1], a->src[2]);
|
|
652
|
+
}
|
|
653
|
+
return Qnil;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
static void
|
|
657
|
+
ca_mask_overlay_attached (CArray *out, int n,
|
|
658
|
+
CArray *s0, CArray *s1, CArray *s2)
|
|
659
|
+
{
|
|
660
|
+
ca_overlay_args_t a;
|
|
661
|
+
a.out = out;
|
|
662
|
+
a.n = n;
|
|
663
|
+
a.src[0] = s0;
|
|
664
|
+
a.src[1] = s1;
|
|
665
|
+
a.src[2] = s2;
|
|
666
|
+
ca_fill_or_detach(1, &out, ca_overlay_attached_body, (VALUE) &a);
|
|
667
|
+
}
|
|
668
|
+
|
|
213
669
|
/* Monop driver. ca1 is input-only (the driver does not attach it); ca2
|
|
214
670
|
is the output (a new entity, attach legit). fast = ca_attach_is_alias(ca1)
|
|
215
671
|
→ 1-shot; slow = ALLOCV + ca_xfer_all without ca_func[X].attach. */
|
|
@@ -229,18 +685,33 @@ rb_ca_call_monop (VALUE self, ca_monop_func_t func[])
|
|
|
229
685
|
self = rb_ca_wrap_readonly(self, INT2NUM(CA_INT64));
|
|
230
686
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
231
687
|
}
|
|
688
|
+
if ( func[ca1->data_type] == ca_monop_not_implement ) {
|
|
689
|
+
ca_op_refuse_unimplemented(self, ca1->data_type, rb_eCADataTypeError);
|
|
690
|
+
}
|
|
232
691
|
|
|
233
692
|
ca2 = ca_has_mask(ca1) ? ca_template_safe(ca1) : ca_template(ca1);
|
|
234
693
|
out = ca_wrap_struct(ca2);
|
|
235
694
|
|
|
236
695
|
ca_mask_overlay_safe(ca2, 1, ca1);
|
|
237
696
|
|
|
697
|
+
/* No cell to compute: the strided walks below hand the kernel a first
|
|
698
|
+
row before they look at the outer counts. */
|
|
699
|
+
if ( ca2->elements == 0 ) return out;
|
|
700
|
+
|
|
238
701
|
if ( ca_attach_is_alias(ca1) ) {
|
|
239
|
-
|
|
240
|
-
func[ca1->data_type]
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
702
|
+
ca_monop_call_t k;
|
|
703
|
+
k.f = func[ca1->data_type];
|
|
704
|
+
k.n = ca1->elements;
|
|
705
|
+
k.m = ( ca2->mask ) ? (boolean8_t *)ca2->mask->ptr : NULL;
|
|
706
|
+
k.p2 = ca2->ptr; k.i2 = 1;
|
|
707
|
+
ca_attach(ca1); /* window: the kernel runs through ca_op_guarded when it can raise */
|
|
708
|
+
k.p1 = ca1->ptr; k.i1 = 1;
|
|
709
|
+
if ( ca_monop_may_raise(func, ca1->data_type) ) {
|
|
710
|
+
ca_op_guarded(ca_monop_call, &k, 1, &ca1, "r");
|
|
711
|
+
}
|
|
712
|
+
else {
|
|
713
|
+
ca_monop_call((VALUE) &k);
|
|
714
|
+
}
|
|
244
715
|
ca_detach(ca1);
|
|
245
716
|
}
|
|
246
717
|
else {
|
|
@@ -301,6 +772,7 @@ rb_ca_call_monop (VALUE self, ca_monop_func_t func[])
|
|
|
301
772
|
ca2->ptr, 1);
|
|
302
773
|
}
|
|
303
774
|
else {
|
|
775
|
+
/* empty: the function returns on a result with no cells, above */
|
|
304
776
|
while ( 1 ) {
|
|
305
777
|
ca_size_t off1 = e1_base;
|
|
306
778
|
for ( k = 0; k < ndim - 1; k++ ) {
|
|
@@ -340,6 +812,8 @@ rb_ca_call_monop (VALUE self, ca_monop_func_t func[])
|
|
|
340
812
|
}
|
|
341
813
|
}
|
|
342
814
|
|
|
815
|
+
/* the INT64 view of a boolean operand is reached only through ca1 */
|
|
816
|
+
RB_GC_GUARD(self);
|
|
343
817
|
return out;
|
|
344
818
|
}
|
|
345
819
|
|
|
@@ -351,14 +825,26 @@ rb_ca_call_monop_bang (VALUE self, ca_monop_func_t func[])
|
|
|
351
825
|
rb_ca_modify(self);
|
|
352
826
|
|
|
353
827
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
828
|
+
if ( func[ca1->data_type] == ca_monop_not_implement ) {
|
|
829
|
+
ca_op_refuse_unimplemented(self, ca1->data_type, rb_eCADataTypeError);
|
|
830
|
+
}
|
|
354
831
|
|
|
355
|
-
ca_attach(ca1);
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
832
|
+
ca_attach(ca1); /* window: the kernel runs through ca_op_guarded when it can raise */
|
|
833
|
+
{
|
|
834
|
+
ca_monop_call_t k;
|
|
835
|
+
k.f = func[ca1->data_type];
|
|
836
|
+
k.n = ca1->elements;
|
|
837
|
+
k.m = ( ca1->mask ) ? (boolean8_t *)ca1->mask->ptr : NULL;
|
|
838
|
+
k.p1 = ca1->ptr; k.i1 = 1;
|
|
839
|
+
k.p2 = ca1->ptr; k.i2 = 1;
|
|
840
|
+
if ( ca_monop_may_raise(func, ca1->data_type) ) {
|
|
841
|
+
ca_op_guarded(ca_monop_call, &k, 1, &ca1, "w");
|
|
842
|
+
}
|
|
843
|
+
else {
|
|
844
|
+
ca_monop_call((VALUE) &k);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
ca_sync_detach(ca1);
|
|
362
848
|
|
|
363
849
|
return self;
|
|
364
850
|
}
|
|
@@ -380,9 +866,7 @@ rb_ca_call_monop_typed (VALUE self, ca_monop_func_t func[],
|
|
|
380
866
|
|
|
381
867
|
out_dt = out_data_types[ca1->data_type];
|
|
382
868
|
if ( out_dt < 0 ) {
|
|
383
|
-
|
|
384
|
-
"data_type-changing monop not implemented for input data_type %d",
|
|
385
|
-
ca1->data_type);
|
|
869
|
+
ca_op_refuse_unimplemented(self, ca1->data_type, rb_eCADataTypeError);
|
|
386
870
|
}
|
|
387
871
|
|
|
388
872
|
/* Allocate output with the per-op output data_type. Same shape as input. */
|
|
@@ -392,12 +876,24 @@ rb_ca_call_monop_typed (VALUE self, ca_monop_func_t func[],
|
|
|
392
876
|
/* Same fast/slow pattern as rb_ca_call_monop. ca1 input-only, ca2 output. */
|
|
393
877
|
ca_mask_overlay_safe(ca2, 1, ca1);
|
|
394
878
|
|
|
879
|
+
/* No cell to compute: the strided walks below hand the kernel a first
|
|
880
|
+
row before they look at the outer counts. */
|
|
881
|
+
if ( ca2->elements == 0 ) return out;
|
|
882
|
+
|
|
395
883
|
if ( ca_attach_is_alias(ca1) ) {
|
|
396
|
-
|
|
397
|
-
func[ca1->data_type]
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
884
|
+
ca_monop_call_t k;
|
|
885
|
+
k.f = func[ca1->data_type];
|
|
886
|
+
k.n = ca1->elements;
|
|
887
|
+
k.m = ( ca2->mask ) ? (boolean8_t *)ca2->mask->ptr : NULL;
|
|
888
|
+
k.p2 = ca2->ptr; k.i2 = 1;
|
|
889
|
+
ca_attach(ca1); /* window: the kernel runs through ca_op_guarded when it can raise */
|
|
890
|
+
k.p1 = ca1->ptr; k.i1 = 1;
|
|
891
|
+
if ( ca_monop_may_raise(func, ca1->data_type) ) {
|
|
892
|
+
ca_op_guarded(ca_monop_call, &k, 1, &ca1, "r");
|
|
893
|
+
}
|
|
894
|
+
else {
|
|
895
|
+
ca_monop_call((VALUE) &k);
|
|
896
|
+
}
|
|
401
897
|
ca_detach(ca1);
|
|
402
898
|
}
|
|
403
899
|
else {
|
|
@@ -454,6 +950,7 @@ rb_ca_call_monop_typed (VALUE self, ca_monop_func_t func[],
|
|
|
454
950
|
ca2->ptr, 1);
|
|
455
951
|
}
|
|
456
952
|
else {
|
|
953
|
+
/* empty: the function returns on a result with no cells, above */
|
|
457
954
|
while ( 1 ) {
|
|
458
955
|
ca_size_t off1 = e1_base;
|
|
459
956
|
for ( k = 0; k < ndim - 1; k++ ) {
|
|
@@ -562,7 +1059,7 @@ VALUE
|
|
|
562
1059
|
ca_kleene_bool_fixup (VALUE vout, VALUE vself, VALUE vother, int is_or)
|
|
563
1060
|
{
|
|
564
1061
|
CArray *out, *a, *b;
|
|
565
|
-
volatile VALUE va, vb;
|
|
1062
|
+
volatile VALUE va, vb, holder;
|
|
566
1063
|
boolean8_t *ov, *om, *av, *am, *bv, *bm;
|
|
567
1064
|
ca_size_t n, i;
|
|
568
1065
|
|
|
@@ -582,9 +1079,12 @@ ca_kleene_bool_fixup (VALUE vout, VALUE vself, VALUE vother, int is_or)
|
|
|
582
1079
|
TypedData_Get_Struct(va, CArray, &carray_data_type, a);
|
|
583
1080
|
TypedData_Get_Struct(vb, CArray, &carray_data_type, b);
|
|
584
1081
|
|
|
1082
|
+
/* One temporary of Ruby's for all four: reading an operand can raise. */
|
|
585
1083
|
n = out->elements;
|
|
586
|
-
av =
|
|
587
|
-
|
|
1084
|
+
av = ALLOCV_N(boolean8_t, holder, 4 * (n > 0 ? n : 1));
|
|
1085
|
+
am = av + n;
|
|
1086
|
+
bv = am + n;
|
|
1087
|
+
bm = bv + n;
|
|
588
1088
|
kleene_gather_bool(a, av, am, n);
|
|
589
1089
|
kleene_gather_bool(b, bv, bm, n);
|
|
590
1090
|
|
|
@@ -608,7 +1108,7 @@ ca_kleene_bool_fixup (VALUE vout, VALUE vself, VALUE vother, int is_or)
|
|
|
608
1108
|
}
|
|
609
1109
|
}
|
|
610
1110
|
|
|
611
|
-
|
|
1111
|
+
ALLOCV_END(holder);
|
|
612
1112
|
return vout;
|
|
613
1113
|
}
|
|
614
1114
|
|
|
@@ -622,6 +1122,71 @@ ca_kleene_bool_fixup (VALUE vout, VALUE vself, VALUE vother, int is_or)
|
|
|
622
1122
|
CATile expansion, attach-hostile roots, and unattachable test
|
|
623
1123
|
fixtures working.
|
|
624
1124
|
Output `ca3` is always a new entity (write target, attach legitimate). */
|
|
1125
|
+
typedef struct {
|
|
1126
|
+
ca_binop_func_t *func;
|
|
1127
|
+
int8_t dt;
|
|
1128
|
+
CArray *out;
|
|
1129
|
+
ca_size_t step_out;
|
|
1130
|
+
} ca_binop_chunk_t;
|
|
1131
|
+
|
|
1132
|
+
static void
|
|
1133
|
+
ca_binop_chunk_kernel (ca_chunked_run_t *run, ca_size_t off, ca_size_t n,
|
|
1134
|
+
char **p)
|
|
1135
|
+
{
|
|
1136
|
+
ca_binop_chunk_t *d = (ca_binop_chunk_t *) run->data;
|
|
1137
|
+
ca_size_t o = d->step_out ? off : 0;
|
|
1138
|
+
d->func[d->dt](n,
|
|
1139
|
+
d->out->mask ? ((boolean8_t *) d->out->mask->ptr) + o : NULL,
|
|
1140
|
+
p[0], run->step[0],
|
|
1141
|
+
p[1], run->step[1],
|
|
1142
|
+
(char *) d->out->ptr + o * d->out->bytes, d->step_out);
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
/* A boolean array as an object array of true / false. The ordinary cast
|
|
1146
|
+
gives 1 / 0, which a logical operator on an object array reads as two
|
|
1147
|
+
true values (0 is true in Ruby). Masked cells stay masked. */
|
|
1148
|
+
static VALUE
|
|
1149
|
+
ca_boolean_as_truth_objects (VALUE vb)
|
|
1150
|
+
{
|
|
1151
|
+
CArray *cb;
|
|
1152
|
+
ca_size_t i;
|
|
1153
|
+
VALUE out;
|
|
1154
|
+
TypedData_Get_Struct(vb, CArray, &carray_data_type, cb);
|
|
1155
|
+
out = rb_carray_new(CA_OBJECT, cb->ndim, cb->dim, 0, NULL);
|
|
1156
|
+
for ( i = 0; i < cb->elements; i++ ) {
|
|
1157
|
+
rb_ca_store_addr(out, i, rb_ca_fetch_addr(vb, i));
|
|
1158
|
+
}
|
|
1159
|
+
return out;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
static int
|
|
1163
|
+
ca_value_is_type (VALUE v, int8_t data_type)
|
|
1164
|
+
{
|
|
1165
|
+
CArray *ca;
|
|
1166
|
+
if ( ! rb_obj_is_carray(v) ) return 0;
|
|
1167
|
+
TypedData_Get_Struct(v, CArray, &carray_data_type, ca);
|
|
1168
|
+
return ca->data_type == data_type;
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/* A logical operator (`logical`: one with a boolean kernel) between a
|
|
1172
|
+
boolean array and an object array reads the boolean cells as true /
|
|
1173
|
+
false. The eager binop and the lazy builder both call this before
|
|
1174
|
+
anything else decides the operands' types, so both read them alike. */
|
|
1175
|
+
void
|
|
1176
|
+
ca_binop_read_boolean_as_truth (int logical, volatile VALUE *pa,
|
|
1177
|
+
volatile VALUE *pb)
|
|
1178
|
+
{
|
|
1179
|
+
if ( ! logical ) {
|
|
1180
|
+
return;
|
|
1181
|
+
}
|
|
1182
|
+
if ( ca_value_is_type(*pa, CA_BOOLEAN) && ca_value_is_type(*pb, CA_OBJECT) ) {
|
|
1183
|
+
*pa = ca_boolean_as_truth_objects(*pa);
|
|
1184
|
+
}
|
|
1185
|
+
else if ( ca_value_is_type(*pb, CA_BOOLEAN) && ca_value_is_type(*pa, CA_OBJECT) ) {
|
|
1186
|
+
*pb = ca_boolean_as_truth_objects(*pb);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
625
1190
|
VALUE
|
|
626
1191
|
rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
627
1192
|
ca_binop_func_t func[])
|
|
@@ -633,6 +1198,9 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
633
1198
|
ca_size_t i1, i2, i3;
|
|
634
1199
|
int fast_path;
|
|
635
1200
|
|
|
1201
|
+
ca_binop_read_boolean_as_truth(func[CA_BOOLEAN] != ca_binop_not_implement,
|
|
1202
|
+
&self, &other);
|
|
1203
|
+
|
|
636
1204
|
/* do implicit casting */
|
|
637
1205
|
rb_ca_cast_self_or_other(&self, &other);
|
|
638
1206
|
|
|
@@ -652,6 +1220,10 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
652
1220
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
653
1221
|
TypedData_Get_Struct(other, CArray, &carray_data_type, ca2);
|
|
654
1222
|
}
|
|
1223
|
+
if ( func[ca1->data_type] == ca_binop_not_implement ) {
|
|
1224
|
+
ca_op_refuse_unimplemented(ca_is_face(ca1) ? self : other, ca1->data_type,
|
|
1225
|
+
rb_eCADataTypeError);
|
|
1226
|
+
}
|
|
655
1227
|
|
|
656
1228
|
self_is_scalar = RTEST(rb_obj_is_cscalar(self));
|
|
657
1229
|
other_is_scalar = RTEST(rb_obj_is_cscalar(other));
|
|
@@ -694,17 +1266,31 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
694
1266
|
input-only operand does not raise. */
|
|
695
1267
|
ca_mask_overlay_safe(ca3, 2, ca1, ca2);
|
|
696
1268
|
|
|
1269
|
+
/* No cell to compute: the strided walks below hand the kernel a first
|
|
1270
|
+
row before they look at the outer counts. */
|
|
1271
|
+
if ( ca3->elements == 0 ) return out;
|
|
1272
|
+
|
|
697
1273
|
/* path decision: alias-attachable both → fast path */
|
|
698
1274
|
fast_path = ca_attach_is_alias(ca1) && ca_attach_is_alias(ca2);
|
|
699
1275
|
|
|
700
1276
|
if ( fast_path ) {
|
|
701
|
-
/* FAST PATH:
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
1277
|
+
/* FAST PATH: both operands alias. */
|
|
1278
|
+
ca_binop_call_t k;
|
|
1279
|
+
CArray *held[2];
|
|
1280
|
+
k.f = func[ca1->data_type];
|
|
1281
|
+
k.n = n_kernel;
|
|
1282
|
+
k.m = ( ca3->mask ) ? (boolean8_t *) ca3->mask->ptr : NULL;
|
|
1283
|
+
k.p3 = ca3->ptr; k.i3 = i3;
|
|
1284
|
+
ca_attach_n(2, ca1, ca2); /* window: the kernel runs through ca_op_guarded when it can raise */
|
|
1285
|
+
k.p1 = ca1->ptr; k.i1 = i1;
|
|
1286
|
+
k.p2 = ca2->ptr; k.i2 = i2;
|
|
1287
|
+
if ( ca_binop_may_raise(func, ca1->data_type) ) {
|
|
1288
|
+
held[0] = ca1; held[1] = ca2;
|
|
1289
|
+
ca_op_guarded(ca_binop_call, &k, 2, held, "rr");
|
|
1290
|
+
}
|
|
1291
|
+
else {
|
|
1292
|
+
ca_binop_call((VALUE) &k);
|
|
1293
|
+
}
|
|
708
1294
|
ca_detach_n(2, ca1, ca2);
|
|
709
1295
|
}
|
|
710
1296
|
else if ( ca1 == ca2 ) {
|
|
@@ -744,13 +1330,8 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
744
1330
|
- both array & both non-alias (= mt + mt2) → CHUNKED (peak 2x win)
|
|
745
1331
|
- one non-alias array + one alias/scalar (= mt + 3.14, mt + entity)
|
|
746
1332
|
→ 1-shot ALLOCV */
|
|
747
|
-
char *p1_src = NULL, *p2_src = NULL;
|
|
748
|
-
void *s1_arena = NULL, *s2_arena = NULL;
|
|
749
|
-
int gather_per_chunk1 = 0, gather_per_chunk2 = 0;
|
|
750
|
-
int attached1 = 0, attached2 = 0;
|
|
751
1333
|
int8_t dt = ca1->data_type;
|
|
752
1334
|
ca_size_t chunk_n;
|
|
753
|
-
ca_size_t off;
|
|
754
1335
|
int nonalias_arrays;
|
|
755
1336
|
int use_chunked;
|
|
756
1337
|
|
|
@@ -770,7 +1351,7 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
770
1351
|
Output ca3 is contig entity; per-row output offset = row_idx *
|
|
771
1352
|
inner_count. The use_chunked / ALLOCV blocks below remain as a
|
|
772
1353
|
fallback (dead on the success path) but fire for views outside the
|
|
773
|
-
CAStride+entity family (CASelect /
|
|
1354
|
+
CAStride+entity family (CASelect / CAGrid / CAReduce
|
|
774
1355
|
etc.). */
|
|
775
1356
|
if ( i1 == 1 && i2 == 1 && nonalias_arrays >= 1 ) {
|
|
776
1357
|
extern ca_operation_function_t ca_stride_func;
|
|
@@ -859,6 +1440,7 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
859
1440
|
ca3->ptr, i3);
|
|
860
1441
|
}
|
|
861
1442
|
else {
|
|
1443
|
+
/* empty: the function returns on a result with no cells, above */
|
|
862
1444
|
while ( 1 ) {
|
|
863
1445
|
ca_size_t off1 = e1_base, off2 = e2_base;
|
|
864
1446
|
for ( k = 0; k < ndim - 1; k++ ) {
|
|
@@ -895,24 +1477,40 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
895
1477
|
int a1 = 0, a2 = 0;
|
|
896
1478
|
(void) h1; (void) h2;
|
|
897
1479
|
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
} else {
|
|
1480
|
+
/* Gathers first: a gather can raise, and nothing is attached yet. */
|
|
1481
|
+
if ( ! ca_attach_is_alias(ca1) ) {
|
|
901
1482
|
p1 = ALLOCV_N(char, h1, ca1->elements * ca1->bytes);
|
|
902
1483
|
ca_xfer_all(ca1, p1, CA_XFER_GET);
|
|
903
1484
|
}
|
|
904
|
-
if ( ca_attach_is_alias(ca2) ) {
|
|
905
|
-
ca_attach(ca2); p2 = (char *) ca2->ptr; a2 = 1;
|
|
906
|
-
} else {
|
|
1485
|
+
if ( ! ca_attach_is_alias(ca2) ) {
|
|
907
1486
|
p2 = ALLOCV_N(char, h2, ca2->elements * ca2->bytes);
|
|
908
1487
|
ca_xfer_all(ca2, p2, CA_XFER_GET);
|
|
909
1488
|
}
|
|
1489
|
+
if ( ca_attach_is_alias(ca1) ) {
|
|
1490
|
+
ca_attach(ca1); p1 = (char *) ca1->ptr; a1 = 1; /* window: an alias attach, cannot raise */
|
|
1491
|
+
}
|
|
1492
|
+
if ( ca_attach_is_alias(ca2) ) {
|
|
1493
|
+
ca_attach(ca2); p2 = (char *) ca2->ptr; a2 = 1; /* window: as above */
|
|
1494
|
+
}
|
|
910
1495
|
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
1496
|
+
{
|
|
1497
|
+
ca_binop_call_t k;
|
|
1498
|
+
CArray *held[2];
|
|
1499
|
+
k.f = func[dt];
|
|
1500
|
+
k.n = n_kernel;
|
|
1501
|
+
k.m = ( ca3->mask ) ? (boolean8_t *) ca3->mask->ptr : NULL;
|
|
1502
|
+
k.p1 = p1; k.i1 = i1;
|
|
1503
|
+
k.p2 = p2; k.i2 = i2;
|
|
1504
|
+
k.p3 = ca3->ptr; k.i3 = i3;
|
|
1505
|
+
if ( ca_binop_may_raise(func, dt) ) {
|
|
1506
|
+
held[0] = a1 ? ca1 : NULL;
|
|
1507
|
+
held[1] = a2 ? ca2 : NULL;
|
|
1508
|
+
ca_op_guarded(ca_binop_call, &k, 2, held, "rr");
|
|
1509
|
+
}
|
|
1510
|
+
else {
|
|
1511
|
+
ca_binop_call((VALUE) &k);
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
916
1514
|
|
|
917
1515
|
if ( a2 ) { ca_detach(ca2); } else { ALLOCV_END(h2); }
|
|
918
1516
|
if ( a1 ) { ca_detach(ca1); } else { ALLOCV_END(h1); }
|
|
@@ -934,88 +1532,34 @@ rb_ca_call_binop (volatile VALUE self, volatile VALUE other,
|
|
|
934
1532
|
chunk_n = ca_chunk_compute_n(n_kernel, inner, maxb);
|
|
935
1533
|
}
|
|
936
1534
|
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
s1_arena = ca_op_acquire_operand_scratch(ca1, 1);
|
|
948
|
-
ca_xfer_all(ca1, s1_arena, CA_XFER_GET);
|
|
949
|
-
p1_src = (char *) s1_arena;
|
|
950
|
-
}
|
|
951
|
-
} else if ( ca_attach_is_alias(ca1) ) {
|
|
952
|
-
ca_attach(ca1);
|
|
953
|
-
p1_src = (char *) ca1->ptr;
|
|
954
|
-
attached1 = 1;
|
|
955
|
-
} else {
|
|
956
|
-
s1_arena = ca_op_acquire_operand_scratch(ca1, chunk_n);
|
|
957
|
-
p1_src = (char *) s1_arena;
|
|
958
|
-
gather_per_chunk1 = 1;
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
/* ca2 acquire (mirror) */
|
|
962
|
-
if ( i2 == 0 ) {
|
|
963
|
-
if ( ca_attach_is_alias(ca2) ) {
|
|
964
|
-
ca_attach(ca2);
|
|
965
|
-
p2_src = (char *) ca2->ptr;
|
|
966
|
-
attached2 = 1;
|
|
967
|
-
} else {
|
|
968
|
-
s2_arena = ca_op_acquire_operand_scratch(ca2, 1);
|
|
969
|
-
ca_xfer_all(ca2, s2_arena, CA_XFER_GET);
|
|
970
|
-
p2_src = (char *) s2_arena;
|
|
971
|
-
}
|
|
972
|
-
} else if ( ca_attach_is_alias(ca2) ) {
|
|
973
|
-
ca_attach(ca2);
|
|
974
|
-
p2_src = (char *) ca2->ptr;
|
|
975
|
-
attached2 = 1;
|
|
976
|
-
} else {
|
|
977
|
-
s2_arena = ca_op_acquire_operand_scratch(ca2, chunk_n);
|
|
978
|
-
p2_src = (char *) s2_arena;
|
|
979
|
-
gather_per_chunk2 = 1;
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
/* chunk loop: walk n_kernel cells in chunk_n strides */
|
|
983
|
-
for ( off = 0; off < n_kernel; off += chunk_n ) {
|
|
984
|
-
ca_size_t n_done = (off + chunk_n > n_kernel) ? n_kernel - off : chunk_n;
|
|
985
|
-
char *p1, *p2;
|
|
986
|
-
|
|
987
|
-
if ( gather_per_chunk1 ) {
|
|
988
|
-
ca_chunked_gather(ca1, off, n_done, s1_arena);
|
|
989
|
-
p1 = (char *) s1_arena;
|
|
990
|
-
} else {
|
|
991
|
-
p1 = p1_src + (i1 ? off * ca1->bytes : 0);
|
|
992
|
-
}
|
|
993
|
-
if ( gather_per_chunk2 ) {
|
|
994
|
-
ca_chunked_gather(ca2, off, n_done, s2_arena);
|
|
995
|
-
p2 = (char *) s2_arena;
|
|
996
|
-
} else {
|
|
997
|
-
p2 = p2_src + (i2 ? off * ca2->bytes : 0);
|
|
998
|
-
}
|
|
999
|
-
|
|
1000
|
-
func[dt](n_done,
|
|
1001
|
-
ca3->mask ? ((boolean8_t *) ca3->mask->ptr) + (i3 ? off : 0)
|
|
1002
|
-
: NULL,
|
|
1003
|
-
p1, i1,
|
|
1004
|
-
p2, i2,
|
|
1005
|
-
(char *) ca3->ptr + (i3 ? off * ca3->bytes : 0), i3);
|
|
1535
|
+
{
|
|
1536
|
+
ca_binop_chunk_t d;
|
|
1537
|
+
ca_chunked_run_t run;
|
|
1538
|
+
d.func = func; d.dt = dt; d.out = ca3; d.step_out = i3;
|
|
1539
|
+
ca_chunked_run_init(&run, n_kernel, chunk_n);
|
|
1540
|
+
ca_chunked_run_add(&run, ca1, i1);
|
|
1541
|
+
ca_chunked_run_add(&run, ca2, i2);
|
|
1542
|
+
run.kernel = ca_binop_chunk_kernel;
|
|
1543
|
+
run.data = &d;
|
|
1544
|
+
ca_chunked_run(&run);
|
|
1006
1545
|
}
|
|
1007
|
-
|
|
1008
|
-
if ( s2_arena ) ca_lazy_arena_release(s2_arena);
|
|
1009
|
-
if ( s1_arena ) ca_lazy_arena_release(s1_arena);
|
|
1010
|
-
if ( attached2 ) ca_detach(ca2);
|
|
1011
|
-
if ( attached1 ) ca_detach(ca1);
|
|
1012
|
-
|
|
1013
|
-
ca_lazy_arena_exit();
|
|
1014
1546
|
}
|
|
1015
1547
|
|
|
1016
1548
|
return out;
|
|
1017
1549
|
}
|
|
1018
1550
|
|
|
1551
|
+
/* An input of a bang operation that lives in the same root as the target
|
|
1552
|
+
may share memory with it, and read in place it would show the kernel
|
|
1553
|
+
cells the kernel has already written (`a[1..5].add!(a[0..4])` would
|
|
1554
|
+
become a running sum). Such an input is gathered whole before the
|
|
1555
|
+
kernel runs. The target itself (`a.add!(a)`) reads each cell before
|
|
1556
|
+
writing it, so it is read in place. */
|
|
1557
|
+
static int
|
|
1558
|
+
ca_bang_input_may_overlap (CArray *target, CArray *input)
|
|
1559
|
+
{
|
|
1560
|
+
return input != target && ca_shares_root(input, target);
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1019
1563
|
/* Bang (in-place) variant. Invariant: the input-only operand (= other)
|
|
1020
1564
|
must not be attached; self IS the output (write target, attach
|
|
1021
1565
|
legitimate). Same fast/slow pattern as rb_ca_call_binop, applied to
|
|
@@ -1035,6 +1579,9 @@ rb_ca_call_binop_bang (VALUE self, VALUE other, ca_binop_func_t func[])
|
|
|
1035
1579
|
|
|
1036
1580
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
1037
1581
|
TypedData_Get_Struct(other, CArray, &carray_data_type, ca2);
|
|
1582
|
+
if ( func[ca1->data_type] == ca_binop_not_implement ) {
|
|
1583
|
+
ca_op_refuse_unimplemented(self, ca1->data_type, rb_eCADataTypeError);
|
|
1584
|
+
}
|
|
1038
1585
|
|
|
1039
1586
|
self_is_scalar = RTEST(rb_obj_is_cscalar(self));
|
|
1040
1587
|
other_is_scalar = RTEST(rb_obj_is_cscalar(other));
|
|
@@ -1042,17 +1589,11 @@ rb_ca_call_binop_bang (VALUE self, VALUE other, ca_binop_func_t func[])
|
|
|
1042
1589
|
/* self is the write target, so its shape is the result's by definition
|
|
1043
1590
|
and the destination rule applies (the same one assignment uses), not
|
|
1044
1591
|
the symmetric one a binary operation is held to. */
|
|
1045
|
-
if ( !
|
|
1592
|
+
if ( !other_is_scalar ) {
|
|
1046
1593
|
ca_broadcast_to_destination(self, &other);
|
|
1047
1594
|
TypedData_Get_Struct(other, CArray, &carray_data_type, ca2);
|
|
1048
1595
|
other_is_scalar = RTEST(rb_obj_is_cscalar(other));
|
|
1049
1596
|
}
|
|
1050
|
-
if ( self_is_scalar && !other_is_scalar &&
|
|
1051
|
-
ca1->elements != ca2->elements ) {
|
|
1052
|
-
rb_raise(rb_eRuntimeError, "elements mismatch (%" PRId64 " <-> %" PRId64 ")",
|
|
1053
|
-
(ca_size_t) ca1->elements,
|
|
1054
|
-
(ca_size_t) ca2->elements);
|
|
1055
|
-
}
|
|
1056
1597
|
|
|
1057
1598
|
/* kernel strides: cscalar → 0 (broadcast). ca1 is both src1 (input)
|
|
1058
1599
|
and dst (output), same stride. ca2 is input only. */
|
|
@@ -1062,37 +1603,55 @@ rb_ca_call_binop_bang (VALUE self, VALUE other, ca_binop_func_t func[])
|
|
|
1062
1603
|
/* self IS the output (= write target). Always attach + ca_sync
|
|
1063
1604
|
(= legitimate per refined invariant; self can be view e.g.
|
|
1064
1605
|
`arr[i,nil].add!(b)`, sync writes back to root). */
|
|
1065
|
-
|
|
1066
|
-
ca_mask_overlay_safe(ca1, 2, ca1, ca2);
|
|
1067
|
-
|
|
1068
|
-
/* other is input only: fast path if alias-cheap, else materialise
|
|
1069
|
-
via ca_xfer_all without ca_func[X].attach. */
|
|
1070
|
-
if ( ca_attach_is_alias(ca2) ) {
|
|
1071
|
-
ca_attach(ca2);
|
|
1072
|
-
func[ca1->data_type](ca1->elements,
|
|
1073
|
-
( ca1->mask ) ? (boolean8_t *) ca1->mask->ptr : NULL,
|
|
1074
|
-
ca1->ptr, i1,
|
|
1075
|
-
ca2->ptr, i2,
|
|
1076
|
-
ca1->ptr, i1);
|
|
1077
|
-
ca_detach(ca2);
|
|
1078
|
-
}
|
|
1079
|
-
else {
|
|
1606
|
+
{
|
|
1080
1607
|
volatile VALUE h2 = Qnil;
|
|
1081
|
-
char *p2;
|
|
1608
|
+
char *p2 = NULL;
|
|
1609
|
+
int a2 = 0;
|
|
1610
|
+
int in_place2 = ca_attach_is_alias(ca2) &&
|
|
1611
|
+
! ca_bang_input_may_overlap(ca1, ca2);
|
|
1612
|
+
ca_binop_call_t k;
|
|
1613
|
+
CArray *held[2];
|
|
1082
1614
|
(void) h2;
|
|
1083
|
-
|
|
1084
|
-
ca_xfer_all
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1615
|
+
|
|
1616
|
+
/* other is input only: materialised via ca_xfer_all when it cannot be
|
|
1617
|
+
read in place -- before self is attached, since a gather can raise. */
|
|
1618
|
+
if ( ! in_place2 ) {
|
|
1619
|
+
p2 = ALLOCV_N(char, h2, ca2->elements * ca2->bytes);
|
|
1620
|
+
ca_xfer_all(ca2, p2, CA_XFER_GET);
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
ca_attach(ca1); /* window: the overlay detaches it if it raises; the
|
|
1624
|
+
kernel runs through ca_op_guarded when it can */
|
|
1625
|
+
ca_mask_overlay_attached(ca1, 2, ca1, ca2, NULL);
|
|
1626
|
+
|
|
1627
|
+
if ( in_place2 ) {
|
|
1628
|
+
ca_attach(ca2); /* window: an alias attach, which cannot raise */
|
|
1629
|
+
p2 = ca2->ptr;
|
|
1630
|
+
a2 = 1;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
k.f = func[ca1->data_type];
|
|
1634
|
+
k.n = ca1->elements;
|
|
1635
|
+
k.m = ( ca1->mask ) ? (boolean8_t *) ca1->mask->ptr : NULL;
|
|
1636
|
+
k.p1 = ca1->ptr; k.i1 = i1;
|
|
1637
|
+
k.p2 = p2; k.i2 = i2;
|
|
1638
|
+
k.p3 = ca1->ptr; k.i3 = i1;
|
|
1639
|
+
if ( ca_binop_may_raise(func, ca1->data_type) ) {
|
|
1640
|
+
held[0] = ca1;
|
|
1641
|
+
held[1] = a2 ? ca2 : NULL;
|
|
1642
|
+
ca_op_guarded(ca_binop_call, &k, 2, held, "wr");
|
|
1643
|
+
}
|
|
1644
|
+
else {
|
|
1645
|
+
ca_binop_call((VALUE) &k);
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
if ( a2 ) { ca_detach(ca2); } else { ALLOCV_END(h2); }
|
|
1091
1649
|
}
|
|
1092
1650
|
|
|
1093
|
-
|
|
1094
|
-
ca_detach(ca1);
|
|
1651
|
+
ca_sync_detach(ca1);
|
|
1095
1652
|
|
|
1653
|
+
/* a cast or broadcast view of other is reached only through ca2 */
|
|
1654
|
+
RB_GC_GUARD(other);
|
|
1096
1655
|
return self;
|
|
1097
1656
|
}
|
|
1098
1657
|
|
|
@@ -1131,18 +1690,24 @@ rb_ca_triop_select_template (VALUE self, VALUE other2, VALUE other3,
|
|
|
1131
1690
|
- alias-cheap operand → ca_attach (= O(1)) + use ca->ptr directly
|
|
1132
1691
|
- else → ALLOCV scratch + ca_xfer_all without ca_func[X].attach
|
|
1133
1692
|
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1693
|
+
GATHER every operand before ATTACH-ing any: a gather can raise, and an
|
|
1694
|
+
operand attached before it would be left attached. Pair with RELEASE;
|
|
1695
|
+
`h` must be a volatile VALUE declared by the caller (Qnil when nothing
|
|
1696
|
+
was gathered, which ALLOCV_END takes as a no-op). */
|
|
1697
|
+
#define EAGER_GATHER_INPUT(ca_, p_, h_) do { \
|
|
1698
|
+
if ( ! ca_attach_is_alias(ca_) ) { \
|
|
1699
|
+
(p_) = ALLOCV_N(char, (h_), (ca_)->elements * (ca_)->bytes); \
|
|
1700
|
+
ca_xfer_all((ca_), (p_), CA_XFER_GET); \
|
|
1701
|
+
} \
|
|
1702
|
+
} while (0)
|
|
1703
|
+
|
|
1704
|
+
#define EAGER_ATTACH_INPUT(ca_, p_, attached_) do { \
|
|
1138
1705
|
if ( ca_attach_is_alias(ca_) ) { \
|
|
1139
|
-
ca_attach(ca_);
|
|
1706
|
+
ca_attach(ca_); /* window: an alias attach, cannot raise */ \
|
|
1140
1707
|
(p_) = (char *)(ca_)->ptr; \
|
|
1141
1708
|
(attached_) = 1; \
|
|
1142
1709
|
} \
|
|
1143
1710
|
else { \
|
|
1144
|
-
(p_) = ALLOCV_N(char, (h_), (ca_)->elements * (ca_)->bytes); \
|
|
1145
|
-
ca_xfer_all((ca_), (p_), CA_XFER_GET); \
|
|
1146
1711
|
(attached_) = 0; \
|
|
1147
1712
|
} \
|
|
1148
1713
|
} while (0)
|
|
@@ -1156,6 +1721,39 @@ rb_ca_triop_select_template (VALUE self, VALUE other2, VALUE other3,
|
|
|
1156
1721
|
them); cao = new entity output (attach is legit). Each input
|
|
1157
1722
|
independently uses fast (= alias) or slow (= ALLOCV + ca_xfer_all)
|
|
1158
1723
|
path. */
|
|
1724
|
+
/* The triop output is always a fresh array the size of the run; for the
|
|
1725
|
+
bang form it is self, read and written in place. */
|
|
1726
|
+
typedef struct {
|
|
1727
|
+
ca_triop_func_t *func;
|
|
1728
|
+
int8_t dt;
|
|
1729
|
+
CArray *out;
|
|
1730
|
+
int in_place;
|
|
1731
|
+
} ca_triop_chunk_t;
|
|
1732
|
+
|
|
1733
|
+
static void
|
|
1734
|
+
ca_triop_chunk_kernel (ca_chunked_run_t *run, ca_size_t off, ca_size_t n,
|
|
1735
|
+
char **p)
|
|
1736
|
+
{
|
|
1737
|
+
ca_triop_chunk_t *d = (ca_triop_chunk_t *) run->data;
|
|
1738
|
+
char *out = (char *) d->out->ptr + off * d->out->bytes;
|
|
1739
|
+
boolean8_t *m = d->out->mask ? ((boolean8_t *) d->out->mask->ptr) + off
|
|
1740
|
+
: NULL;
|
|
1741
|
+
if ( d->in_place ) {
|
|
1742
|
+
d->func[d->dt](n, m,
|
|
1743
|
+
out, 1,
|
|
1744
|
+
p[0], run->step[0],
|
|
1745
|
+
p[1], run->step[1],
|
|
1746
|
+
out, 1);
|
|
1747
|
+
}
|
|
1748
|
+
else {
|
|
1749
|
+
d->func[d->dt](n, m,
|
|
1750
|
+
p[0], run->step[0],
|
|
1751
|
+
p[1], run->step[1],
|
|
1752
|
+
p[2], run->step[2],
|
|
1753
|
+
out, 1);
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1159
1757
|
VALUE
|
|
1160
1758
|
rb_ca_call_triop (VALUE self, VALUE other2, VALUE other3,
|
|
1161
1759
|
ca_triop_func_t func[])
|
|
@@ -1194,6 +1792,11 @@ rb_ca_call_triop (VALUE self, VALUE other2, VALUE other3,
|
|
|
1194
1792
|
TypedData_Get_Struct(other2, CArray, &carray_data_type, ca2);
|
|
1195
1793
|
TypedData_Get_Struct(other3, CArray, &carray_data_type, ca3);
|
|
1196
1794
|
}
|
|
1795
|
+
if ( func[ca1->data_type] == ca_triop_not_implement ) {
|
|
1796
|
+
ca_op_refuse_unimplemented(ca_is_face(ca1) ? self
|
|
1797
|
+
: ca_is_face(ca2) ? other2 : other3,
|
|
1798
|
+
ca1->data_type, rb_eCADataTypeError);
|
|
1799
|
+
}
|
|
1197
1800
|
|
|
1198
1801
|
/* Element-count check: all non-scalar operands must agree. */
|
|
1199
1802
|
{
|
|
@@ -1242,16 +1845,33 @@ rb_ca_call_triop (VALUE self, VALUE other2, VALUE other3,
|
|
|
1242
1845
|
int attached1, attached2, attached3;
|
|
1243
1846
|
(void) h1; (void) h2; (void) h3;
|
|
1244
1847
|
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1848
|
+
EAGER_GATHER_INPUT(ca1, p1, h1);
|
|
1849
|
+
EAGER_GATHER_INPUT(ca2, p2, h2);
|
|
1850
|
+
EAGER_GATHER_INPUT(ca3, p3, h3);
|
|
1851
|
+
EAGER_ATTACH_INPUT(ca1, p1, attached1);
|
|
1852
|
+
EAGER_ATTACH_INPUT(ca2, p2, attached2);
|
|
1853
|
+
EAGER_ATTACH_INPUT(ca3, p3, attached3);
|
|
1248
1854
|
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1855
|
+
{
|
|
1856
|
+
ca_triop_call_t k;
|
|
1857
|
+
CArray *held[3];
|
|
1858
|
+
k.f = func[ca1->data_type];
|
|
1859
|
+
k.n = cao->elements;
|
|
1860
|
+
k.m = ( cao->mask ) ? (boolean8_t *) cao->mask->ptr : NULL;
|
|
1861
|
+
k.p1 = p1; k.i1 = s1;
|
|
1862
|
+
k.p2 = p2; k.i2 = s2;
|
|
1863
|
+
k.p3 = p3; k.i3 = s3;
|
|
1864
|
+
k.p4 = cao->ptr; k.i4 = 1;
|
|
1865
|
+
if ( ca_triop_may_raise(func, ca1->data_type) ) {
|
|
1866
|
+
held[0] = attached1 ? ca1 : NULL;
|
|
1867
|
+
held[1] = attached2 ? ca2 : NULL;
|
|
1868
|
+
held[2] = attached3 ? ca3 : NULL;
|
|
1869
|
+
ca_op_guarded(ca_triop_call, &k, 3, held, "rrr");
|
|
1870
|
+
}
|
|
1871
|
+
else {
|
|
1872
|
+
ca_triop_call((VALUE) &k);
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1255
1875
|
|
|
1256
1876
|
EAGER_RELEASE_INPUT(ca3, h3, attached3);
|
|
1257
1877
|
EAGER_RELEASE_INPUT(ca2, h2, attached2);
|
|
@@ -1261,13 +1881,8 @@ rb_ca_call_triop (VALUE self, VALUE other2, VALUE other3,
|
|
|
1261
1881
|
/* CHUNKED PATH: per-operand decision matrix (the binop pattern
|
|
1262
1882
|
extended to 3 inputs). scalar/alias same as before; non-alias
|
|
1263
1883
|
array goes through per-chunk gather. */
|
|
1264
|
-
char *p1_src = NULL, *p2_src = NULL, *p3_src = NULL;
|
|
1265
|
-
void *s1_arena = NULL, *s2_arena = NULL, *s3_arena = NULL;
|
|
1266
|
-
int gpc1 = 0, gpc2 = 0, gpc3 = 0; /* gather-per-chunk flags */
|
|
1267
|
-
int att1 = 0, att2 = 0, att3 = 0;
|
|
1268
1884
|
int8_t dt = ca1->data_type;
|
|
1269
1885
|
ca_size_t chunk_n;
|
|
1270
|
-
ca_size_t off;
|
|
1271
1886
|
ca_size_t n_total = cao->elements;
|
|
1272
1887
|
|
|
1273
1888
|
{
|
|
@@ -1288,95 +1903,38 @@ rb_ca_call_triop (VALUE self, VALUE other2, VALUE other3,
|
|
|
1288
1903
|
chunk_n = ca_chunk_compute_n(n_total, inner, maxb);
|
|
1289
1904
|
}
|
|
1290
1905
|
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
} else if ( ca_attach_is_alias(ca1) ) {
|
|
1303
|
-
ca_attach(ca1); p1_src = (char *) ca1->ptr; att1 = 1;
|
|
1304
|
-
} else {
|
|
1305
|
-
s1_arena = ca_op_acquire_operand_scratch(ca1, chunk_n);
|
|
1306
|
-
p1_src = (char *) s1_arena; gpc1 = 1;
|
|
1307
|
-
}
|
|
1308
|
-
/* ca2 acquire (mirror) */
|
|
1309
|
-
if ( s2 == 0 ) {
|
|
1310
|
-
if ( ca_attach_is_alias(ca2) ) {
|
|
1311
|
-
ca_attach(ca2); p2_src = (char *) ca2->ptr; att2 = 1;
|
|
1312
|
-
} else {
|
|
1313
|
-
s2_arena = ca_op_acquire_operand_scratch(ca2, 1);
|
|
1314
|
-
ca_xfer_all(ca2, s2_arena, CA_XFER_GET);
|
|
1315
|
-
p2_src = (char *) s2_arena;
|
|
1316
|
-
}
|
|
1317
|
-
} else if ( ca_attach_is_alias(ca2) ) {
|
|
1318
|
-
ca_attach(ca2); p2_src = (char *) ca2->ptr; att2 = 1;
|
|
1319
|
-
} else {
|
|
1320
|
-
s2_arena = ca_op_acquire_operand_scratch(ca2, chunk_n);
|
|
1321
|
-
p2_src = (char *) s2_arena; gpc2 = 1;
|
|
1322
|
-
}
|
|
1323
|
-
/* ca3 acquire (mirror) */
|
|
1324
|
-
if ( s3 == 0 ) {
|
|
1325
|
-
if ( ca_attach_is_alias(ca3) ) {
|
|
1326
|
-
ca_attach(ca3); p3_src = (char *) ca3->ptr; att3 = 1;
|
|
1327
|
-
} else {
|
|
1328
|
-
s3_arena = ca_op_acquire_operand_scratch(ca3, 1);
|
|
1329
|
-
ca_xfer_all(ca3, s3_arena, CA_XFER_GET);
|
|
1330
|
-
p3_src = (char *) s3_arena;
|
|
1331
|
-
}
|
|
1332
|
-
} else if ( ca_attach_is_alias(ca3) ) {
|
|
1333
|
-
ca_attach(ca3); p3_src = (char *) ca3->ptr; att3 = 1;
|
|
1334
|
-
} else {
|
|
1335
|
-
s3_arena = ca_op_acquire_operand_scratch(ca3, chunk_n);
|
|
1336
|
-
p3_src = (char *) s3_arena; gpc3 = 1;
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
for ( off = 0; off < n_total; off += chunk_n ) {
|
|
1340
|
-
ca_size_t n_done = (off + chunk_n > n_total) ? n_total - off
|
|
1341
|
-
: chunk_n;
|
|
1342
|
-
char *p1, *p2, *p3;
|
|
1343
|
-
|
|
1344
|
-
if ( gpc1 ) { ca_chunked_gather(ca1, off, n_done, s1_arena);
|
|
1345
|
-
p1 = (char *) s1_arena; }
|
|
1346
|
-
else { p1 = p1_src + (s1 ? off * ca1->bytes : 0); }
|
|
1347
|
-
if ( gpc2 ) { ca_chunked_gather(ca2, off, n_done, s2_arena);
|
|
1348
|
-
p2 = (char *) s2_arena; }
|
|
1349
|
-
else { p2 = p2_src + (s2 ? off * ca2->bytes : 0); }
|
|
1350
|
-
if ( gpc3 ) { ca_chunked_gather(ca3, off, n_done, s3_arena);
|
|
1351
|
-
p3 = (char *) s3_arena; }
|
|
1352
|
-
else { p3 = p3_src + (s3 ? off * ca3->bytes : 0); }
|
|
1353
|
-
|
|
1354
|
-
func[dt](n_done,
|
|
1355
|
-
cao->mask ? ((boolean8_t *) cao->mask->ptr) + off
|
|
1356
|
-
: NULL,
|
|
1357
|
-
p1, s1,
|
|
1358
|
-
p2, s2,
|
|
1359
|
-
p3, s3,
|
|
1360
|
-
(char *) cao->ptr + off * cao->bytes, 1);
|
|
1906
|
+
{
|
|
1907
|
+
ca_triop_chunk_t d;
|
|
1908
|
+
ca_chunked_run_t run;
|
|
1909
|
+
d.func = func; d.dt = dt; d.out = cao; d.in_place = 0;
|
|
1910
|
+
ca_chunked_run_init(&run, n_total, chunk_n);
|
|
1911
|
+
ca_chunked_run_add(&run, ca1, s1);
|
|
1912
|
+
ca_chunked_run_add(&run, ca2, s2);
|
|
1913
|
+
ca_chunked_run_add(&run, ca3, s3);
|
|
1914
|
+
run.kernel = ca_triop_chunk_kernel;
|
|
1915
|
+
run.data = &d;
|
|
1916
|
+
ca_chunked_run(&run);
|
|
1361
1917
|
}
|
|
1362
|
-
|
|
1363
|
-
if ( s3_arena ) ca_lazy_arena_release(s3_arena);
|
|
1364
|
-
if ( s2_arena ) ca_lazy_arena_release(s2_arena);
|
|
1365
|
-
if ( s1_arena ) ca_lazy_arena_release(s1_arena);
|
|
1366
|
-
if ( att3 ) ca_detach(ca3);
|
|
1367
|
-
if ( att2 ) ca_detach(ca2);
|
|
1368
|
-
if ( att1 ) ca_detach(ca1);
|
|
1369
|
-
|
|
1370
|
-
ca_lazy_arena_exit();
|
|
1371
1918
|
}
|
|
1372
1919
|
}
|
|
1373
1920
|
|
|
1921
|
+
/* the INT64 views of a boolean triop are reached only through ca1..ca3 */
|
|
1922
|
+
RB_GC_GUARD(self);
|
|
1923
|
+
RB_GC_GUARD(other2);
|
|
1924
|
+
RB_GC_GUARD(other3);
|
|
1374
1925
|
return out;
|
|
1375
1926
|
}
|
|
1376
1927
|
|
|
1377
1928
|
/* triop_bang (in-place) driver. ca1 = self = output (write target,
|
|
1378
1929
|
attach legit; keep ca_attach + ca_sync); ca2/ca3 = input only (fast/slow
|
|
1379
1930
|
dispatch via EAGER_ACQUIRE/RELEASE). */
|
|
1931
|
+
static VALUE
|
|
1932
|
+
ca_triop_bang_chunked (VALUE arg)
|
|
1933
|
+
{
|
|
1934
|
+
ca_chunked_run((ca_chunked_run_t *) arg);
|
|
1935
|
+
return Qnil;
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1380
1938
|
VALUE
|
|
1381
1939
|
rb_ca_call_triop_bang (VALUE self, VALUE other2, VALUE other3,
|
|
1382
1940
|
ca_triop_func_t func[])
|
|
@@ -1391,6 +1949,9 @@ rb_ca_call_triop_bang (VALUE self, VALUE other2, VALUE other3,
|
|
|
1391
1949
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
1392
1950
|
TypedData_Get_Struct(other2, CArray, &carray_data_type, ca2);
|
|
1393
1951
|
TypedData_Get_Struct(other3, CArray, &carray_data_type, ca3);
|
|
1952
|
+
if ( func[ca1->data_type] == ca_triop_not_implement ) {
|
|
1953
|
+
ca_op_refuse_unimplemented(self, ca1->data_type, rb_eCADataTypeError);
|
|
1954
|
+
}
|
|
1394
1955
|
|
|
1395
1956
|
/* self is the write target, so the destination rule applies to each
|
|
1396
1957
|
input operand in turn (the same one assignment uses). */
|
|
@@ -1403,10 +1964,6 @@ rb_ca_call_triop_bang (VALUE self, VALUE other2, VALUE other3,
|
|
|
1403
1964
|
TypedData_Get_Struct(other3, CArray, &carray_data_type, ca3);
|
|
1404
1965
|
}
|
|
1405
1966
|
|
|
1406
|
-
/* self IS the output (= write target; attach legit per refined invariant) */
|
|
1407
|
-
ca_attach(ca1);
|
|
1408
|
-
ca_mask_overlay_safe(ca1, 3, ca1, ca2, ca3);
|
|
1409
|
-
|
|
1410
1967
|
{
|
|
1411
1968
|
ca_size_t s2 = rb_obj_is_cscalar(other2) ? 0 : 1;
|
|
1412
1969
|
ca_size_t s3 = rb_obj_is_cscalar(other3) ? 0 : 1;
|
|
@@ -1414,82 +1971,124 @@ rb_ca_call_triop_bang (VALUE self, VALUE other2, VALUE other3,
|
|
|
1414
1971
|
/* Threshold dispatch on the input-only operands (ca2 / ca3). Self
|
|
1415
1972
|
(ca1) IS the output (attached + ca_sync as usual). >= 2 non-alias
|
|
1416
1973
|
input arrays → chunked, else 1-shot ALLOCV. */
|
|
1974
|
+
/* An input that may share memory with self is gathered whole, never
|
|
1975
|
+
read in place or per chunk: either would read cells already
|
|
1976
|
+
written. */
|
|
1977
|
+
int overlap2 = ca_bang_input_may_overlap(ca1, ca2);
|
|
1978
|
+
int overlap3 = ca_bang_input_may_overlap(ca1, ca3);
|
|
1417
1979
|
int nonalias_arrays = 0;
|
|
1418
1980
|
int use_chunked;
|
|
1419
1981
|
if ( s2 == 1 && !ca_attach_is_alias(ca2) ) nonalias_arrays++;
|
|
1420
1982
|
if ( s3 == 1 && !ca_attach_is_alias(ca3) ) nonalias_arrays++;
|
|
1421
|
-
use_chunked = (nonalias_arrays >= 2);
|
|
1983
|
+
use_chunked = (nonalias_arrays >= 2) && !overlap2 && !overlap3;
|
|
1422
1984
|
|
|
1423
1985
|
if ( !use_chunked ) {
|
|
1424
1986
|
volatile VALUE h2 = Qnil, h3 = Qnil;
|
|
1425
1987
|
char *p2, *p3;
|
|
1426
1988
|
int attached2, attached3;
|
|
1989
|
+
ca_triop_call_t k;
|
|
1990
|
+
CArray *held[3];
|
|
1427
1991
|
(void) h2; (void) h3;
|
|
1428
1992
|
|
|
1429
|
-
|
|
1430
|
-
|
|
1993
|
+
int in_place2 = ca_attach_is_alias(ca2) && !overlap2;
|
|
1994
|
+
int in_place3 = ca_attach_is_alias(ca3) && !overlap3;
|
|
1431
1995
|
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1996
|
+
/* The gathers can raise, so they run before self is attached. */
|
|
1997
|
+
if ( ! in_place2 ) {
|
|
1998
|
+
p2 = ALLOCV_N(char, h2, ca2->elements * ca2->bytes);
|
|
1999
|
+
ca_xfer_all(ca2, p2, CA_XFER_GET);
|
|
2000
|
+
}
|
|
2001
|
+
if ( ! in_place3 ) {
|
|
2002
|
+
p3 = ALLOCV_N(char, h3, ca3->elements * ca3->bytes);
|
|
2003
|
+
ca_xfer_all(ca3, p3, CA_XFER_GET);
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
/* self IS the output (= write target; attach legit per refined
|
|
2007
|
+
invariant) */
|
|
2008
|
+
ca_attach(ca1); /* window: the overlay detaches it if it raises; the
|
|
2009
|
+
kernel runs through ca_op_guarded */
|
|
2010
|
+
ca_mask_overlay_attached(ca1, 3, ca1, ca2, ca3);
|
|
2011
|
+
|
|
2012
|
+
attached2 = attached3 = 0;
|
|
2013
|
+
if ( in_place2 ) {
|
|
2014
|
+
ca_attach(ca2); /* window: an alias attach, cannot raise */
|
|
2015
|
+
p2 = (char *) ca2->ptr;
|
|
2016
|
+
attached2 = 1;
|
|
2017
|
+
}
|
|
2018
|
+
if ( in_place3 ) {
|
|
2019
|
+
ca_attach(ca3); /* window: as above */
|
|
2020
|
+
p3 = (char *) ca3->ptr;
|
|
2021
|
+
attached3 = 1;
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
k.f = func[ca1->data_type];
|
|
2025
|
+
k.n = ca1->elements;
|
|
2026
|
+
k.m = ( ca1->mask ) ? (boolean8_t *) ca1->mask->ptr : NULL;
|
|
2027
|
+
k.p1 = ca1->ptr; k.i1 = 1;
|
|
2028
|
+
k.p2 = p2; k.i2 = s2;
|
|
2029
|
+
k.p3 = p3; k.i3 = s3;
|
|
2030
|
+
k.p4 = ca1->ptr; k.i4 = 1;
|
|
2031
|
+
if ( ca_triop_may_raise(func, ca1->data_type) ) {
|
|
2032
|
+
held[0] = ca1;
|
|
2033
|
+
held[1] = attached2 ? ca2 : NULL;
|
|
2034
|
+
held[2] = attached3 ? ca3 : NULL;
|
|
2035
|
+
ca_op_guarded(ca_triop_call, &k, 3, held, "wrr");
|
|
2036
|
+
}
|
|
2037
|
+
else {
|
|
2038
|
+
ca_triop_call((VALUE) &k);
|
|
2039
|
+
}
|
|
1438
2040
|
|
|
1439
2041
|
EAGER_RELEASE_INPUT(ca3, h3, attached3);
|
|
1440
2042
|
EAGER_RELEASE_INPUT(ca2, h2, attached2);
|
|
1441
2043
|
}
|
|
1442
2044
|
else {
|
|
1443
2045
|
/* CHUNKED PATH: both ca2 and ca3 non-alias arrays. ca1 (= self =
|
|
1444
|
-
output) is
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
2046
|
+
output) is attached; its ptr is contig (= ca_attach materialise +
|
|
2047
|
+
alias for entity, or full materialise for view). Write to
|
|
2048
|
+
ca1->ptr + off*bytes in chunks; sync at end. The run gathers
|
|
2049
|
+
per chunk, and any gather can raise, so the run is guarded
|
|
2050
|
+
whatever the kernel. */
|
|
2051
|
+
ca_attach(ca1); /* window: the overlay detaches it if it raises; the
|
|
2052
|
+
kernel runs through ca_op_guarded */
|
|
2053
|
+
ca_mask_overlay_attached(ca1, 3, ca1, ca2, ca3);
|
|
1453
2054
|
{
|
|
1454
|
-
|
|
1455
|
-
ca_size_t
|
|
1456
|
-
ca_size_t
|
|
1457
|
-
ca_size_t maxb = ca1->bytes;
|
|
1458
|
-
if ( ca2->bytes > maxb ) maxb = ca2->bytes;
|
|
1459
|
-
if ( ca3->bytes > maxb ) maxb = ca3->bytes;
|
|
1460
|
-
if ( inn2 > inner ) inner = inn2;
|
|
1461
|
-
if ( inn3 > inner ) inner = inn3;
|
|
1462
|
-
chunk_n = ca_chunk_compute_n(n_total, inner, maxb);
|
|
1463
|
-
}
|
|
2055
|
+
int8_t dt = ca1->data_type;
|
|
2056
|
+
ca_size_t chunk_n;
|
|
2057
|
+
ca_size_t n_total = ca1->elements;
|
|
1464
2058
|
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
ca1->mask ? ((boolean8_t *) ca1->mask->ptr) + off
|
|
1477
|
-
: NULL,
|
|
1478
|
-
(char *) ca1->ptr + off * ca1->bytes, 1,
|
|
1479
|
-
(char *) s2_arena, s2,
|
|
1480
|
-
(char *) s3_arena, s3,
|
|
1481
|
-
(char *) ca1->ptr + off * ca1->bytes, 1);
|
|
1482
|
-
}
|
|
2059
|
+
{
|
|
2060
|
+
ca_size_t inner = ca_chunk_inner_size(ca1);
|
|
2061
|
+
ca_size_t inn2 = ca_chunk_inner_size(ca2);
|
|
2062
|
+
ca_size_t inn3 = ca_chunk_inner_size(ca3);
|
|
2063
|
+
ca_size_t maxb = ca1->bytes;
|
|
2064
|
+
if ( ca2->bytes > maxb ) maxb = ca2->bytes;
|
|
2065
|
+
if ( ca3->bytes > maxb ) maxb = ca3->bytes;
|
|
2066
|
+
if ( inn2 > inner ) inner = inn2;
|
|
2067
|
+
if ( inn3 > inner ) inner = inn3;
|
|
2068
|
+
chunk_n = ca_chunk_compute_n(n_total, inner, maxb);
|
|
2069
|
+
}
|
|
1483
2070
|
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
2071
|
+
{
|
|
2072
|
+
ca_triop_chunk_t d;
|
|
2073
|
+
ca_chunked_run_t run;
|
|
2074
|
+
d.func = func; d.dt = dt; d.out = ca1; d.in_place = 1;
|
|
2075
|
+
ca_chunked_run_init(&run, n_total, chunk_n);
|
|
2076
|
+
ca_chunked_run_add(&run, ca2, s2);
|
|
2077
|
+
ca_chunked_run_add(&run, ca3, s3);
|
|
2078
|
+
run.kernel = ca_triop_chunk_kernel;
|
|
2079
|
+
run.data = &d;
|
|
2080
|
+
ca_op_guarded(ca_triop_bang_chunked, &run, 1, &ca1, "w");
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
1487
2083
|
}
|
|
1488
2084
|
}
|
|
1489
2085
|
|
|
1490
|
-
|
|
1491
|
-
ca_detach(ca1);
|
|
2086
|
+
ca_sync_detach(ca1);
|
|
1492
2087
|
|
|
2088
|
+
/* cast or broadcast views of the operands are reached only through
|
|
2089
|
+
ca2 and ca3 */
|
|
2090
|
+
RB_GC_GUARD(other2);
|
|
2091
|
+
RB_GC_GUARD(other3);
|
|
1493
2092
|
return self;
|
|
1494
2093
|
}
|
|
1495
2094
|
|
|
@@ -1502,6 +2101,9 @@ rb_ca_call_moncmp (VALUE self, ca_moncmp_func_t func[])
|
|
|
1502
2101
|
CArray *ca1, *ca2; /* ca2 = ca1.op */
|
|
1503
2102
|
|
|
1504
2103
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
2104
|
+
if ( func[ca1->data_type] == ca_moncmp_not_implement ) {
|
|
2105
|
+
ca_op_refuse_unimplemented(self, ca1->data_type, rb_eCADataTypeError);
|
|
2106
|
+
}
|
|
1505
2107
|
|
|
1506
2108
|
if ( ca_is_scalar(ca1) ) {
|
|
1507
2109
|
out = rb_cscalar_new(CA_BOOLEAN, 0, NULL);
|
|
@@ -1514,6 +2116,9 @@ rb_ca_call_moncmp (VALUE self, ca_moncmp_func_t func[])
|
|
|
1514
2116
|
|
|
1515
2117
|
ca_mask_overlay_safe(ca2, 1, ca1);
|
|
1516
2118
|
|
|
2119
|
+
/* No cell to compute, as in the operators above. */
|
|
2120
|
+
if ( ca2->elements == 0 ) return out;
|
|
2121
|
+
|
|
1517
2122
|
/* The kernel's masked branch skips masked cells, so they would keep the
|
|
1518
2123
|
uninitialised data from rb_carray_new. Zero the output when a mask is
|
|
1519
2124
|
present so masked cells read as 0 (matching binop's ca_template_safe
|
|
@@ -1528,11 +2133,20 @@ rb_ca_call_moncmp (VALUE self, ca_moncmp_func_t func[])
|
|
|
1528
2133
|
int attached1;
|
|
1529
2134
|
(void) h1;
|
|
1530
2135
|
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
2136
|
+
ca_moncmp_call_t k;
|
|
2137
|
+
EAGER_GATHER_INPUT(ca1, p1, h1);
|
|
2138
|
+
EAGER_ATTACH_INPUT(ca1, p1, attached1);
|
|
2139
|
+
k.f = func[ca1->data_type];
|
|
2140
|
+
k.n = ca1->elements;
|
|
2141
|
+
k.m = ( ca2->mask ) ? (boolean8_t *) ca2->mask->ptr : NULL;
|
|
2142
|
+
k.p1 = p1; k.i1 = 1;
|
|
2143
|
+
k.p2 = (boolean8_t *) ca2->ptr; k.i2 = 1;
|
|
2144
|
+
if ( attached1 && ca_moncmp_may_raise(func, ca1->data_type) ) {
|
|
2145
|
+
ca_op_guarded(ca_moncmp_call, &k, 1, &ca1, "r");
|
|
2146
|
+
}
|
|
2147
|
+
else {
|
|
2148
|
+
ca_moncmp_call((VALUE) &k);
|
|
2149
|
+
}
|
|
1536
2150
|
EAGER_RELEASE_INPUT(ca1, h1, attached1);
|
|
1537
2151
|
}
|
|
1538
2152
|
|
|
@@ -1545,6 +2159,180 @@ rb_ca_call_moncmp (VALUE self, ca_moncmp_func_t func[])
|
|
|
1545
2159
|
UNDEF-comparison identity checks below cast to ca_bincmp_func_t
|
|
1546
2160
|
explicitly. */
|
|
1547
2161
|
|
|
2162
|
+
typedef struct {
|
|
2163
|
+
ca_bincmp_func_t *func;
|
|
2164
|
+
int8_t dt;
|
|
2165
|
+
CArray *out;
|
|
2166
|
+
ca_size_t step_out;
|
|
2167
|
+
double tol;
|
|
2168
|
+
} ca_bincmp_chunk_t;
|
|
2169
|
+
|
|
2170
|
+
static void
|
|
2171
|
+
ca_bincmp_chunk_kernel (ca_chunked_run_t *run, ca_size_t off, ca_size_t n,
|
|
2172
|
+
char **p)
|
|
2173
|
+
{
|
|
2174
|
+
ca_bincmp_chunk_t *d = (ca_bincmp_chunk_t *) run->data;
|
|
2175
|
+
ca_size_t o = d->step_out ? off : 0;
|
|
2176
|
+
d->func[d->dt](n,
|
|
2177
|
+
d->out->mask ? ((boolean8_t *) d->out->mask->ptr) + o : NULL,
|
|
2178
|
+
p[0], run->ca[0]->bytes, run->step[0],
|
|
2179
|
+
p[1], run->ca[1]->bytes, run->step[1],
|
|
2180
|
+
(char *) d->out->ptr + o * d->out->bytes, d->out->bytes,
|
|
2181
|
+
d->step_out,
|
|
2182
|
+
d->tol);
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
/* The six comparisons answer by value between integers. Promotion takes
|
|
2186
|
+
uint64 against int64 (and uint8 against int8, ...) to the unsigned type,
|
|
2187
|
+
where -7 is 2**64 - 7 and 7 < -7 would be true; and an Integer outside an
|
|
2188
|
+
array's type has no cell of that type to stand in for it. */
|
|
2189
|
+
|
|
2190
|
+
static int
|
|
2191
|
+
ca_is_unsigned_integer_dt (int8_t dt)
|
|
2192
|
+
{
|
|
2193
|
+
return dt == CA_UINT8 || dt == CA_UINT16 || dt == CA_UINT32 ||
|
|
2194
|
+
dt == CA_UINT64;
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
static int
|
|
2198
|
+
ca_is_signed_integer_dt (int8_t dt)
|
|
2199
|
+
{
|
|
2200
|
+
return dt == CA_INT8 || dt == CA_INT16 || dt == CA_INT32 ||
|
|
2201
|
+
dt == CA_INT64;
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
/* The answer of `left op right` when every right is greater than every
|
|
2205
|
+
left (right_greater = 1), or every left greater than every right. */
|
|
2206
|
+
static int
|
|
2207
|
+
ca_bincmp_ordered_answer (int op_id, int right_greater)
|
|
2208
|
+
{
|
|
2209
|
+
switch ( op_id ) {
|
|
2210
|
+
case CA_BINCMP_LT: case CA_BINCMP_LE: return right_greater;
|
|
2211
|
+
case CA_BINCMP_GT: case CA_BINCMP_GE: return ! right_greater;
|
|
2212
|
+
case CA_BINCMP_EQ: return 0;
|
|
2213
|
+
default: return 1; /* CA_BINCMP_NE */
|
|
2214
|
+
}
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
/* The answer at a cell whose signed operand is negative: the unsigned
|
|
2218
|
+
operand is then the greater. signed_side is 1 (left) or 2 (right). */
|
|
2219
|
+
int
|
|
2220
|
+
ca_bincmp_answer_when_negative (int op_id, int signed_side)
|
|
2221
|
+
{
|
|
2222
|
+
return ca_bincmp_ordered_answer(op_id, signed_side == 1);
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
static int
|
|
2226
|
+
ca_bincmp_plain_integer_array (VALUE v, int8_t *dt)
|
|
2227
|
+
{
|
|
2228
|
+
CArray *ca;
|
|
2229
|
+
if ( ! rb_obj_is_carray(v) ) return 0;
|
|
2230
|
+
TypedData_Get_Struct(v, CArray, &carray_data_type, ca);
|
|
2231
|
+
if ( ca_is_face(ca) ) return 0;
|
|
2232
|
+
*dt = ca->data_type;
|
|
2233
|
+
return ca_is_unsigned_integer_dt(*dt) || ca_is_signed_integer_dt(*dt);
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
/* Prepares the operands of comparison op_id (CA_BINCMP_*). Returns
|
|
2237
|
+
0 nothing to do,
|
|
2238
|
+
1 every cell answers true, 2 every cell answers false
|
|
2239
|
+
(*self and *other are then both the array, whose mask the answer
|
|
2240
|
+
keeps),
|
|
2241
|
+
3 the operands are integers of mixed signedness that promote to the
|
|
2242
|
+
unsigned type; *signed_side is 1 or 2 for the signed one. */
|
|
2243
|
+
int
|
|
2244
|
+
ca_bincmp_integer_prepare (volatile VALUE *self, volatile VALUE *other,
|
|
2245
|
+
int op_id, int *signed_side)
|
|
2246
|
+
{
|
|
2247
|
+
int8_t dt1, dt2;
|
|
2248
|
+
int side;
|
|
2249
|
+
|
|
2250
|
+
if ( op_id < CA_BINCMP_LT || op_id > CA_BINCMP_NE ) return 0;
|
|
2251
|
+
|
|
2252
|
+
if ( RB_INTEGER_TYPE_P(*other) &&
|
|
2253
|
+
ca_bincmp_plain_integer_array(*self, &dt1) ) {
|
|
2254
|
+
side = ca_integer_range_side(*other, dt1);
|
|
2255
|
+
if ( side == 0 ) return 0;
|
|
2256
|
+
*other = *self;
|
|
2257
|
+
return ca_bincmp_ordered_answer(op_id, side > 0) ? 1 : 2;
|
|
2258
|
+
}
|
|
2259
|
+
if ( RB_INTEGER_TYPE_P(*self) &&
|
|
2260
|
+
ca_bincmp_plain_integer_array(*other, &dt2) ) {
|
|
2261
|
+
side = ca_integer_range_side(*self, dt2);
|
|
2262
|
+
if ( side == 0 ) return 0;
|
|
2263
|
+
*self = *other;
|
|
2264
|
+
return ca_bincmp_ordered_answer(op_id, side < 0) ? 1 : 2;
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
if ( ! ca_bincmp_plain_integer_array(*self, &dt1) ||
|
|
2268
|
+
! ca_bincmp_plain_integer_array(*other, &dt2) ) {
|
|
2269
|
+
return 0;
|
|
2270
|
+
}
|
|
2271
|
+
if ( ca_is_unsigned_integer_dt(dt1) == ca_is_unsigned_integer_dt(dt2) ) {
|
|
2272
|
+
return 0;
|
|
2273
|
+
}
|
|
2274
|
+
if ( ! ca_is_unsigned_integer_dt(ca_promote_type(dt1, dt2)) ) {
|
|
2275
|
+
return 0;
|
|
2276
|
+
}
|
|
2277
|
+
*signed_side = ca_is_signed_integer_dt(dt1) ? 1 : 2;
|
|
2278
|
+
return 3;
|
|
2279
|
+
}
|
|
2280
|
+
|
|
2281
|
+
static int
|
|
2282
|
+
ca_bincmp_op_id_of (ca_bincmp_func_t *func)
|
|
2283
|
+
{
|
|
2284
|
+
if ( func == ca_bincmp_lt ) return CA_BINCMP_LT;
|
|
2285
|
+
if ( func == ca_bincmp_gt ) return CA_BINCMP_GT;
|
|
2286
|
+
if ( func == ca_bincmp_le ) return CA_BINCMP_LE;
|
|
2287
|
+
if ( func == ca_bincmp_ge ) return CA_BINCMP_GE;
|
|
2288
|
+
if ( func == ca_bincmp_eq ) return CA_BINCMP_EQ;
|
|
2289
|
+
if ( func == ca_bincmp_ne ) return CA_BINCMP_NE;
|
|
2290
|
+
return -1;
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
/* Compares the unsigned operand as uint64 with the signed one taken to
|
|
2294
|
+
uint64 modulo 2**64, then answers the cells where the signed operand is
|
|
2295
|
+
negative. */
|
|
2296
|
+
static VALUE
|
|
2297
|
+
ca_bincmp_mixed_sign (VALUE self, VALUE other, ca_bincmp_func_t func[],
|
|
2298
|
+
int op_id, int signed_side)
|
|
2299
|
+
{
|
|
2300
|
+
volatile VALUE l, r, su, out;
|
|
2301
|
+
CArray *cs, *co;
|
|
2302
|
+
int64_t *sp;
|
|
2303
|
+
boolean8_t *op, *m;
|
|
2304
|
+
ca_size_t i, step;
|
|
2305
|
+
int answer = ca_bincmp_answer_when_negative(op_id, signed_side);
|
|
2306
|
+
|
|
2307
|
+
l = rb_funcall(self, rb_intern("to_type"), 1,
|
|
2308
|
+
INT2NUM(( signed_side == 1 ) ? CA_INT64 : CA_UINT64));
|
|
2309
|
+
r = rb_funcall(other, rb_intern("to_type"), 1,
|
|
2310
|
+
INT2NUM(( signed_side == 2 ) ? CA_INT64 : CA_UINT64));
|
|
2311
|
+
ca_broadcast_pair(&l, &r);
|
|
2312
|
+
|
|
2313
|
+
/* A fresh uint64 entity holding the signed operand modulo 2**64; read
|
|
2314
|
+
as int64 it still says which cells are negative. */
|
|
2315
|
+
su = rb_funcall(( signed_side == 1 ) ? l : r, rb_intern("to_type"), 1,
|
|
2316
|
+
INT2NUM(CA_UINT64));
|
|
2317
|
+
out = ( signed_side == 1 ) ? rb_ca_call_bincmp(su, r, func, 0.0)
|
|
2318
|
+
: rb_ca_call_bincmp(l, su, func, 0.0);
|
|
2319
|
+
|
|
2320
|
+
TypedData_Get_Struct(su, CArray, &carray_data_type, cs);
|
|
2321
|
+
TypedData_Get_Struct(out, CArray, &carray_data_type, co);
|
|
2322
|
+
sp = (int64_t *) cs->ptr;
|
|
2323
|
+
op = (boolean8_t *) co->ptr;
|
|
2324
|
+
m = co->mask ? (boolean8_t *) co->mask->ptr : NULL;
|
|
2325
|
+
step = ( cs->elements == 1 ) ? 0 : 1;
|
|
2326
|
+
for ( i = 0; i < co->elements; i++ ) {
|
|
2327
|
+
if ( ( ! m || ! m[i] ) && sp[i * step] < 0 ) {
|
|
2328
|
+
op[i] = (boolean8_t) answer;
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
RB_GC_GUARD(l);
|
|
2332
|
+
RB_GC_GUARD(r);
|
|
2333
|
+
return out;
|
|
2334
|
+
}
|
|
2335
|
+
|
|
1548
2336
|
VALUE
|
|
1549
2337
|
rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
1550
2338
|
ca_bincmp_func_t func[],
|
|
@@ -1562,7 +2350,7 @@ rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
|
1562
2350
|
return rb_ca_is_not_masked(self);
|
|
1563
2351
|
}
|
|
1564
2352
|
else {
|
|
1565
|
-
rb_raise(
|
|
2353
|
+
rb_raise(rb_eArgError, "array can not be compared with UNDEF");
|
|
1566
2354
|
}
|
|
1567
2355
|
}
|
|
1568
2356
|
|
|
@@ -1572,11 +2360,25 @@ rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
|
1572
2360
|
for fixlen-storage Faces (memcmp is already correct there). */
|
|
1573
2361
|
ca_face_reconcile_comparison(&self, &other);
|
|
1574
2362
|
|
|
2363
|
+
{
|
|
2364
|
+
int op_id = ca_bincmp_op_id_of(func);
|
|
2365
|
+
int signed_side = 0;
|
|
2366
|
+
switch ( ca_bincmp_integer_prepare(&self, &other, op_id, &signed_side) ) {
|
|
2367
|
+
case 1: return rb_ca_call_bincmp(self, self, ca_bincmp_eq, 0.0);
|
|
2368
|
+
case 2: return rb_ca_call_bincmp(self, self, ca_bincmp_ne, 0.0);
|
|
2369
|
+
case 3: return ca_bincmp_mixed_sign(self, other, func, op_id, signed_side);
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
|
|
1575
2373
|
/* do implicit casting */
|
|
1576
2374
|
rb_ca_cast_self_or_other(&self, &other);
|
|
1577
2375
|
|
|
1578
2376
|
TypedData_Get_Struct(self, CArray, &carray_data_type, ca1);
|
|
1579
2377
|
TypedData_Get_Struct(other, CArray, &carray_data_type, ca2);
|
|
2378
|
+
if ( func[ca1->data_type] == ca_bincmp_not_implement ) {
|
|
2379
|
+
ca_op_refuse_unimplemented(ca_is_face(ca1) ? self : other, ca1->data_type,
|
|
2380
|
+
rb_eTypeError);
|
|
2381
|
+
}
|
|
1580
2382
|
|
|
1581
2383
|
/* Same fast/slow shape as rb_ca_call_binop, but the output data_type is
|
|
1582
2384
|
fixed boolean (rb_ca_call_binop's output data_type matches its input). */
|
|
@@ -1610,6 +2412,9 @@ rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
|
1610
2412
|
|
|
1611
2413
|
ca_mask_overlay_safe(ca3, 2, ca1, ca2);
|
|
1612
2414
|
|
|
2415
|
+
/* No cell to compute, as in the operators above. */
|
|
2416
|
+
if ( ca3->elements == 0 ) return out;
|
|
2417
|
+
|
|
1613
2418
|
/* The kernel's masked branch skips masked cells, so they would keep the
|
|
1614
2419
|
uninitialised data from rb_carray_new. Zero the output when a mask is
|
|
1615
2420
|
present so masked cells read as 0 (matching binop's ca_template_safe
|
|
@@ -1651,15 +2456,29 @@ rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
|
1651
2456
|
int attached1, attached2;
|
|
1652
2457
|
(void) h1; (void) h2;
|
|
1653
2458
|
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
2459
|
+
ca_bincmp_call_t k;
|
|
2460
|
+
CArray *held[2];
|
|
2461
|
+
EAGER_GATHER_INPUT(ca1, p1, h1);
|
|
2462
|
+
EAGER_GATHER_INPUT(ca2, p2, h2);
|
|
2463
|
+
EAGER_ATTACH_INPUT(ca1, p1, attached1);
|
|
2464
|
+
EAGER_ATTACH_INPUT(ca2, p2, attached2);
|
|
2465
|
+
|
|
2466
|
+
k.f = func[ca1->data_type];
|
|
2467
|
+
k.n = n_kernel;
|
|
2468
|
+
k.m = ( ca3->mask ) ? (boolean8_t *) ca3->mask->ptr : NULL;
|
|
2469
|
+
k.p1 = p1; k.b1 = ca1->bytes; k.i1 = i1;
|
|
2470
|
+
k.p2 = p2; k.b2 = ca2->bytes; k.i2 = i2;
|
|
2471
|
+
k.p3 = ca3->ptr; k.b3 = ca3->bytes; k.i3 = i3;
|
|
2472
|
+
k.tol = tol;
|
|
2473
|
+
if ( ( attached1 || attached2 ) &&
|
|
2474
|
+
ca_bincmp_may_raise(func, ca1->data_type) ) {
|
|
2475
|
+
held[0] = attached1 ? ca1 : NULL;
|
|
2476
|
+
held[1] = attached2 ? ca2 : NULL;
|
|
2477
|
+
ca_op_guarded(ca_bincmp_call, &k, 2, held, "rr");
|
|
2478
|
+
}
|
|
2479
|
+
else {
|
|
2480
|
+
ca_bincmp_call((VALUE) &k);
|
|
2481
|
+
}
|
|
1663
2482
|
|
|
1664
2483
|
EAGER_RELEASE_INPUT(ca2, h2, attached2);
|
|
1665
2484
|
EAGER_RELEASE_INPUT(ca1, h1, attached1);
|
|
@@ -1667,10 +2486,9 @@ rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
|
1667
2486
|
else {
|
|
1668
2487
|
/* CHUNKED PATH (both operands non-alias array): per-chunk gather
|
|
1669
2488
|
into arena scratch. Mirrors the binop chunked branch. */
|
|
1670
|
-
void *s1_arena = NULL, *s2_arena = NULL;
|
|
1671
2489
|
ca_size_t b1 = ca1->bytes, b2 = ca2->bytes, b3 = ca3->bytes;
|
|
1672
2490
|
int8_t dt = ca1->data_type;
|
|
1673
|
-
ca_size_t chunk_n
|
|
2491
|
+
ca_size_t chunk_n;
|
|
1674
2492
|
|
|
1675
2493
|
{
|
|
1676
2494
|
ca_size_t inner1 = ca_chunk_inner_size(ca1);
|
|
@@ -1681,28 +2499,17 @@ rb_ca_call_bincmp (volatile VALUE self, volatile VALUE other,
|
|
|
1681
2499
|
chunk_n = ca_chunk_compute_n(n_kernel, inner, maxb);
|
|
1682
2500
|
}
|
|
1683
2501
|
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
func[dt](n_done,
|
|
1695
|
-
ca3->mask ? ((boolean8_t *) ca3->mask->ptr) + off
|
|
1696
|
-
: NULL,
|
|
1697
|
-
(char *) s1_arena, b1, i1,
|
|
1698
|
-
(char *) s2_arena, b2, i2,
|
|
1699
|
-
(char *) ca3->ptr + off * b3, b3, i3,
|
|
1700
|
-
tol);
|
|
2502
|
+
{
|
|
2503
|
+
ca_bincmp_chunk_t d;
|
|
2504
|
+
ca_chunked_run_t run;
|
|
2505
|
+
d.func = func; d.dt = dt; d.out = ca3; d.step_out = i3; d.tol = tol;
|
|
2506
|
+
ca_chunked_run_init(&run, n_kernel, chunk_n);
|
|
2507
|
+
ca_chunked_run_add(&run, ca1, i1);
|
|
2508
|
+
ca_chunked_run_add(&run, ca2, i2);
|
|
2509
|
+
run.kernel = ca_bincmp_chunk_kernel;
|
|
2510
|
+
run.data = &d;
|
|
2511
|
+
ca_chunked_run(&run);
|
|
1701
2512
|
}
|
|
1702
|
-
|
|
1703
|
-
ca_lazy_arena_release(s2_arena);
|
|
1704
|
-
ca_lazy_arena_release(s1_arena);
|
|
1705
|
-
ca_lazy_arena_exit();
|
|
1706
2513
|
}
|
|
1707
2514
|
}
|
|
1708
2515
|
}
|
|
@@ -1774,7 +2581,7 @@ ca_math_call (VALUE mod, VALUE arg, ID id)
|
|
|
1774
2581
|
return rb_funcall(rb_mMath, id, 1, arg);
|
|
1775
2582
|
}
|
|
1776
2583
|
else {
|
|
1777
|
-
rb_raise(
|
|
2584
|
+
rb_raise(rb_eTypeError, "unknown method for Math");
|
|
1778
2585
|
}
|
|
1779
2586
|
}
|
|
1780
2587
|
#endif
|
|
@@ -1784,7 +2591,7 @@ ca_math_call (VALUE mod, VALUE arg, ID id)
|
|
|
1784
2591
|
return rb_funcall(rb_mMath, id, 1, arg);
|
|
1785
2592
|
}
|
|
1786
2593
|
else {
|
|
1787
|
-
rb_raise(
|
|
2594
|
+
rb_raise(rb_eTypeError, "unknown method for Math");
|
|
1788
2595
|
}
|
|
1789
2596
|
}
|
|
1790
2597
|
}
|