carray 3.0.1 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +824 -0
- data/README.md +5 -6
- data/carray.gemspec +1 -1
- data/ext/ca_axis_dispatch.c +258 -4
- data/ext/ca_axis_group.c +368 -184
- data/ext/ca_binop_dispatch.c +54 -32
- data/ext/ca_binop_dispatch.h +1 -0
- data/ext/ca_categorical_iterator.c +90 -41
- data/ext/ca_compare.c +8 -2
- data/ext/ca_for_buffer.h +21 -68
- data/ext/ca_group_iter.c +27 -11
- data/ext/ca_kernel_iterator.c +960 -245
- data/ext/ca_kernel_iterator.h +352 -181
- data/ext/ca_monop_dispatch.c +4 -4
- data/ext/ca_monop_dispatch.h +4 -0
- data/ext/ca_obj_array.c +194 -42
- data/ext/ca_obj_bincmp.c +152 -121
- data/ext/ca_obj_binop.c +107 -210
- data/ext/ca_obj_bitarray.c +37 -24
- data/ext/ca_obj_bitfield.c +95 -62
- data/ext/ca_obj_block.c +5 -4
- data/ext/ca_obj_byte_swap.c +40 -40
- data/ext/ca_obj_const_string.c +180 -50
- data/ext/ca_obj_face.c +428 -62
- data/ext/ca_obj_face.h +107 -47
- data/ext/ca_obj_fake.c +21 -50
- data/ext/ca_obj_farray.c +1 -0
- data/ext/ca_obj_field.c +16 -3
- data/ext/ca_obj_fixlen_string.c +39 -26
- data/ext/ca_obj_grid.c +181 -124
- data/ext/ca_obj_meld.c +235 -55
- data/ext/ca_obj_moncmp.c +61 -61
- data/ext/ca_obj_monop.c +125 -107
- data/ext/ca_obj_object.c +551 -233
- data/ext/ca_obj_record.c +51 -16
- data/ext/ca_obj_reduce.c +5 -8
- data/ext/ca_obj_refer.c +117 -33
- data/ext/ca_obj_remap.c +57 -17
- data/ext/ca_obj_repeat.c +21 -5
- data/ext/ca_obj_roll.c +115 -11
- data/ext/ca_obj_select.c +88 -75
- data/ext/ca_obj_select_axis.c +58 -60
- data/ext/ca_obj_shift.c +22 -102
- data/ext/ca_obj_stack.c +223 -123
- data/ext/ca_obj_stride.c +267 -35
- data/ext/ca_obj_string.c +25 -22
- data/ext/ca_obj_tile.c +32 -16
- data/ext/ca_obj_time.c +23 -25
- data/ext/ca_obj_timedelta.c +21 -23
- data/ext/ca_obj_transpose.c +12 -3
- data/ext/ca_obj_triop.c +110 -117
- data/ext/ca_obj_window.c +157 -29
- data/ext/ca_op_ipower.c +25 -13
- data/ext/ca_op_powi.h +57 -30
- data/ext/ca_pow5_table.h +656 -0
- data/ext/ca_rng_normal.h +42 -0
- data/ext/ca_rng_xoshiro256pp.h +105 -0
- data/ext/ca_sort_kernels.h +70 -0
- data/ext/ca_sweep_engine.c +385 -198
- data/ext/ca_sweep_engine.h +31 -23
- data/ext/ca_triop_dispatch.c +4 -1
- data/ext/ca_triop_dispatch.h +5 -2
- data/ext/caframe_csv_reader.c +463 -0
- data/ext/carray.h +90 -28
- data/ext/carray_access.c +487 -231
- data/ext/carray_address_basis.c +731 -0
- data/ext/carray_attribute.c +45 -3
- data/ext/carray_bincount.c +32 -2
- data/ext/carray_broadcast.c +49 -17
- data/ext/carray_call_cfunc.c +667 -511
- data/ext/carray_cast.c +1556 -247
- data/ext/carray_cast_func.rb +34 -21
- data/ext/carray_conversion.c +197 -73
- data/ext/carray_copy.c +84 -38
- data/ext/carray_core.c +591 -119
- data/ext/carray_count.c +148 -96
- data/ext/carray_element.c +18 -27
- data/ext/carray_factorize.c +604 -291
- data/ext/carray_generate.c +76 -40
- data/ext/carray_histogram.c +152 -94
- data/ext/carray_hold.c +2 -1
- data/ext/carray_index_classifier.c +63 -96
- data/ext/carray_internal.h +400 -1
- data/ext/carray_lazy.c +834 -79
- data/ext/carray_loop.c +88 -66
- data/ext/carray_mask.c +705 -324
- data/ext/carray_math_kernel.h +1 -0
- data/ext/carray_median_percentile.c +263 -63
- data/ext/carray_memory_view.c +254 -248
- data/ext/carray_operator.c +1093 -363
- data/ext/carray_order.c +133 -69
- data/ext/carray_partition.c +102 -56
- data/ext/carray_random.c +582 -204
- data/ext/carray_scatter.c +308 -65
- data/ext/carray_segment.c +257 -0
- data/ext/carray_slab.c +183 -70
- data/ext/carray_sort.c +235 -240
- data/ext/carray_sort_kernel.c +8 -1
- data/ext/carray_struct.c +99 -21
- data/ext/carray_test.c +167 -64
- data/ext/carray_time_text.c +1182 -0
- data/ext/carray_utils.c +363 -98
- data/ext/extconf.rb +27 -2
- data/ext/mk_call_cfunc.rb +103 -118
- data/ext/mk_pow5_table.rb +35 -0
- data/ext/mkkernel.rb +1349 -851
- data/ext/ruby_carray.c +17 -2
- data/ext/version.h +4 -4
- data/ext/version.rb +4 -2
- data/lib/carray/attribute.rb +39 -0
- data/lib/carray/autoload_carray.rb +24 -3
- data/lib/carray/autoload_method_extension.rb +18 -0
- data/lib/carray/axis_group.rb +245 -20
- data/lib/carray/basics.rb +95 -25
- data/lib/carray/bincount_nd.rb +6 -0
- data/lib/carray/block_iterator.rb +200 -54
- data/lib/carray/boolean_reduce.rb +21 -29
- data/lib/carray/categorical.rb +244 -59
- data/lib/carray/categorical_iterator.rb +391 -757
- data/lib/carray/complex.rb +36 -29
- data/lib/carray/conditional.rb +94 -7
- data/lib/carray/const_string.rb +177 -27
- data/lib/carray/construct.rb +40 -0
- data/lib/carray/data_type_extension.rb +3 -0
- data/lib/carray/data_type_limits.rb +91 -0
- data/lib/carray/fixlen_string.rb +1 -1
- data/lib/carray/frame/concat.rb +157 -6
- data/lib/carray/frame/convert.rb +13 -7
- data/lib/carray/frame/csv_parser.rb +590 -76
- data/lib/carray/frame/describe.rb +79 -0
- data/lib/carray/frame/frame.rb +205 -35
- data/lib/carray/frame/group.rb +199 -40
- data/lib/carray/frame/io.rb +452 -60
- data/lib/carray/frame/join.rb +134 -46
- data/lib/carray/frame/records.rb +54 -19
- data/lib/carray/frame/reshape.rb +448 -0
- data/lib/carray/frame/sort.rb +26 -2
- data/lib/carray/frame/verbs.rb +726 -56
- data/lib/carray/frame.rb +2 -0
- data/lib/carray/fuse_source.rb +111 -2
- data/lib/carray/fusion.rb +229 -38
- data/lib/carray/inspect.rb +71 -20
- data/lib/carray/iterator.rb +157 -2
- data/lib/carray/lazy.rb +54 -67
- data/lib/carray/mask_gap_fill.rb +66 -50
- data/lib/carray/math.rb +32 -19
- data/lib/carray/meld_reduce.rb +40 -7
- data/lib/carray/methods/align_addr.rb +8 -4
- data/lib/carray/methods/bincount.rb +2 -2
- data/lib/carray/methods/bit_string.rb +14 -3
- data/lib/carray/methods/composition.rb +16 -10
- data/lib/carray/methods/discovery_along.rb +74 -0
- data/lib/carray/methods/factorize.rb +50 -0
- data/lib/carray/methods/gather_nd.rb +13 -2
- data/lib/carray/methods/insert_block.rb +9 -14
- data/lib/carray/methods/is_in.rb +130 -6
- data/lib/carray/methods/join.rb +1 -1
- data/lib/carray/methods/locate_addr.rb +235 -21
- data/lib/carray/methods/mask_duplicates.rb +35 -1
- data/lib/carray/methods/mode.rb +18 -1
- data/lib/carray/methods/nunique.rb +22 -1
- data/lib/carray/methods/repeat.rb +91 -0
- data/lib/carray/methods/resize.rb +21 -23
- data/lib/carray/methods/snap.rb +34 -12
- data/lib/carray/methods/unique.rb +52 -3
- data/lib/carray/record_order.rb +208 -0
- data/lib/carray/rng.rb +86 -0
- data/lib/carray/runtime.rb +14 -11
- data/lib/carray/segment_iterator.rb +990 -0
- data/lib/carray/serialize.rb +197 -27
- data/lib/carray/slab_iterator.rb +109 -40
- data/lib/carray/stack.rb +10 -4
- data/lib/carray/string_operation_extension.rb +9 -4
- data/lib/carray/struct.rb +142 -51
- data/lib/carray/struct_builder.rb +110 -14
- data/lib/carray/table.rb +2 -2
- data/lib/carray/time.rb +535 -105
- data/lib/carray/window_iterator.rb +259 -82
- data/lib/carray.rb +4 -2
- data/yard-stubs/ca_obj_array.rb +4 -4
- data/yard-stubs/ca_obj_bitarray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +23 -12
- data/yard-stubs/ca_obj_block.rb +2 -7
- data/yard-stubs/ca_obj_fake.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +1 -1
- data/yard-stubs/ca_obj_refer.rb +3 -3
- data/yard-stubs/ca_obj_roll.rb +0 -1
- data/yard-stubs/ca_obj_stride.rb +10 -7
- data/yard-stubs/ca_obj_transpose.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +11 -4
- data/yard-stubs/carray_access.rb +6 -5
- data/yard-stubs/carray_broadcast.rb +2 -2
- data/yard-stubs/carray_cast.rb +78 -43
- data/yard-stubs/carray_conversion.rb +13 -1
- data/yard-stubs/carray_copy.rb +8 -3
- data/yard-stubs/carray_core.rb +3 -83
- data/yard-stubs/carray_count.rb +16 -2
- data/yard-stubs/carray_lazy.rb +210 -1
- data/yard-stubs/carray_loop.rb +4 -1
- data/yard-stubs/carray_mask.rb +34 -10
- data/yard-stubs/carray_math.rb +1499 -4
- data/yard-stubs/carray_median_percentile.rb +26 -8
- data/yard-stubs/carray_order.rb +43 -82
- data/yard-stubs/carray_scatter.rb +10 -3
- data/yard-stubs/carray_segment.rb +72 -0
- data/yard-stubs/carray_slab.rb +11 -8
- data/yard-stubs/carray_sort.rb +50 -8
- data/yard-stubs/carray_test.rb +10 -0
- data/yard-stubs/carray_utils.rb +0 -26
- metadata +19 -19
- data/ext/ca_for_each_element.h +0 -239
- data/ext/carray_build_flags.h +0 -3
- data/ext/carray_config.h +0 -28
- data/ext/carray_kernels_bincmp.c +0 -4446
- data/ext/carray_kernels_binop.c +0 -11001
- data/ext/carray_kernels_init.c +0 -1131
- data/ext/carray_kernels_map.c +0 -3467
- data/ext/carray_kernels_moncmp.c +0 -2097
- data/ext/carray_kernels_monop.c +0 -18313
- data/ext/carray_kernels_reduce_aggregate.c +0 -25837
- data/ext/carray_kernels_reduce_boolean.c +0 -330
- data/ext/carray_kernels_reduce_cumulative.c +0 -14593
- data/ext/carray_kernels_reduce_extreme.c +0 -16948
- data/ext/carray_kernels_reduce_variance.c +0 -3910
- data/ext/carray_kernels_scan.c +0 -3693
- data/ext/carray_kernels_search.c +0 -32138
- data/ext/carray_kernels_sort.c +0 -10626
- data/ext/carray_kernels_triop.c +0 -1392
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
# max / min -> value data type, empty/all-masked = MASKED
|
|
36
36
|
# mean -> float64, empty/all-masked = MASKED
|
|
37
37
|
# median / percentile -> float64, empty/all-masked = MASKED
|
|
38
|
-
# variance / stddev (sample, ddof=1) -> float64,
|
|
39
|
-
#
|
|
38
|
+
# variance / stddev (sample, ddof=1) -> float64, fewer than two present
|
|
39
|
+
# values = MASKED
|
|
40
40
|
# all / any -> boolean (boolean value data type only)
|
|
41
41
|
# labels -> cat.labels
|
|
42
42
|
#
|
|
@@ -60,17 +60,17 @@
|
|
|
60
60
|
# ----------------------------------------------------------------------------
|
|
61
61
|
|
|
62
62
|
require "carray"
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
class CACategoricalIterator <
|
|
63
|
+
require "carray/segment_iterator"
|
|
64
|
+
|
|
65
|
+
# A CAIterator over the categories of a CACategorical. It is a
|
|
66
|
+
# CASegmentIterator over a category-sorted copy of the value: construction
|
|
67
|
+
# gathers the value through the categorical's cached grouping plan, so each
|
|
68
|
+
# category is one contiguous segment, and every reduction without `axis:` is
|
|
69
|
+
# the segment reduction over that copy (one value per category, in {#labels}
|
|
70
|
+
# order). This class adds the construction, the labels, and the per-fiber
|
|
71
|
+
# form (`axis:`), which classifies each fiber along an axis and keeps the
|
|
72
|
+
# other axes.
|
|
73
|
+
class CACategoricalIterator < CASegmentIterator
|
|
74
74
|
|
|
75
75
|
# value : the payload CArray to reduce, one cell per categorical cell.
|
|
76
76
|
# cat : the CACategorical carrying the classification.
|
|
@@ -103,8 +103,9 @@ class CACategoricalIterator < CAIterator
|
|
|
103
103
|
# index): offsets[c] = sum of counts[0...c]. Both come off the shared
|
|
104
104
|
# plan, so the counting sort is not repeated here.
|
|
105
105
|
@elements = cat.category_sizes.int64
|
|
106
|
-
nvalid =
|
|
106
|
+
nvalid = counts.sum
|
|
107
107
|
@offsets = cat.reduceat_index # cached segment STARTS (int64[k])
|
|
108
|
+
@bounds = CArray.segment_offsets(lengths: @elements) # every boundary (int64[k+1])
|
|
108
109
|
# Group-major source indices = the valid prefix of the cached sort_addr.
|
|
109
110
|
# With no classified cell the prefix is empty (and slicing a length-0
|
|
110
111
|
# sort_addr would be out of range), so take the empty permutation directly.
|
|
@@ -118,44 +119,35 @@ class CACategoricalIterator < CAIterator
|
|
|
118
119
|
# is nothing to gather — an empty index into an empty source is out of
|
|
119
120
|
# range — so build the empty grouped buffer directly.
|
|
120
121
|
@grouped = nvalid > 0 ? value.reshape(value.elements)[@perm].copy
|
|
121
|
-
: CArray.new(value.data_type, [0])
|
|
122
|
-
@empty = CArray.new(
|
|
122
|
+
: CArray.new(value.data_type, [0], bytes: value.bytes)
|
|
123
|
+
@empty = CArray.new(grouped.data_type, [0], bytes: grouped.bytes)
|
|
123
124
|
else
|
|
124
125
|
# Shape mismatch: only per-fiber axis: dispatch could still work. With a
|
|
125
126
|
# 1-D value there is no fiber structure to broadcast into, so a mismatch
|
|
126
127
|
# is unrecoverable (preserves the old strict check). For higher-rank
|
|
127
128
|
# value, defer validation to reduce time — check only that cat.ndim fits
|
|
128
129
|
# one of the 3 axis: cases;
|
|
129
|
-
#
|
|
130
|
-
#
|
|
130
|
+
# A no-axis reduce has no grouped buffer to work from. The reason is
|
|
131
|
+
# recorded here and raised from wherever one is asked for, so what
|
|
132
|
+
# surfaces names the mismatch instead of being whatever NoMethodError
|
|
133
|
+
# the nil produced first.
|
|
134
|
+
mismatch = "group_by_category: value.elements (#{value.elements}) != " \
|
|
135
|
+
"cat.elements (#{cat.elements})"
|
|
131
136
|
if value.ndim == 1 ||
|
|
132
137
|
! [1, value.ndim - 1, value.ndim].include?(cat.ndim)
|
|
133
138
|
raise ArgumentError,
|
|
134
|
-
|
|
135
|
-
"cat.elements (#{cat.elements})" +
|
|
139
|
+
mismatch +
|
|
136
140
|
(value.ndim == 1 ? "" :
|
|
137
141
|
". For per-fiber reduce use `.sum(axis: k)`; cat.ndim=" \
|
|
138
142
|
"#{cat.ndim} must be 1 (case A), #{value.ndim} (case B), " \
|
|
139
143
|
"or #{value.ndim - 1} (band-only) for h.ndim=#{value.ndim}.")
|
|
140
144
|
end
|
|
145
|
+
@no_flat = mismatch + ". This iterator answers only the per-fiber form, " \
|
|
146
|
+
"`.<reduce>(axis: k)`."
|
|
141
147
|
end
|
|
142
148
|
self
|
|
143
149
|
end
|
|
144
150
|
|
|
145
|
-
# @overload each { |members| ... }
|
|
146
|
-
# Yields each category's members (a CArray slice of the grouped copy, in
|
|
147
|
-
# {#labels} order; an empty category yields an empty array). Without a
|
|
148
|
-
# block, returns an Enumerator. This is the own iteration that drives the
|
|
149
|
-
# inherited Enumerable methods (map / count / to_a / ...); it does not use
|
|
150
|
-
# the CAIterator base each / kernel_at_addr path.
|
|
151
|
-
# @yieldparam members [CArray]
|
|
152
|
-
# @return [Enumerator, self]
|
|
153
|
-
def each
|
|
154
|
-
return to_enum(:each) unless block_given?
|
|
155
|
-
@k.times { |c| yield group_slice(c) }
|
|
156
|
-
self
|
|
157
|
-
end
|
|
158
|
-
|
|
159
151
|
# @overload labels
|
|
160
152
|
# Returns the category vocabulary the results are aligned to.
|
|
161
153
|
# @return [Array]
|
|
@@ -169,16 +161,6 @@ class CACategoricalIterator < CAIterator
|
|
|
169
161
|
@k
|
|
170
162
|
end
|
|
171
163
|
|
|
172
|
-
# @overload elements
|
|
173
|
-
# Returns per-group cell counts (classified cells, including value-masked
|
|
174
|
-
# ones; = `cat.category_sizes`), a length-ngroups CArray aligned to
|
|
175
|
-
# {#labels}. The CAIterator count-family member — `CArray#elements`
|
|
176
|
-
# (structural, mask-independent) lifted per group.
|
|
177
|
-
# @return [CArray]
|
|
178
|
-
def elements
|
|
179
|
-
@elements
|
|
180
|
-
end
|
|
181
|
-
|
|
182
164
|
# Group-vocabulary alias for {#elements}; reads naturally next to
|
|
183
165
|
# {#ngroups} and mirrors `CACategorical#category_sizes`.
|
|
184
166
|
alias group_sizes elements
|
|
@@ -189,552 +171,256 @@ class CACategoricalIterator < CAIterator
|
|
|
189
171
|
# internal grouped/value/codes buffers.
|
|
190
172
|
# @return [String]
|
|
191
173
|
def inspect
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
# — the denominator the value reductions actually divide by. Equals
|
|
199
|
-
# {#elements} unless the value carries a mask. A count is always defined,
|
|
200
|
-
# so an empty category is `0` (never masked).
|
|
201
|
-
# @return [CArray]
|
|
202
|
-
# @overload count_not_masked(axis:)
|
|
203
|
-
# Per-fiber per-category count of present (non-masked) values along `axis`
|
|
204
|
-
# (int64, shape [K, ...band]). Empty cells are `0`.
|
|
205
|
-
# @param axis [Integer]
|
|
206
|
-
# @return [CArray]
|
|
207
|
-
def count_not_masked(axis: nil)
|
|
208
|
-
return axis_moments(axis)[:count] if axis
|
|
209
|
-
m = moments
|
|
210
|
-
m ? m[:count] : per_category(CA_INT64) { |s| s.count_not_masked }
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
# @overload count(v = <none>)
|
|
214
|
-
# Per-category count, mirroring `CArray#count` per group. No argument
|
|
215
|
-
# returns {#count_not_masked} (present cells); `count(UNDEF)` returns
|
|
216
|
-
# {#count_masked}; `count(v)` counts cells whose value equals `v`.
|
|
217
|
-
# @return [CArray] length-k int64, aligned to {#labels}
|
|
218
|
-
# @overload count(axis:)
|
|
219
|
-
# No-arg + axis: = per-fiber per-category count_not_masked (shape [K, ...band]).
|
|
220
|
-
# `count(v, axis:)` (value equality) and `count(UNDEF, axis:)` are not
|
|
221
|
-
# implemented; use them without `axis:`.
|
|
222
|
-
# @param axis [Integer]
|
|
223
|
-
# @return [CArray]
|
|
224
|
-
def count (*args, axis: nil)
|
|
225
|
-
if axis
|
|
226
|
-
return count_not_masked(axis: axis) if args.empty?
|
|
227
|
-
raise NotImplementedError,
|
|
228
|
-
"CACategoricalIterator#count(v, axis:) is not implemented — " \
|
|
229
|
-
"value-equality count is available without axis:."
|
|
230
|
-
end
|
|
231
|
-
return count_not_masked if args.empty?
|
|
232
|
-
# Delegate per group to CArray#count (handles count(UNDEF) -> masked count and
|
|
233
|
-
# count(v) alike, with core's exact data type equality). The group slice is a
|
|
234
|
-
# CABlock, whose own #count is the block geometry accessor, so dispatch
|
|
235
|
-
# CArray#count explicitly. (Not fused: a value-equality reduceat would have
|
|
236
|
-
# to reproduce core's cross-type / out-of-range equality exactly.)
|
|
237
|
-
cnt = CArray.instance_method(:count)
|
|
238
|
-
per_category(CA_INT64) { |s| cnt.bind_call(s, *args) }
|
|
239
|
-
end
|
|
240
|
-
|
|
241
|
-
# @overload count_masked
|
|
242
|
-
# Returns the per-category count of masked (missing) values as int64.
|
|
243
|
-
# Empty categories are `0`.
|
|
244
|
-
# @return [CArray]
|
|
245
|
-
# @overload count_masked(axis:)
|
|
246
|
-
# Not implemented; call it without `axis:`.
|
|
247
|
-
# @param axis [Integer]
|
|
248
|
-
# @return [CArray]
|
|
249
|
-
def count_masked(axis: nil)
|
|
250
|
-
if axis
|
|
251
|
-
raise NotImplementedError,
|
|
252
|
-
"CACategoricalIterator#count_masked(axis:) is not implemented — " \
|
|
253
|
-
"call it without axis:."
|
|
254
|
-
end
|
|
255
|
-
m = moments
|
|
256
|
-
m ? @elements - m[:count] : per_category(CA_INT64) { |s| s.count_masked }
|
|
257
|
-
end
|
|
258
|
-
|
|
259
|
-
# @overload sum
|
|
260
|
-
# Returns per-category sums in the data type `CArray#sum` promotes the value
|
|
261
|
-
# to (float64 for an integer value). `accumulate` is the same fold kept in
|
|
262
|
-
# the value's own type. An empty or fully-masked category sums the empty
|
|
263
|
-
# set, which is the additive identity `0` (unmasked) — the same contract as
|
|
264
|
-
# `CArray#sum` on an empty / all-masked array.
|
|
265
|
-
# @return [CArray]
|
|
266
|
-
# @overload sum(axis:)
|
|
267
|
-
# Returns per-category sums per fiber along `axis`. Cat may be 1-D (case
|
|
268
|
-
# A, broadcasts across band axes), same rank as source (case B, per-fiber
|
|
269
|
-
# independent classifier), or one rank less (band-only, constant along
|
|
270
|
-
# reduce axis). Output shape = `[K, ...source.shape without axis]`.
|
|
271
|
-
# @param axis [Integer] reduce axis of the source value.
|
|
272
|
-
# @return [CArray]
|
|
273
|
-
def sum(axis: nil)
|
|
274
|
-
return axis_sum(axis) if axis
|
|
275
|
-
m = moments
|
|
276
|
-
return per_category(core_reduce_type(:sum)) { |s| s.sum } unless m
|
|
277
|
-
m[:sum].copy # the moments sum IS the core fold (empty -> 0.0)
|
|
278
|
-
end
|
|
279
|
-
|
|
280
|
-
# @overload accumulate
|
|
281
|
-
# Returns per-category sums folded in the value's own data type, wrapping at
|
|
282
|
-
# its width, as the core `accumulate` does. This is the exact in-type fold:
|
|
283
|
-
# `sum` reads its answer off a float64 moment and casts back, so it loses
|
|
284
|
-
# the low bits of a wide integer payload and does not wrap. An empty or
|
|
285
|
-
# fully-masked category accumulates the empty set, the additive identity `0`
|
|
286
|
-
# (unmasked).
|
|
287
|
-
# @return [CArray]
|
|
288
|
-
# @overload accumulate(axis:)
|
|
289
|
-
# Per-fiber per-category in-type sums along `axis`. Output shape =
|
|
290
|
-
# `[K, ...source.shape without axis]`.
|
|
291
|
-
# @param axis [Integer] reduce axis of the source value.
|
|
292
|
-
# @return [CArray]
|
|
293
|
-
def accumulate(axis: nil)
|
|
294
|
-
return axis_by_masked_copy(axis, :accumulate, core_reduce_type(:accumulate)) if axis
|
|
295
|
-
per_category(core_reduce_type(:accumulate)) { |s| s.accumulate }
|
|
296
|
-
end
|
|
297
|
-
|
|
298
|
-
# @overload max
|
|
299
|
-
# Returns per-category maxima in the value data type. Empty categories are
|
|
300
|
-
# MASKED.
|
|
301
|
-
# @return [CArray]
|
|
302
|
-
# @overload max(axis:)
|
|
303
|
-
# Per-fiber per-category maxima along `axis` (h's data type, masked where empty).
|
|
304
|
-
# @param axis [Integer]
|
|
305
|
-
# @return [CArray]
|
|
306
|
-
def max(axis: nil)
|
|
307
|
-
return axis_moments(axis)[:max] if axis
|
|
308
|
-
m = moments
|
|
309
|
-
m ? m[:max] : per_category(core_reduce_type(:max)) { |s| s.max }
|
|
310
|
-
end
|
|
311
|
-
|
|
312
|
-
# @overload min
|
|
313
|
-
# Returns per-category minima in the value data type. Empty categories are
|
|
314
|
-
# MASKED.
|
|
315
|
-
# @return [CArray]
|
|
316
|
-
# @overload min(axis:)
|
|
317
|
-
# Per-fiber per-category minima along `axis` (h's data type, masked where empty).
|
|
318
|
-
# @param axis [Integer]
|
|
319
|
-
# @return [CArray]
|
|
320
|
-
def min(axis: nil)
|
|
321
|
-
return axis_moments(axis)[:min] if axis
|
|
322
|
-
m = moments
|
|
323
|
-
m ? m[:min] : per_category(core_reduce_type(:min)) { |s| s.min }
|
|
324
|
-
end
|
|
325
|
-
|
|
326
|
-
# @overload mean
|
|
327
|
-
# Returns per-category means as float64. Empty categories are MASKED.
|
|
328
|
-
# @return [CArray]
|
|
329
|
-
# @overload mean(axis:)
|
|
330
|
-
# Per-fiber per-category means (float64, empty group cells MASKED).
|
|
331
|
-
# @param axis [Integer]
|
|
332
|
-
# @return [CArray]
|
|
333
|
-
def mean(axis: nil)
|
|
334
|
-
return axis_mean(axis) if axis
|
|
335
|
-
m = moments
|
|
336
|
-
return per_category(core_reduce_type(:mean)) { |s| s.mean } unless m
|
|
337
|
-
cnt = m[:count]
|
|
338
|
-
out = m[:sum] / cnt.float64 # count 0 -> NaN, masked next
|
|
339
|
-
out[cnt.eq(0)] = UNDEF # empty / all-masked category -> MASKED
|
|
340
|
-
out
|
|
341
|
-
end
|
|
342
|
-
|
|
343
|
-
# @overload median
|
|
344
|
-
# Returns per-category medians as float64. Empty categories are MASKED.
|
|
345
|
-
# @return [CArray]
|
|
346
|
-
def median(axis: nil)
|
|
347
|
-
axis_order_stat_defer!(:median) if axis
|
|
348
|
-
percentile(50.0)
|
|
349
|
-
end
|
|
350
|
-
|
|
351
|
-
# @overload percentile(p)
|
|
352
|
-
# Returns the per-category `p`-th percentile as float64 (`p` in 0..100,
|
|
353
|
-
# `:linear` interpolation, matching `CArray#percentile`). Empty categories
|
|
354
|
-
# are MASKED. Order statistics need every value of a group held together —
|
|
355
|
-
# this is the reduceat that only the eager grouped copy can serve.
|
|
356
|
-
# @param p [Numeric] percentile in 0..100.
|
|
357
|
-
# @return [CArray]
|
|
358
|
-
def percentile (p, axis: nil)
|
|
359
|
-
axis_order_stat_defer!(:percentile) if axis
|
|
360
|
-
unless MONOID_TYPES.include?(@grouped.data_type)
|
|
361
|
-
return per_category(core_reduce_type(:percentile, p)) { |s| s.percentile(p) }
|
|
362
|
-
end
|
|
363
|
-
out = CArray.float64(@k)
|
|
364
|
-
@grouped.send(:__reduceat_percentile__, @offsets, p.to_f, out)
|
|
365
|
-
out
|
|
366
|
-
end
|
|
367
|
-
|
|
368
|
-
# @overload quantile
|
|
369
|
-
# Returns the per-category five-number summary `[min, Q1, median, Q3, max]`
|
|
370
|
-
# as five length-k float64 CArrays (matching `CArray#quantile`): the
|
|
371
|
-
# percentiles at 0 / 25 / 50 / 75 / 100. Empty / all-masked categories are
|
|
372
|
-
# MASKED. For a single fraction q in 0..1 use `percentile(q * 100)`.
|
|
373
|
-
# @return [Array<CArray>]
|
|
374
|
-
def quantile
|
|
375
|
-
unless MONOID_TYPES.include?(@grouped.data_type)
|
|
376
|
-
return [0, 25, 50, 75, 100].map { |p| percentile(p) }
|
|
377
|
-
end
|
|
378
|
-
outs = Array.new(5) { CArray.float64(@k) }
|
|
379
|
-
@grouped.send(:__reduceat_quantile__, @offsets, *outs)
|
|
380
|
-
outs
|
|
381
|
-
end
|
|
382
|
-
|
|
383
|
-
# @overload variance
|
|
384
|
-
# Returns per-category SAMPLE variance (ddof=1) as float64. Matches
|
|
385
|
-
# `CArray#variance` per group: an empty or fully-masked category is MASKED,
|
|
386
|
-
# a single-value category is `0.0` (CArray's n=1 contract), n>=2 is the
|
|
387
|
-
# sample variance.
|
|
388
|
-
# @return [CArray]
|
|
389
|
-
def variance(axis: nil)
|
|
390
|
-
return axis_by_masked_copy(axis, :variance) if axis
|
|
391
|
-
m = moments
|
|
392
|
-
return per_category(core_reduce_type(:variance)) { |s| s.variance } unless m
|
|
393
|
-
cnt = m[:count]
|
|
394
|
-
means = m[:sum] / cnt.float64 # per-segment mean (garbage where count 0/1,
|
|
395
|
-
out = CArray.float64(@k) # ignored by the kernel's n<2 guards)
|
|
396
|
-
@grouped.send(:__reduceat_variance__, @offsets, means, cnt, out)
|
|
397
|
-
out
|
|
398
|
-
end
|
|
399
|
-
|
|
400
|
-
# @overload stddev
|
|
401
|
-
# Returns per-category SAMPLE standard deviation (ddof=1) as float64.
|
|
402
|
-
# Matches `CArray#stddev` per group (empty / all-masked MASKED,
|
|
403
|
-
# single-value `0.0`).
|
|
404
|
-
# @return [CArray]
|
|
405
|
-
def stddev(axis: nil)
|
|
406
|
-
return axis_by_masked_copy(axis, :stddev) if axis
|
|
407
|
-
m = moments
|
|
408
|
-
return per_category(core_reduce_type(:stddev)) { |s| s.stddev } unless m
|
|
409
|
-
variance.sqrt # sqrt propagates the n=0 mask
|
|
410
|
-
end
|
|
411
|
-
|
|
412
|
-
# @overload prod
|
|
413
|
-
# Returns per-category products as float64 (matching `CArray#prod`). An
|
|
414
|
-
# empty / fully-masked category is `1.0` (the multiplicative identity).
|
|
415
|
-
# Single-pass reduceat for numeric values; per-group fallback otherwise.
|
|
416
|
-
# @return [CArray]
|
|
417
|
-
# @overload prod(axis:)
|
|
418
|
-
# Per-fiber per-category products (float64, shape [K, ...band]). Empty
|
|
419
|
-
# group cells `1.0` (identity).
|
|
420
|
-
# @param axis [Integer]
|
|
421
|
-
# @return [CArray]
|
|
422
|
-
def prod(axis: nil)
|
|
423
|
-
return axis_prod(axis) if axis
|
|
424
|
-
return per_category(core_reduce_type(:prod)) { |s| s.prod } unless MONOID_TYPES.include?(@grouped.data_type)
|
|
425
|
-
out = CArray.float64(@k)
|
|
426
|
-
@grouped.send(:__reduceat_prod__, @offsets, out)
|
|
427
|
-
out
|
|
428
|
-
end
|
|
429
|
-
|
|
430
|
-
# @overload all
|
|
431
|
-
# Returns the per-category `all` as boolean (matching `CArray#all`): true
|
|
432
|
-
# iff every present value is truthy (empty category -> true, vacuously).
|
|
433
|
-
# The value data type must be boolean, as for `CArray#all`.
|
|
434
|
-
# @return [CArray]
|
|
435
|
-
def all
|
|
436
|
-
aa = all_any
|
|
437
|
-
aa ? aa[:all] : per_category(CA_BOOLEAN) { |s| s.all }
|
|
438
|
-
end
|
|
439
|
-
|
|
440
|
-
# @overload any
|
|
441
|
-
# Returns the per-category `any` as boolean (matching `CArray#any`): true
|
|
442
|
-
# iff some present value is truthy (empty category -> false). The value
|
|
443
|
-
# data type must be boolean, as for `CArray#any`.
|
|
444
|
-
# @return [CArray]
|
|
445
|
-
def any
|
|
446
|
-
aa = all_any
|
|
447
|
-
aa ? aa[:any] : per_category(CA_BOOLEAN) { |s| s.any }
|
|
448
|
-
end
|
|
449
|
-
|
|
450
|
-
# ---- tier 2 (fused / population / position) ------------------------------
|
|
451
|
-
|
|
452
|
-
# @overload minmax
|
|
453
|
-
# Returns the per-category `[min, max]` pair (each a length-k CArray in the
|
|
454
|
-
# value data type; empty categories MASKED), matching `CArray#minmax`. Both come
|
|
455
|
-
# from the single cached moments pass.
|
|
456
|
-
# @return [Array<CArray>]
|
|
457
|
-
# @overload minmax(axis:)
|
|
458
|
-
# Per-fiber `[min_ca, max_ca]` along `axis` (each shape [K, ...band], h's data type,
|
|
459
|
-
# empty group cells MASKED). Ruby Array of two CArrays, not stacked.
|
|
460
|
-
# @param axis [Integer]
|
|
461
|
-
# @return [Array<CArray>]
|
|
462
|
-
def minmax(axis: nil)
|
|
463
|
-
return [min(axis: axis), max(axis: axis)] if axis
|
|
464
|
-
[min, max]
|
|
465
|
-
end
|
|
466
|
-
|
|
467
|
-
# @overload variancep
|
|
468
|
-
# Per-category POPULATION variance (ddof=0) as float64, matching
|
|
469
|
-
# `CArray#variancep`: empty / all-masked -> MASKED, single value -> 0.0.
|
|
470
|
-
# Derived from the sample variance (variancep = variance * (n-1) / n), so it
|
|
471
|
-
# reuses the centred two-pass kernel with no extra walk.
|
|
472
|
-
# @return [CArray]
|
|
473
|
-
def variancep(axis: nil)
|
|
474
|
-
return axis_by_masked_copy(axis, :variancep) if axis
|
|
475
|
-
m = moments
|
|
476
|
-
return per_category(core_reduce_type(:variancep)) { |s| s.variancep } unless m
|
|
477
|
-
cnt = m[:count]
|
|
478
|
-
vp = variance * (cnt - 1).float64 / cnt.float64
|
|
479
|
-
vp[cnt.eq(0)] = UNDEF # empty / all-masked stays masked
|
|
480
|
-
vp
|
|
481
|
-
end
|
|
482
|
-
|
|
483
|
-
# @overload stddevp
|
|
484
|
-
# Per-category POPULATION standard deviation (ddof=0) as float64.
|
|
485
|
-
# @return [CArray]
|
|
486
|
-
# @overload stddevp(axis:)
|
|
487
|
-
# Per-fiber per-category population stddev (float64, empty group cells MASKED).
|
|
488
|
-
# @param axis [Integer]
|
|
489
|
-
# @return [CArray]
|
|
490
|
-
def stddevp(axis: nil)
|
|
491
|
-
return axis_by_masked_copy(axis, :stddevp) if axis
|
|
492
|
-
m = moments
|
|
493
|
-
return per_category(core_reduce_type(:stddevp)) { |s| s.stddevp } unless m
|
|
494
|
-
variancep.sqrt
|
|
495
|
-
end
|
|
496
|
-
|
|
497
|
-
# @overload min_index
|
|
498
|
-
# Per-category group-local index of the minimum — the position within the
|
|
499
|
-
# category's members (source order) — matching `CArray#min_index` per group.
|
|
500
|
-
# Empty / all-masked categories are MASKED. Single-pass fused reduceat for
|
|
501
|
-
# numeric values; per-group fallback otherwise.
|
|
502
|
-
# @return [CArray] length-k int64
|
|
503
|
-
def min_index
|
|
504
|
-
am = arg_minmax
|
|
505
|
-
am ? am[:min] : per_category(CA_INT64) { |s| s.min_index }
|
|
506
|
-
end
|
|
507
|
-
|
|
508
|
-
# @overload max_index
|
|
509
|
-
# Per-category group-local index of the maximum. See {#min_index}.
|
|
510
|
-
# @return [CArray] length-k int64
|
|
511
|
-
def max_index
|
|
512
|
-
am = arg_minmax
|
|
513
|
-
am ? am[:max] : per_category(CA_INT64) { |s| s.max_index }
|
|
514
|
-
end
|
|
515
|
-
|
|
516
|
-
# @overload min_addr
|
|
517
|
-
# Per-category flat source address of the minimum — which cell of the source
|
|
518
|
-
# value holds it, matching `CArray#min_addr` per group. Unlike {#min_index}
|
|
519
|
-
# (the group-local rank) this indexes back into the original array
|
|
520
|
-
# (`value.reshape(value.elements)[grp.min_addr]`). Empty categories MASKED.
|
|
521
|
-
# @return [CArray] length-k int64
|
|
522
|
-
def min_addr
|
|
523
|
-
group_addr(min_index)
|
|
524
|
-
end
|
|
525
|
-
|
|
526
|
-
# @overload max_addr
|
|
527
|
-
# Per-category flat source address of the maximum. See {#min_addr}.
|
|
528
|
-
# @return [CArray] length-k int64
|
|
529
|
-
def max_addr
|
|
530
|
-
group_addr(max_index)
|
|
174
|
+
# An iterator that answers only the per-fiber form has no per-category
|
|
175
|
+
# counts to show. Saying so beats both raising -- inspect is what you
|
|
176
|
+
# reach for when something is already puzzling -- and printing an empty
|
|
177
|
+
# list, which reads as a grouping that classified nothing.
|
|
178
|
+
tail = @elements ? "elements=#{@elements.to_a.inspect}" : "per-fiber only"
|
|
179
|
+
"#<#{self.class} ngroups=#{@k} labels=#{@labels.inspect} #{tail}>"
|
|
531
180
|
end
|
|
532
181
|
|
|
533
|
-
#
|
|
534
|
-
# Per-category sort by flat source address. Returns a length-nvalid
|
|
535
|
-
# (= `elements.sum`) int64 CArray of the flat SOURCE addresses that sort each
|
|
536
|
-
# category's members, in group-major order: segment `c` holds category `c`'s
|
|
537
|
-
# source addresses in ascending-value order, segments concatenated in
|
|
538
|
-
# {#labels} order. So `value.reshape(value.elements)[grp.sort_addr]` yields
|
|
539
|
-
# the values grouped and sorted within each group, and splitting by the
|
|
540
|
-
# {#elements} prefix sum gives per-group. Excluded cells (in no category) are
|
|
541
|
-
# omitted. A masked value sorts to the tail of its segment (as `CArray#sort`
|
|
542
|
-
# sends masked cells to the end), so with a mask the first address is the
|
|
543
|
-
# minimum but the last is the masked cell, not the maximum.
|
|
182
|
+
# ---- the per-fiber form ---------------------------------------------------
|
|
544
183
|
#
|
|
545
|
-
#
|
|
546
|
-
#
|
|
547
|
-
#
|
|
548
|
-
#
|
|
549
|
-
#
|
|
550
|
-
# @
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
184
|
+
# These reductions also take `axis:` on a categorical. Without it each is
|
|
185
|
+
# the CASegmentIterator reduction of the same name, over the category-sorted
|
|
186
|
+
# copy: one value per category, in {#labels} order.
|
|
187
|
+
#
|
|
188
|
+
# @!method accumulate(axis: nil)
|
|
189
|
+
# @overload accumulate
|
|
190
|
+
# The per-category form: {CASegmentIterator#accumulate}, one value per category.
|
|
191
|
+
# @overload accumulate(axis:)
|
|
192
|
+
# Per-fiber per-category in-type sums along `axis`. Output shape =
|
|
193
|
+
# `[K, ...source.shape without axis]`.
|
|
194
|
+
# @param axis [Integer] reduce axis of the source value.
|
|
195
|
+
# @return [CArray]
|
|
196
|
+
# @!method count(*args, axis: nil)
|
|
197
|
+
# @overload count
|
|
198
|
+
# The per-category form: {CASegmentIterator#count}, one value per category.
|
|
199
|
+
# @overload count(axis:)
|
|
200
|
+
# No-arg + axis: = per-fiber per-category count_not_masked (shape [K, ...band]).
|
|
201
|
+
# `count(v, axis:)` (value equality) and `count(UNDEF, axis:)` are not
|
|
202
|
+
# implemented; use them without `axis:`.
|
|
203
|
+
# @param axis [Integer]
|
|
204
|
+
# @return [CArray]
|
|
205
|
+
# @!method count_masked(axis: nil)
|
|
206
|
+
# @overload count_masked
|
|
207
|
+
# The per-category form: {CASegmentIterator#count_masked}, one value per category.
|
|
208
|
+
# @overload count_masked(axis:)
|
|
209
|
+
# Not implemented; call it without `axis:`.
|
|
210
|
+
# @param axis [Integer]
|
|
211
|
+
# @return [CArray]
|
|
212
|
+
# @!method count_not_masked(axis: nil)
|
|
213
|
+
# @overload count_not_masked
|
|
214
|
+
# The per-category form: {CASegmentIterator#count_not_masked}, one value per category.
|
|
215
|
+
# @overload count_not_masked(axis:)
|
|
216
|
+
# Per-fiber per-category count of present (non-masked) values along `axis`
|
|
217
|
+
# (int64, shape [K, ...band]). Empty cells are `0`.
|
|
218
|
+
# @param axis [Integer]
|
|
219
|
+
# @return [CArray]
|
|
220
|
+
# @!method max(axis: nil)
|
|
221
|
+
# @overload max
|
|
222
|
+
# The per-category form: {CASegmentIterator#max}, one value per category.
|
|
223
|
+
# @overload max(axis:)
|
|
224
|
+
# Per-fiber per-category maxima along `axis` (h's data type, masked where empty).
|
|
225
|
+
# @param axis [Integer]
|
|
226
|
+
# @return [CArray]
|
|
227
|
+
# @!method mean(axis: nil)
|
|
228
|
+
# @overload mean
|
|
229
|
+
# The per-category form: {CASegmentIterator#mean}, one value per category.
|
|
230
|
+
# @overload mean(axis:)
|
|
231
|
+
# Per-fiber per-category means (float64, empty group cells MASKED).
|
|
232
|
+
# @param axis [Integer]
|
|
233
|
+
# @return [CArray]
|
|
234
|
+
# @!method min(axis: nil)
|
|
235
|
+
# @overload min
|
|
236
|
+
# The per-category form: {CASegmentIterator#min}, one value per category.
|
|
237
|
+
# @overload min(axis:)
|
|
238
|
+
# Per-fiber per-category minima along `axis` (h's data type, masked where empty).
|
|
239
|
+
# @param axis [Integer]
|
|
240
|
+
# @return [CArray]
|
|
241
|
+
# @!method minmax(axis: nil)
|
|
242
|
+
# @overload minmax
|
|
243
|
+
# The per-category form: {CASegmentIterator#minmax}, one value per category.
|
|
244
|
+
# @overload minmax(axis:)
|
|
245
|
+
# Per-fiber `[min_ca, max_ca]` along `axis` (each shape [K, ...band], h's data type,
|
|
246
|
+
# empty group cells MASKED). Ruby Array of two CArrays, not stacked.
|
|
247
|
+
# Both come from one kernel run.
|
|
248
|
+
# @param axis [Integer]
|
|
249
|
+
# @return [Array<CArray>]
|
|
250
|
+
# @!method prod(axis: nil)
|
|
251
|
+
# @overload prod
|
|
252
|
+
# The per-category form: {CASegmentIterator#prod}, one value per category.
|
|
253
|
+
# @overload prod(axis:)
|
|
254
|
+
# Per-fiber per-category products (float64, shape [K, ...band]). Empty
|
|
255
|
+
# group cells `1.0` (identity).
|
|
256
|
+
# @param axis [Integer]
|
|
257
|
+
# @return [CArray]
|
|
258
|
+
# @!method stddevp(axis: nil)
|
|
259
|
+
# @overload stddevp
|
|
260
|
+
# The per-category form: {CASegmentIterator#stddevp}, one value per category.
|
|
261
|
+
# @overload stddevp(axis:)
|
|
262
|
+
# Per-fiber per-category population stddev (float64, empty group cells MASKED).
|
|
263
|
+
# @param axis [Integer]
|
|
264
|
+
# @return [CArray]
|
|
265
|
+
# @!method sum(axis: nil)
|
|
266
|
+
# @overload sum
|
|
267
|
+
# The per-category form: {CASegmentIterator#sum}, one value per category.
|
|
268
|
+
# @overload sum(axis:)
|
|
269
|
+
# Returns per-category sums per fiber along `axis`. Cat may be 1-D (case
|
|
270
|
+
# A, broadcasts across band axes), same rank as source (case B, per-fiber
|
|
271
|
+
# independent classifier), or one rank less (band-only, constant along
|
|
272
|
+
# reduce axis). Output shape = `[K, ...source.shape without axis]`.
|
|
273
|
+
# @param axis [Integer] reduce axis of the source value.
|
|
274
|
+
# @return [CArray]
|
|
275
|
+
# @!method wmean(weights, axis: nil)
|
|
276
|
+
# @overload wmean
|
|
277
|
+
# The per-category form: {CASegmentIterator#wmean}, one value per category.
|
|
278
|
+
# @overload wmean(weights, axis:)
|
|
279
|
+
# Per-fiber per-category weighted mean along `axis`. Same weights-shape
|
|
280
|
+
# contract as {#wsum} (weights.shape == source.shape). Empty cell → MASKED;
|
|
281
|
+
# a present cell whose weights sum to zero → NaN (0/0 core contract).
|
|
282
|
+
# @param weights [CArray]
|
|
283
|
+
# @param axis [Integer]
|
|
284
|
+
# @return [CArray]
|
|
285
|
+
# @!method wsum(weights, axis: nil)
|
|
286
|
+
# @overload wsum
|
|
287
|
+
# The per-category form: {CASegmentIterator#wsum}, one value per category.
|
|
288
|
+
# @overload wsum(weights, axis:)
|
|
289
|
+
# Per-fiber per-category weighted sum along `axis`. `weights` must have
|
|
290
|
+
# shape == source.shape (rev3 requires explicit broadcast; wrap 1-D or
|
|
291
|
+
# band-shape weights via `.broadcast_to(*source.shape)` at the call site).
|
|
292
|
+
# Empty group cell → `0.0` (identity). Mask contract: cell contributes iff
|
|
293
|
+
# value AND weight are present.
|
|
294
|
+
# @param weights [CArray]
|
|
295
|
+
# @param axis [Integer]
|
|
296
|
+
# @return [CArray]
|
|
563
297
|
|
|
564
|
-
|
|
565
|
-
# Per-category weighted sum as float64, matching `CArray#wsum`. `weights` is
|
|
566
|
-
# a per-cell weight CArray in the source order (same elements as the value).
|
|
567
|
-
# Empty / all-masked category -> 0.0 (the additive identity). A cell is
|
|
568
|
-
# skipped iff its value OR its weight is masked (core's contract).
|
|
569
|
-
# @param weights [CArray]
|
|
570
|
-
# @return [CArray]
|
|
571
|
-
# @overload wsum(weights, axis:)
|
|
572
|
-
# Per-fiber per-category weighted sum along `axis`. `weights` must have
|
|
573
|
-
# shape == source.shape (rev3 requires explicit broadcast; wrap 1-D or
|
|
574
|
-
# band-shape weights via `.broadcast_to(*source.shape)` at the call site).
|
|
575
|
-
# Empty group cell → `0.0` (identity). Mask contract: cell contributes iff
|
|
576
|
-
# value AND weight are present.
|
|
577
|
-
# @param weights [CArray]
|
|
578
|
-
# @param axis [Integer]
|
|
579
|
-
# @return [CArray]
|
|
580
|
-
def wsum (weights, axis: nil)
|
|
581
|
-
return axis_wsum_wmean(weights, axis)[0] if axis
|
|
582
|
-
wg = scatter_weights(weights)
|
|
583
|
-
return kernel_weighted(wg)[0] if MONOID_TYPES.include?(@grouped.data_type)
|
|
584
|
-
fold_weighted(wg, 0.0) { |v, ws| v.wsum(ws) }
|
|
585
|
-
end
|
|
298
|
+
private
|
|
586
299
|
|
|
587
|
-
#
|
|
588
|
-
#
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
# @param weights [CArray]
|
|
592
|
-
# @return [CArray]
|
|
593
|
-
# @overload wmean(weights, axis:)
|
|
594
|
-
# Per-fiber per-category weighted mean along `axis`. Same weights-shape
|
|
595
|
-
# contract as {#wsum} (weights.shape == source.shape). Empty cell → MASKED;
|
|
596
|
-
# a present cell whose weights sum to zero → NaN (0/0 core contract).
|
|
597
|
-
# @param weights [CArray]
|
|
598
|
-
# @param axis [Integer]
|
|
599
|
-
# @return [CArray]
|
|
600
|
-
def wmean (weights, axis: nil)
|
|
601
|
-
return axis_wsum_wmean(weights, axis)[1] if axis
|
|
602
|
-
wg = scatter_weights(weights)
|
|
603
|
-
return kernel_weighted(wg)[1] if MONOID_TYPES.include?(@grouped.data_type)
|
|
604
|
-
fold_weighted(wg, UNDEF) { |v, ws| v.wmean(ws) }
|
|
300
|
+
# Both are built with the grouping plan, and only on the flat path; the
|
|
301
|
+
# per-fiber form has neither and must not build a segment's.
|
|
302
|
+
def perm
|
|
303
|
+
@perm
|
|
605
304
|
end
|
|
606
305
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
# named surface), mirroring `CArray#reduce_slab`. The block receives each
|
|
610
|
-
# category's members (a CArray) and returns one value per category.
|
|
611
|
-
# @yieldparam members [CArray]
|
|
612
|
-
# @return [CArray] length-k, aligned to {#labels}
|
|
613
|
-
# @overload reduce(init) { |acc, elem| ... }
|
|
614
|
-
# Per-category fiber fold: each category's members are folded element by
|
|
615
|
-
# element starting from `init`.
|
|
616
|
-
# @param init [Object] initial accumulator.
|
|
617
|
-
# @return [CArray] length-k
|
|
618
|
-
def reduce (*args, data_type: nil, &blk)
|
|
619
|
-
raise LocalJumpError, "no block given (yield)" unless blk
|
|
620
|
-
dt = data_type || CA_OBJECT
|
|
621
|
-
if args.empty?
|
|
622
|
-
per_category(dt) { |s| blk.call(s) }
|
|
623
|
-
else
|
|
624
|
-
init = args[0]
|
|
625
|
-
per_category(dt) { |s|
|
|
626
|
-
acc = init
|
|
627
|
-
s.each { |e| acc = blk.call(acc, e) }
|
|
628
|
-
acc
|
|
629
|
-
}
|
|
630
|
-
end
|
|
306
|
+
def codes
|
|
307
|
+
@codes
|
|
631
308
|
end
|
|
632
309
|
|
|
633
|
-
#
|
|
634
|
-
#
|
|
635
|
-
#
|
|
636
|
-
#
|
|
637
|
-
#
|
|
638
|
-
#
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
310
|
+
# Axis-aware count of present cells. Counting how many cells fall in a
|
|
311
|
+
# group does not look at what is in them. An integer or float payload goes
|
|
312
|
+
# through the fused moments kernel, which counts in the same pass that
|
|
313
|
+
# sums. Any other payload -- complex, boolean, object, a Face -- the kernel
|
|
314
|
+
# refuses, for an answer that never depended on the payload, so it is
|
|
315
|
+
# counted from the codes and the value's mask, one group at a time.
|
|
316
|
+
def axis_counts (axis, op)
|
|
317
|
+
h = @value
|
|
318
|
+
return axis_moments(axis, op)[:count] if h.integer? || h.float?
|
|
319
|
+
axis = checked_axis(axis, op)
|
|
320
|
+
full_c = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
321
|
+
band = h.shape.dup; band.delete_at(axis)
|
|
322
|
+
out = CArray.int64(*([@k] + band))
|
|
323
|
+
present = h.has_mask? ? h.is_not_masked : nil
|
|
324
|
+
slot = [nil] * (band.size + 1)
|
|
648
325
|
@k.times do |c|
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
326
|
+
# a masked code belongs to no group, and eq yields UNDEF there
|
|
327
|
+
belongs = full_c.eq(c)
|
|
328
|
+
belongs = belongs.strip_mask(false) if belongs.has_mask?
|
|
329
|
+
belongs = belongs & present if present
|
|
330
|
+
slot[0] = c
|
|
331
|
+
# a boolean sums to a count without widening the whole array first
|
|
332
|
+
out[*slot] = belongs.sum(axis: axis)
|
|
652
333
|
end
|
|
653
|
-
# Scatter back to source positions via the permutation (grouped-order source
|
|
654
|
-
# indices). Excluded cells are absent from perm and stay UNDEF.
|
|
655
|
-
out = CArray.new(dt, @src_shape)
|
|
656
|
-
out[] = UNDEF
|
|
657
|
-
out.reshape(@codes.elements)[perm] = transformed
|
|
658
334
|
out
|
|
659
335
|
end
|
|
660
336
|
|
|
661
|
-
#
|
|
662
|
-
#
|
|
663
|
-
# The per-element-emit siblings of the reductions: unlike a reduction (which
|
|
664
|
-
# collapses each category to one value) a scan preserves the source shape,
|
|
665
|
-
# each cell holding its category's running statistic up to and including that
|
|
666
|
-
# cell, in source (row-major) order. A category is a partition (each cell is
|
|
667
|
-
# in exactly one category), so the running value is single-valued. The flat
|
|
668
|
-
# categorical grouping is the one-band case of the axis-group scan, so each
|
|
669
|
-
# routes straight through the fused C kernel __axis_group_scan__ (the same one
|
|
670
|
-
# CAGroupIterator drives) with the whole source as a single grouped axis and
|
|
671
|
-
# the categorical's codes as the single bundle -- which yields SOURCE-ORDER
|
|
672
|
-
# output directly, so no counting-sort inverse permutation is needed.
|
|
673
|
-
# Excluded (out-of-vocabulary / masked-code) and source-masked cells join no
|
|
674
|
-
# running total and are UNDEF. Mirroring the reductions (sum / mean), a scan
|
|
675
|
-
# takes no axis argument. cumsum / cumprod -> float64, cummax / cummin
|
|
676
|
-
# preserve the value data type, cumcount -> int64 (1-based within-category
|
|
677
|
-
# ordinal); an object value data type is carried by the kernel's object branch.
|
|
678
|
-
|
|
679
|
-
# @overload cumsum
|
|
680
|
-
# Per-category inclusive running sum (float64), source-shaped.
|
|
681
|
-
# @return [CArray]
|
|
682
|
-
# @overload cumprod
|
|
683
|
-
# Per-category inclusive running product (float64), source-shaped.
|
|
684
|
-
# @return [CArray]
|
|
685
|
-
# @overload cummax
|
|
686
|
-
# Per-category inclusive running maximum (value data type), source-shaped.
|
|
687
|
-
# @return [CArray]
|
|
688
|
-
# @overload cummin
|
|
689
|
-
# Per-category inclusive running minimum (value data type), source-shaped.
|
|
690
|
-
# @return [CArray]
|
|
691
|
-
# @overload cumcount
|
|
692
|
-
# Per-category 1-based within-category ordinal (int64), source-shaped.
|
|
693
|
-
# @return [CArray]
|
|
694
|
-
[:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
|
|
695
|
-
define_method(op) { scan(op) }
|
|
696
|
-
end
|
|
697
|
-
|
|
698
|
-
private
|
|
699
|
-
|
|
700
|
-
# Axis-aware moments (count / sum / min / max) — computed once per axis via
|
|
701
|
-
# the fused per-fiber scatter-reduce C kernel and cached (matches the flat
|
|
702
|
-
# #moments caching in spirit: pay one kernel per {iterator, axis} pair, share
|
|
703
|
-
# across sum / mean / min / max / minmax / count* consumers). Returns
|
|
337
|
+
# Axis-aware moments (count / sum / min / max) via the fused per-fiber
|
|
338
|
+
# scatter-reduce C kernel. Returns
|
|
704
339
|
# `{count: <int64>, sum: <float64>, min: <h's type, masked>, max: <h's type, masked>}`,
|
|
705
340
|
# all shape [K, ...band].
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
341
|
+
#
|
|
342
|
+
# Read fresh on every call, deliberately. It used to be kept per axis, and
|
|
343
|
+
# since the rest of the axis: family (prod, the variance family, wsum /
|
|
344
|
+
# wmean) reads the source when asked, half of the family answered about the
|
|
345
|
+
# values as they were and half about the values as they are. Writing
|
|
346
|
+
# through another view of the source between two calls got you a mean of 2.0
|
|
347
|
+
# beside a variance of 4704.5 for the same cell, which is not a pair any
|
|
348
|
+
# data can produce. The axis: path materialises nothing else, so holding
|
|
349
|
+
# this one thing was the odd choice; a caller who wants a fused kernel's
|
|
350
|
+
# four answers shares them by keeping the result.
|
|
351
|
+
def axis_moments (axis, op)
|
|
352
|
+
return axis_moments_by_core(axis, op) unless MONOID_TYPES.include?(@value.data_type)
|
|
710
353
|
h = @value
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
"Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
715
|
-
end
|
|
354
|
+
axis = checked_axis(axis, op)
|
|
355
|
+
band = h.shape.dup; band.delete_at(axis)
|
|
356
|
+
return band_only_moments(h, axis, band, op) if band_only?(@cat.codes, h.shape, axis)
|
|
716
357
|
codes_h_shape = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
717
|
-
band = h.shape.dup; band.delete_at(axis)
|
|
718
358
|
out_shape = [@k] + band
|
|
719
359
|
counts = CArray.int64(*out_shape)
|
|
720
360
|
sums = CArray.float64(*out_shape)
|
|
721
|
-
mins = CArray.new(h.data_type, out_shape)
|
|
722
|
-
maxs = CArray.new(h.data_type, out_shape)
|
|
361
|
+
mins = CArray.new(h.data_type, out_shape, bytes: h.bytes)
|
|
362
|
+
maxs = CArray.new(h.data_type, out_shape, bytes: h.bytes)
|
|
723
363
|
h.__send__(:__fiber_scatter_moments__, codes_h_shape, axis, @k,
|
|
724
364
|
counts, sums, mins, maxs)
|
|
725
|
-
|
|
365
|
+
{count: counts, sum: sums, min: mins, max: maxs}
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
# The classifier is band-only, as resolve_axis_codes decides it (case A
|
|
369
|
+
# wins when both shapes fit).
|
|
370
|
+
def band_only? (codes, h_shape, axis)
|
|
371
|
+
band = h_shape.dup; band.delete_at(axis)
|
|
372
|
+
!band.empty? && codes.shape == band && codes.shape != [h_shape[axis]]
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# Moments for a band-only classifier. Every cell of a fiber has the same
|
|
376
|
+
# code, so a fiber falls into one group whole: the core's own reductions
|
|
377
|
+
# along the axis answer for it, and each answer is placed in its code's
|
|
378
|
+
# row. A fiber whose code is masked belongs to no group. A group no fiber
|
|
379
|
+
# falls in, or one whose fibers are all masked, has count 0, sum 0.0 and
|
|
380
|
+
# masked min and max, as the fused kernel leaves it. Only the answers the
|
|
381
|
+
# caller reads are computed; each is a pass of its own here, where the
|
|
382
|
+
# kernel had all four from one.
|
|
383
|
+
BAND_ONLY_KEYS = { sum: [:sum], mean: [:sum, :count], min: [:min],
|
|
384
|
+
max: [:max], minmax: [:min, :max] }.freeze
|
|
385
|
+
private_constant :BAND_ONLY_KEYS
|
|
386
|
+
|
|
387
|
+
def band_only_moments (h, axis, band, op)
|
|
388
|
+
nb = band.inject(1, :*)
|
|
389
|
+
shape = [@k] + band
|
|
390
|
+
codes = @cat.codes
|
|
391
|
+
valid = codes.lt(@k).flatten
|
|
392
|
+
valid = valid.strip_mask(false) if valid.has_mask?
|
|
393
|
+
addr = (codes.value.int64.flatten * nb + CArray.int64(nb).seq)[valid]
|
|
394
|
+
out = {}
|
|
395
|
+
BAND_ONLY_KEYS.fetch(op, [:count]).each do |key|
|
|
396
|
+
res = case key
|
|
397
|
+
when :count then CArray.int64(*shape)
|
|
398
|
+
when :sum then CArray.float64(*shape)
|
|
399
|
+
else CArray.new(h.data_type, shape, bytes: h.bytes).tap { |a| a[] = UNDEF }
|
|
400
|
+
end
|
|
401
|
+
per_fiber = case key
|
|
402
|
+
when :count then h.count_not_masked(axis: axis)
|
|
403
|
+
when :sum then h.sum(axis: axis)
|
|
404
|
+
when :min then h.min(axis: axis)
|
|
405
|
+
when :max then h.max(axis: axis)
|
|
406
|
+
end
|
|
407
|
+
res.reshape(@k * nb)[addr] = per_fiber.flatten[valid]
|
|
408
|
+
out[key] = res
|
|
409
|
+
end
|
|
410
|
+
out
|
|
726
411
|
end
|
|
727
412
|
|
|
728
413
|
# Axis-aware sum: the moments sum is already the core fold in the core's own
|
|
729
414
|
# type, so it is handed back as is (an empty group cell carries identity 0.0).
|
|
730
415
|
def axis_sum (axis)
|
|
731
|
-
axis_moments(axis)[:sum]
|
|
416
|
+
axis_moments(axis, :sum)[:sum]
|
|
732
417
|
end
|
|
733
418
|
|
|
734
419
|
# Axis-aware mean: sums / counts (float64); empty group cells (count=0) MASKED.
|
|
735
420
|
# Matches flat #mean per fiber.
|
|
736
421
|
def axis_mean (axis)
|
|
737
|
-
|
|
422
|
+
return axis_by_core(axis, :mean) unless MONOID_TYPES.include?(@value.data_type)
|
|
423
|
+
m = axis_moments(axis, :mean)
|
|
738
424
|
cnt = m[:count]
|
|
739
425
|
out = m[:sum] / cnt.float64 # count 0 -> NaN
|
|
740
426
|
out[cnt.eq(0)] = UNDEF # empty / all-masked -> MASKED
|
|
@@ -753,13 +439,9 @@ class CACategoricalIterator < CAIterator
|
|
|
753
439
|
# Cost: K axis-reductions over an h-shaped local (most cells masked away for
|
|
754
440
|
# each c) — bounded by K, typically small. A fused per-fiber variance
|
|
755
441
|
# kernel is a natural follow-on if bench demands it.
|
|
756
|
-
def axis_by_masked_copy (axis, op, out_data_type =
|
|
442
|
+
def axis_by_masked_copy (axis, op, out_data_type = core_reduce_type(op))
|
|
757
443
|
h = @value
|
|
758
|
-
|
|
759
|
-
raise ArgumentError,
|
|
760
|
-
"group_by_category.#{op}(axis: #{axis.inspect}): axis must be an " \
|
|
761
|
-
"Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
762
|
-
end
|
|
444
|
+
axis = checked_axis(axis, op)
|
|
763
445
|
full_c = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
764
446
|
band = h.shape.dup; band.delete_at(axis)
|
|
765
447
|
out = CArray.new(out_data_type, [@k] + band)
|
|
@@ -774,27 +456,128 @@ class CACategoricalIterator < CAIterator
|
|
|
774
456
|
exclude = in_c.not
|
|
775
457
|
exclude = exclude | codes_bad if codes_bad
|
|
776
458
|
h_local[exclude] = UNDEF
|
|
777
|
-
slice = h_local.__send__(op, axis: axis) #
|
|
459
|
+
slice = h_local.__send__(op, axis: axis) # band shape, mask carries n<contract
|
|
778
460
|
slot_idx[0] = c
|
|
779
461
|
out[*slot_idx] = slice
|
|
780
462
|
end
|
|
781
463
|
out
|
|
782
464
|
end
|
|
783
465
|
|
|
466
|
+
# The axis: reductions for a value the fused kernels cannot read (boolean,
|
|
467
|
+
# object, a Face): each group's cells are reduced by the core's own
|
|
468
|
+
# reduction along the axis and the answers stacked, so the data type, the
|
|
469
|
+
# refusals and the empty-group answer are the core's -- a time column's mean
|
|
470
|
+
# is a time, its sum is refused, an empty group's sum is the identity.
|
|
471
|
+
# Classified along the axis (case A), a group is its positions on the axis,
|
|
472
|
+
# taken by index; otherwise its cells are kept by masking the rest of a copy.
|
|
473
|
+
# A read-only Face's answers (a const string's records index a pool of
|
|
474
|
+
# their own) cannot be put together, so they are collected as the surface
|
|
475
|
+
# values they are, as the flat form does.
|
|
476
|
+
def axis_by_core (axis, op, *args)
|
|
477
|
+
h = @value
|
|
478
|
+
axis = checked_axis(axis, op)
|
|
479
|
+
groups = (0...[@k, 1].max).map { |c| group_cells(h, axis, c, @k.zero?) }
|
|
480
|
+
answers = groups.map { |g| g.__send__(op, *args, axis: axis) }
|
|
481
|
+
if answers.first.face? && answers.first.read_only?
|
|
482
|
+
answers = answers.map { |a| CArray.object(*a.shape) { a.to_a } }
|
|
483
|
+
end
|
|
484
|
+
out = CArray.stack(answers, axis: 0).copy
|
|
485
|
+
@k.zero? ? out[0...0, *([nil] * (out.ndim - 1))].copy : out
|
|
486
|
+
end
|
|
487
|
+
|
|
488
|
+
def axis_moments_by_core (axis, op)
|
|
489
|
+
BAND_ONLY_KEYS.fetch(op, [:count]).to_h do |key|
|
|
490
|
+
[key, axis_by_core(axis, key == :count ? :count_not_masked : key)]
|
|
491
|
+
end
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
def group_cells (h, axis, c, none)
|
|
495
|
+
codes = @cat.codes
|
|
496
|
+
if codes.shape == [h.shape[axis]]
|
|
497
|
+
sel = none ? CArray.boolean(codes.elements) { false } : codes.eq(c)
|
|
498
|
+
sel = sel.strip_mask(false) if sel.has_mask?
|
|
499
|
+
idx = [nil] * h.ndim
|
|
500
|
+
idx[axis] = sel.where
|
|
501
|
+
h[*idx]
|
|
502
|
+
else
|
|
503
|
+
full_c = resolve_axis_codes(codes, h.shape, axis)
|
|
504
|
+
exclude = none ? CArray.boolean(*h.shape) { true } : full_c.eq(c).not
|
|
505
|
+
exclude = exclude | full_c.is_masked if full_c.has_mask?
|
|
506
|
+
local = h.copy
|
|
507
|
+
local[exclude] = UNDEF
|
|
508
|
+
local
|
|
509
|
+
end
|
|
510
|
+
end
|
|
511
|
+
private :axis_by_core, :axis_moments_by_core, :group_cells
|
|
512
|
+
|
|
513
|
+
# A running statistic along an axis, for a categorical that classifies the
|
|
514
|
+
# positions along it (case A): each cell holds its group's running value up
|
|
515
|
+
# to it along the axis, per fiber. The answer is the core's, as the flat
|
|
516
|
+
# form's is; the kernel runs when it answers in the core's data type, and
|
|
517
|
+
# otherwise each group's positions are scanned by the core along the axis.
|
|
518
|
+
# Read when called, as the other axis: members are.
|
|
519
|
+
def axis_scan (op, axis)
|
|
520
|
+
h = @value
|
|
521
|
+
axis = checked_axis(axis, op)
|
|
522
|
+
classified_along!(op, axis)
|
|
523
|
+
answer = core_scan_answer(op)
|
|
524
|
+
if kernel_scans?(op, answer)
|
|
525
|
+
bundle = @k > 0 ? [@cat.codes, @k, [axis]]
|
|
526
|
+
: [CArray.int64(h.shape[axis]).fill(-1), 1, [axis]]
|
|
527
|
+
return h.__axis_group_scan__([axis], [bundle], op)
|
|
528
|
+
end
|
|
529
|
+
each_group_along(axis, ->(shape) { blank_like(answer, shape) }) do |g|
|
|
530
|
+
g.public_send(op, axis: axis)
|
|
531
|
+
end
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
# #map along an axis (case A): the block receives each group's positions
|
|
535
|
+
# along the axis, every other axis kept, and its answer goes back to them.
|
|
536
|
+
def axis_map (axis, data_type, &block)
|
|
537
|
+
axis = checked_axis(axis, :map)
|
|
538
|
+
classified_along!(:map, axis)
|
|
539
|
+
each_group_along(axis, ->(shape) { blank_output(shape, data_type) }, &block)
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
# Yields each group's positions along +axis+ as a view of the value and
|
|
543
|
+
# writes the block's answer back to them; a position in no group is UNDEF.
|
|
544
|
+
def each_group_along (axis, blank)
|
|
545
|
+
h = @value
|
|
546
|
+
out = blank.call(h.shape)
|
|
547
|
+
out[] = UNDEF
|
|
548
|
+
@k.times do |c|
|
|
549
|
+
sel = @cat.codes.eq(c)
|
|
550
|
+
sel = sel.strip_mask(false) if sel.has_mask?
|
|
551
|
+
next if sel.count(true).zero?
|
|
552
|
+
idx = [nil] * h.ndim
|
|
553
|
+
idx[axis] = sel.where
|
|
554
|
+
out[*idx] = surface_for(out, yield(h[*idx]))
|
|
555
|
+
end
|
|
556
|
+
out
|
|
557
|
+
end
|
|
558
|
+
|
|
559
|
+
# A running value or a transform along an axis follows the positions along
|
|
560
|
+
# it, so the categorical has to classify those positions.
|
|
561
|
+
def classified_along! (op, axis)
|
|
562
|
+
n = @value.shape[axis]
|
|
563
|
+
return if @cat.codes.shape == [n]
|
|
564
|
+
raise ArgumentError,
|
|
565
|
+
"#{op}(axis: #{axis}): needs a categorical that classifies the " \
|
|
566
|
+
"positions along axis #{axis} (shape [#{n}]); got shape " \
|
|
567
|
+
"#{@cat.codes.shape.inspect}"
|
|
568
|
+
end
|
|
569
|
+
private :axis_scan, :axis_map, :each_group_along, :classified_along!
|
|
570
|
+
|
|
784
571
|
# Axis-aware wsum + wmean fused (single kernel call, both outputs). Returns
|
|
785
572
|
# [wsum_ca, wmean_ca]. Weights must match source shape exactly (explicit
|
|
786
573
|
# broadcast on the call site for 1-D or band-shape weights). A cell
|
|
787
574
|
# contributes iff its value AND its weight are present.
|
|
788
|
-
def axis_wsum_wmean (weights, axis)
|
|
575
|
+
def axis_wsum_wmean (weights, axis, op)
|
|
789
576
|
h = @value
|
|
790
|
-
|
|
791
|
-
raise ArgumentError,
|
|
792
|
-
"group_by_category.wsum/wmean(axis: #{axis.inspect}): axis must " \
|
|
793
|
-
"be an Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
794
|
-
end
|
|
577
|
+
axis = checked_axis(axis, op)
|
|
795
578
|
unless weights.is_a?(CArray) && weights.shape == h.shape
|
|
796
579
|
raise ArgumentError,
|
|
797
|
-
"
|
|
580
|
+
"#{op}(axis: #{axis}): weights.shape " \
|
|
798
581
|
"#{weights.respond_to?(:shape) ? weights.shape.inspect : weights.class} " \
|
|
799
582
|
"must equal source.shape #{h.shape.inspect}. Wrap 1-D / band-shape " \
|
|
800
583
|
"weights via `.broadcast_to(*source.shape)` before passing."
|
|
@@ -812,12 +595,9 @@ class CACategoricalIterator < CAIterator
|
|
|
812
595
|
# Axis-aware prod: dedicated kernel (identity 1.0, separate from moments to
|
|
813
596
|
# avoid conflating with sum's zero-identity memset).
|
|
814
597
|
def axis_prod (axis)
|
|
598
|
+
return axis_by_core(axis, :prod) unless MONOID_TYPES.include?(@value.data_type)
|
|
815
599
|
h = @value
|
|
816
|
-
|
|
817
|
-
raise ArgumentError,
|
|
818
|
-
"group_by_category.prod(axis: #{axis.inspect}): axis must be an " \
|
|
819
|
-
"Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
820
|
-
end
|
|
600
|
+
axis = checked_axis(axis, :prod)
|
|
821
601
|
codes_h_shape = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
822
602
|
band = h.shape.dup; band.delete_at(axis)
|
|
823
603
|
out = CArray.float64(*([@k] + band))
|
|
@@ -828,25 +608,29 @@ class CACategoricalIterator < CAIterator
|
|
|
828
608
|
# Broadcast `codes` to `h_shape` per PROPOSAL §2.2 3-case positional rule.
|
|
829
609
|
# Returns a broadcast view of codes at h_shape (or codes itself for case B).
|
|
830
610
|
# Raises ArgumentError with a message that enumerates all 3 accepted shapes.
|
|
611
|
+
# The axis of an axis: reduction, normalised against the source: a
|
|
612
|
+
# negative axis counts from the end, as everywhere else.
|
|
613
|
+
def checked_axis (axis, op)
|
|
614
|
+
@value.normalize_axis(axis, op.to_s)
|
|
615
|
+
end
|
|
616
|
+
|
|
831
617
|
def resolve_axis_codes (codes, h_shape, axis)
|
|
832
618
|
ndim = h_shape.size
|
|
833
619
|
band = h_shape.dup; band.delete_at(axis)
|
|
834
|
-
case
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
620
|
+
# Chosen by shape, not by rank. For a 2-D source the case A shape and the
|
|
621
|
+
# band-only shape are both rank 1, so choosing by rank took case A every
|
|
622
|
+
# time and band-only could never be reached there -- while the refusal
|
|
623
|
+
# went on to list the very shape it was refusing among the ones it
|
|
624
|
+
# accepts. When both fit, which a square source makes possible, case A
|
|
625
|
+
# wins: classifying along the reduce axis is the reading that holds at
|
|
626
|
+
# every rank.
|
|
627
|
+
case
|
|
628
|
+
when codes.shape == [h_shape[axis]] # case A
|
|
839
629
|
view_shape = Array.new(ndim, 1); view_shape[axis] = h_shape[axis]
|
|
840
630
|
codes.reshape(*view_shape).broadcast_to(*h_shape)
|
|
841
|
-
when
|
|
842
|
-
unless codes.shape == h_shape
|
|
843
|
-
axis_shape_mismatch!(codes.shape, h_shape, axis, band)
|
|
844
|
-
end
|
|
631
|
+
when codes.shape == h_shape # case B
|
|
845
632
|
codes
|
|
846
|
-
when
|
|
847
|
-
unless codes.shape == band
|
|
848
|
-
axis_shape_mismatch!(codes.shape, h_shape, axis, band)
|
|
849
|
-
end
|
|
633
|
+
when codes.shape == band # band-only
|
|
850
634
|
view_shape = h_shape.dup; view_shape[axis] = 1
|
|
851
635
|
codes.reshape(*view_shape).broadcast_to(*h_shape)
|
|
852
636
|
else
|
|
@@ -855,175 +639,21 @@ class CACategoricalIterator < CAIterator
|
|
|
855
639
|
end
|
|
856
640
|
|
|
857
641
|
def axis_shape_mismatch! (cat_shape, h_shape, axis, band)
|
|
642
|
+
# No method name: the one place that resolves this serves sum, mean, min,
|
|
643
|
+
# max, count and the rest alike, and naming one of them would be wrong for
|
|
644
|
+
# the others. The backtrace says which was called.
|
|
858
645
|
raise ArgumentError,
|
|
859
|
-
"group_by_category
|
|
646
|
+
"group_by_category (axis: #{axis}): cat.shape=#{cat_shape.inspect} " \
|
|
860
647
|
"does not fit any of the 3 accepted forms for h.shape=#{h_shape.inspect}: " \
|
|
861
648
|
"case A cat.shape=[#{h_shape[axis]}], " \
|
|
862
649
|
"case B cat.shape=#{h_shape.inspect}, " \
|
|
863
650
|
"band-only cat.shape=#{band.inspect}."
|
|
864
651
|
end
|
|
865
652
|
|
|
866
|
-
#
|
|
867
|
-
#
|
|
868
|
-
def
|
|
869
|
-
|
|
870
|
-
"CACategoricalIterator##{op}(axis:) is not implemented — order " \
|
|
871
|
-
"statistics are available without axis:."
|
|
872
|
-
end
|
|
873
|
-
|
|
874
|
-
# Drive a segment scan through the axis-group scan kernel: the whole value as
|
|
875
|
-
# one grouped axis, the flat codes as the single bundle. The kernel emits in
|
|
876
|
-
# source order, so the flat result reshapes straight back to the source shape.
|
|
877
|
-
def scan (op)
|
|
878
|
-
@value.reshape(@value.elements)
|
|
879
|
-
.__axis_group_scan__([0], [[@codes, @k, [0]]], op)
|
|
880
|
-
.reshape(*@src_shape)
|
|
881
|
-
end
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
# Permutation: perm[slot] = the source index whose value sits at that grouped
|
|
885
|
-
# slot. This is the valid prefix of the categorical's cached sort_addr, sliced
|
|
886
|
-
# at construction (the same counting sort that lays out @grouped), so #map /
|
|
887
|
-
# #sort_addr / the *_addr reductions read it for free.
|
|
888
|
-
def perm
|
|
889
|
-
@perm
|
|
890
|
-
end
|
|
891
|
-
|
|
892
|
-
# Lay a per-cell weight array out in category-contiguous order (same layout as
|
|
893
|
-
# @grouped), so wsum / wmean can pair each group's values with its weights.
|
|
894
|
-
# Weights are coerced to float64; the same counting-sort scatter propagates
|
|
895
|
-
# the weight mask and skips excluded cells, so wg lines up with @grouped.
|
|
896
|
-
def scatter_weights (weights)
|
|
897
|
-
unless weights.elements == @codes.elements
|
|
898
|
-
raise ArgumentError,
|
|
899
|
-
"wsum/wmean: weights.elements (#{weights.elements}) != " \
|
|
900
|
-
"value.elements (#{@codes.elements})"
|
|
901
|
-
end
|
|
902
|
-
wf = weights.float64
|
|
903
|
-
wg = CArray.float64(@grouped.elements)
|
|
904
|
-
@codes.send(:__categorical_scatter__, wf.reshape(wf.elements),
|
|
905
|
-
@offsets.copy, wg, @k)
|
|
906
|
-
wg
|
|
907
|
-
end
|
|
908
|
-
|
|
909
|
-
# Map a per-category group-local index to the flat source address via the
|
|
910
|
-
# permutation (grouped slot -> source index). The min/max sits at grouped slot
|
|
911
|
-
# offsets[c] + local_index[c]; perm carries it back to the source. Empty
|
|
912
|
-
# categories (masked local index) stay masked.
|
|
913
|
-
def group_addr (local_index)
|
|
914
|
-
out = CArray.int64(@k)
|
|
915
|
-
@k.times do |c|
|
|
916
|
-
out[c] = local_index.is_masked[c] ? UNDEF
|
|
917
|
-
: perm[@offsets[c] + local_index[c]]
|
|
918
|
-
end
|
|
919
|
-
out
|
|
920
|
-
end
|
|
921
|
-
|
|
922
|
-
# Fused per-segment weighted sum + weighted mean (one C pass over the grouped
|
|
923
|
-
# copy, weights in group order). Returns [wsum, wmean]; wmean is masked where a
|
|
924
|
-
# segment has no present (value AND weight) pair. Numeric value data types only.
|
|
925
|
-
def kernel_weighted (wg)
|
|
926
|
-
ws = CArray.float64(@k)
|
|
927
|
-
wm = CArray.float64(@k)
|
|
928
|
-
@grouped.send(:__reduceat_wsum_wmean__, @offsets, wg, ws, wm)
|
|
929
|
-
[ws, wm]
|
|
930
|
-
end
|
|
931
|
-
|
|
932
|
-
# Per-group weighted fallback for non-numeric value data types (complex): delegate
|
|
933
|
-
# each group to CArray#wsum / #wmean. Empty segments take the given identity.
|
|
934
|
-
def fold_weighted (wg, empty)
|
|
935
|
-
out = CArray.float64(@k)
|
|
936
|
-
@k.times do |c|
|
|
937
|
-
lo = @offsets[c]
|
|
938
|
-
hi = (c + 1 < @k) ? @offsets[c + 1] : @grouped.elements
|
|
939
|
-
out[c] = hi > lo ? yield(@grouped[lo...hi], wg[lo...hi]) : empty
|
|
940
|
-
end
|
|
941
|
-
out
|
|
942
|
-
end
|
|
943
|
-
|
|
944
|
-
# The members of category `c` as a CArray slice of the grouped copy. An empty
|
|
945
|
-
# category (zero-width segment) yields the shared empty array — a zero-length
|
|
946
|
-
# slice cannot be taken directly, and an empty array carries the same reduction
|
|
947
|
-
# contract we want (identity for sum, UNDEF for ratios).
|
|
948
|
-
def group_slice (c)
|
|
949
|
-
lo = @offsets[c]
|
|
950
|
-
hi = (c + 1 < @k) ? @offsets[c + 1] : @grouped.elements
|
|
951
|
-
hi > lo ? @grouped[lo...hi] : @empty
|
|
952
|
-
end
|
|
953
|
-
|
|
954
|
-
# Single-pass reduceat moments (count / sum / min / max per category), computed
|
|
955
|
-
# once over the grouped copy and cached — the whole point of the eager copy is
|
|
956
|
-
# that one scatter is followed by cheap single-pass reductions with no
|
|
957
|
-
# per-segment views. Nil for a non-numeric value data type (complex / object /
|
|
958
|
-
# bool), where the monoid reductions fall back to per_category.
|
|
959
|
-
# numeric value data types the C moments kernel handles (int8..float64); bool /
|
|
960
|
-
# complex / object fall back to per_category.
|
|
961
|
-
MONOID_TYPES = %i[int8 uint8 int16 uint16 int32 uint32
|
|
962
|
-
int64 uint64 float32 float64].freeze
|
|
963
|
-
|
|
964
|
-
def moments
|
|
965
|
-
return @moments if defined?(@moments)
|
|
966
|
-
@moments =
|
|
967
|
-
if MONOID_TYPES.include?(@grouped.data_type)
|
|
968
|
-
dt = @grouped.data_type
|
|
969
|
-
counts = CArray.int64(@k)
|
|
970
|
-
sums = CArray.float64(@k)
|
|
971
|
-
mins = CArray.new(dt, [@k])
|
|
972
|
-
maxs = CArray.new(dt, [@k])
|
|
973
|
-
@grouped.send(:__reduceat_moments__, @offsets, counts, sums, mins, maxs)
|
|
974
|
-
{ count: counts, sum: sums, min: mins, max: maxs }
|
|
975
|
-
end
|
|
976
|
-
end
|
|
977
|
-
|
|
978
|
-
# Single-pass fused group-local argmin / argmax (min_index / max_index),
|
|
979
|
-
# cached. Nil for a non-numeric value data type (fall back to per_category).
|
|
980
|
-
def arg_minmax
|
|
981
|
-
return @arg_minmax if defined?(@arg_minmax)
|
|
982
|
-
@arg_minmax =
|
|
983
|
-
if MONOID_TYPES.include?(@grouped.data_type)
|
|
984
|
-
mn = CArray.int64(@k)
|
|
985
|
-
mx = CArray.int64(@k)
|
|
986
|
-
@grouped.send(:__reduceat_argminmax__, @offsets, mn, mx)
|
|
987
|
-
{ min: mn, max: mx }
|
|
988
|
-
end
|
|
989
|
-
end
|
|
990
|
-
|
|
991
|
-
# Single-pass fused per-category boolean all / any, cached. Nil unless the
|
|
992
|
-
# value data type is boolean (fall back to per_category, which raises like
|
|
993
|
-
# CArray#all on a non-boolean).
|
|
994
|
-
def all_any
|
|
995
|
-
return @all_any if defined?(@all_any)
|
|
996
|
-
@all_any =
|
|
997
|
-
if @grouped.data_type == CA_BOOLEAN
|
|
998
|
-
a = CArray.boolean(@k)
|
|
999
|
-
o = CArray.boolean(@k)
|
|
1000
|
-
@grouped.send(:__reduceat_all_any__, @offsets, a, o)
|
|
1001
|
-
{ all: a, any: o }
|
|
1002
|
-
end
|
|
1003
|
-
end
|
|
1004
|
-
|
|
1005
|
-
# Build a length-k typed output by folding each category's members with the
|
|
1006
|
-
# given reduction block. Fallback path (order statistics, and monoids on a
|
|
1007
|
-
# non-numeric value data type): each group is delegated to the same CArray
|
|
1008
|
-
# reduction, so the per-group result matches `CArray#<reduction>` over that
|
|
1009
|
-
# group's members — the mask carries the "insufficient present data" contract
|
|
1010
|
-
# for free (an all-masked group reduces like an empty one; identity-bearing
|
|
1011
|
-
# reductions return their identity, ratios return UNDEF; see ext ERI).
|
|
1012
|
-
# The data type the core reduction `op` promotes this value to. Asked of the
|
|
1013
|
-
# core itself -- a one-cell reduction of the value's type -- rather than
|
|
1014
|
-
# restated here, so a per-category answer cannot drift from `CArray#<op>`
|
|
1015
|
-
# (`sum` on an integer promotes, `accumulate` stays, `min` / `max` keep the
|
|
1016
|
-
# type but a boolean widens, `prod` on an object stays an object). A payload
|
|
1017
|
-
# the core refuses to fold this way raises here, with the core's own error.
|
|
1018
|
-
def core_reduce_type (op, *args)
|
|
1019
|
-
(@core_reduce_type ||= {})[[op, args]] ||=
|
|
1020
|
-
CArray.new(@grouped.data_type, [1, 1]).public_send(op, *args, axis: 1).data_type
|
|
1021
|
-
end
|
|
1022
|
-
|
|
1023
|
-
def per_category (data_type)
|
|
1024
|
-
out = CArray.new(data_type, [@k])
|
|
1025
|
-
@k.times { |c| out[c] = yield(group_slice(c)) }
|
|
1026
|
-
out
|
|
653
|
+
# median / percentile along an axis: the core's own order statistic of each
|
|
654
|
+
# group's cells, stacked -- the axis_by_core path, one core call per group.
|
|
655
|
+
def axis_order_stat (axis, op, *args)
|
|
656
|
+
axis_by_core(axis, op, *args)
|
|
1027
657
|
end
|
|
1028
658
|
end
|
|
1029
659
|
|
|
@@ -1034,8 +664,14 @@ class CArray
|
|
|
1034
664
|
# per category of `cat`. Requires `self.elements == cat.elements`.
|
|
1035
665
|
# @param cat [CACategorical] the classifier.
|
|
1036
666
|
# @return [CACategoricalIterator]
|
|
667
|
+
# @raise [TypeError] when `cat` is not a CACategorical.
|
|
1037
668
|
# @raise [ArgumentError] when element counts differ.
|
|
1038
669
|
def group_by_category (cat)
|
|
670
|
+
unless cat.is_a?(CACategorical)
|
|
671
|
+
raise TypeError,
|
|
672
|
+
"group_by_category: expects a CACategorical (got #{cat.class}); " \
|
|
673
|
+
"classify the values first with .categorize"
|
|
674
|
+
end
|
|
1039
675
|
CACategoricalIterator.new(self, cat)
|
|
1040
676
|
end
|
|
1041
677
|
|
|
@@ -1060,18 +696,16 @@ class CArray
|
|
|
1060
696
|
# `grp.labels` is `[0, 1, ...]` in position order.
|
|
1061
697
|
#
|
|
1062
698
|
# @return [CACategoricalIterator] one category per run, in order.
|
|
1063
|
-
# @raise [
|
|
699
|
+
# @raise [ArgumentError] when `self` is not 1-D.
|
|
1064
700
|
def group_by_run
|
|
1065
|
-
raise "group_by_run: 1-D only (got #{ndim}-D)" unless ndim == 1
|
|
701
|
+
raise ArgumentError, "group_by_run: 1-D only (got #{ndim}-D)" unless ndim == 1
|
|
1066
702
|
if elements == 0
|
|
1067
703
|
code = CArray.int64(0)
|
|
1068
704
|
else
|
|
1069
705
|
present = is_not_masked
|
|
1070
706
|
edge = present & present.shift(1).not # rising edge = run start
|
|
1071
|
-
#
|
|
1072
|
-
|
|
1073
|
-
# running count never overflows int8.
|
|
1074
|
-
code = edge.refer(:int8).cumsum.int64 - 1 # 0-based run index per cell
|
|
707
|
+
# a boolean cumsum counts in uint64, exactly
|
|
708
|
+
code = edge.cumsum.int64 - 1 # 0-based run index per cell
|
|
1075
709
|
code[present.not] = UNDEF # masked cells join no run
|
|
1076
710
|
end
|
|
1077
711
|
# categorize turns the dense run indices into the run categories: it derives
|