carray 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +5 -25
- data/CHANGELOG.md +16 -0
- data/LICENSE +1 -1
- data/NEWS.md +3 -0
- data/README.md +128 -44
- data/carray.gemspec +22 -24
- data/ext/ca_array_pool.c +91 -0
- data/ext/ca_axis_descriptor.h +186 -0
- data/ext/ca_axis_dispatch.c +924 -0
- data/ext/ca_axis_group.c +1208 -0
- data/ext/ca_bincmp_dispatch.c +76 -0
- data/ext/ca_bincmp_dispatch.h +85 -0
- data/ext/ca_binop_dispatch.c +125 -0
- data/ext/ca_binop_dispatch.h +159 -0
- data/ext/ca_categorical_iterator.c +1375 -0
- data/ext/ca_compare.c +94 -0
- data/ext/ca_compare.h +26 -0
- data/ext/ca_composite_dispatch.c +414 -0
- data/ext/ca_composite_dispatch.h +116 -0
- data/ext/ca_for_buffer.h +96 -0
- data/ext/ca_for_each_element.h +241 -0
- data/ext/ca_group_iter.c +304 -0
- data/ext/ca_iter_substrate.h +325 -0
- data/ext/ca_kernel_iterator.c +4321 -0
- data/ext/ca_kernel_iterator.h +2603 -0
- data/ext/ca_moncmp_dispatch.c +37 -0
- data/ext/ca_moncmp_dispatch.h +62 -0
- data/ext/ca_monop_dispatch.c +200 -0
- data/ext/ca_monop_dispatch.h +235 -0
- data/ext/ca_obj_array.c +355 -359
- data/ext/ca_obj_bincmp.c +809 -0
- data/ext/ca_obj_binop.c +892 -0
- data/ext/ca_obj_bitarray.c +369 -164
- data/ext/ca_obj_bitfield.c +294 -234
- data/ext/ca_obj_block.c +189 -711
- data/ext/ca_obj_byte_swap.c +766 -0
- data/ext/ca_obj_const_string.c +965 -0
- data/ext/ca_obj_face.c +670 -0
- data/ext/ca_obj_face.h +247 -0
- data/ext/ca_obj_fake.c +228 -100
- data/ext/ca_obj_farray.c +54 -441
- data/ext/ca_obj_field.c +82 -529
- data/ext/ca_obj_fixlen_string.c +306 -0
- data/ext/ca_obj_grid.c +858 -440
- data/ext/ca_obj_meld.c +1034 -0
- data/ext/ca_obj_moncmp.c +569 -0
- data/ext/ca_obj_monop.c +1111 -0
- data/ext/ca_obj_object.c +774 -298
- data/ext/ca_obj_record.c +468 -0
- data/ext/ca_obj_reduce.c +97 -82
- data/ext/ca_obj_refer.c +569 -459
- data/ext/ca_obj_remap.c +475 -0
- data/ext/ca_obj_repeat.c +92 -477
- data/ext/ca_obj_roll.c +616 -0
- data/ext/ca_obj_select.c +344 -296
- data/ext/ca_obj_select_axis.c +1296 -0
- data/ext/ca_obj_shift.c +230 -792
- data/ext/ca_obj_source.c +78 -0
- data/ext/ca_obj_stack.c +1173 -0
- data/ext/ca_obj_stride.c +2501 -0
- data/ext/ca_obj_string.c +268 -0
- data/ext/ca_obj_tile.c +614 -0
- data/ext/ca_obj_time.c +546 -0
- data/ext/ca_obj_timedelta.c +435 -0
- data/ext/ca_obj_transpose.c +62 -516
- data/ext/ca_obj_triop.c +746 -0
- data/ext/ca_obj_unbound_repeat.c +208 -241
- data/ext/ca_obj_window.c +1131 -563
- data/ext/ca_op_byte_swap.c +175 -0
- data/ext/ca_op_ipower.c +319 -0
- data/ext/ca_op_powi.h +88 -0
- data/ext/ca_sort_kernels.h +132 -0
- data/ext/ca_sweep_engine.c +430 -0
- data/ext/ca_sweep_engine.h +157 -0
- data/ext/ca_transform_common.c +228 -0
- data/ext/ca_triop_dispatch.c +55 -0
- data/ext/ca_triop_dispatch.h +62 -0
- data/ext/carray.h +795 -402
- data/ext/carray_access.c +831 -711
- data/ext/carray_attribute.c +98 -330
- data/ext/carray_bincount.c +255 -0
- data/ext/carray_broadcast.c +283 -0
- data/ext/carray_call_cfunc.c +1360 -828
- data/ext/carray_call_cfunc.h +160 -0
- data/ext/carray_cast.c +1212 -301
- data/ext/carray_cast_func.rb +81 -40
- data/ext/carray_class.c +53 -63
- data/ext/carray_config.h +28 -0
- data/ext/carray_conversion.c +350 -346
- data/ext/carray_copy.c +156 -268
- data/ext/carray_core.c +1342 -199
- data/ext/carray_count.c +312 -0
- data/ext/carray_data_type.c +43 -19
- data/ext/carray_element.c +585 -213
- data/ext/carray_factorize.c +2542 -0
- data/ext/carray_generate.c +230 -559
- data/ext/carray_histogram.c +490 -0
- data/ext/carray_hold.c +228 -0
- data/ext/carray_index_classifier.c +1035 -0
- data/ext/carray_index_classifier.h +27 -0
- data/ext/carray_internal.h +120 -0
- data/ext/carray_kernels_bincmp.c +4445 -0
- data/ext/carray_kernels_binop.c +10979 -0
- data/ext/carray_kernels_init.c +36 -0
- data/ext/carray_kernels_map.c +3466 -0
- data/ext/carray_kernels_moncmp.c +2096 -0
- data/ext/carray_kernels_monop.c +18312 -0
- data/ext/carray_kernels_reduce_aggregate.c +25836 -0
- data/ext/carray_kernels_reduce_boolean.c +329 -0
- data/ext/carray_kernels_reduce_cumulative.c +14592 -0
- data/ext/carray_kernels_reduce_extreme.c +16947 -0
- data/ext/carray_kernels_reduce_variance.c +3909 -0
- data/ext/carray_kernels_scan.c +3692 -0
- data/ext/carray_kernels_search.c +32137 -0
- data/ext/carray_kernels_sort.c +10625 -0
- data/ext/carray_kernels_triop.c +1391 -0
- data/ext/carray_lazy.c +567 -0
- data/ext/carray_loop.c +88 -200
- data/ext/carray_mask.c +848 -154
- data/ext/carray_math_kernel.h +120 -0
- data/ext/carray_mathfunc.c +10 -241
- data/ext/carray_median_percentile.c +1257 -0
- data/ext/carray_memory_view.c +1625 -0
- data/ext/carray_operator.c +1526 -318
- data/ext/carray_order.c +664 -1394
- data/ext/carray_partition.c +416 -0
- data/ext/carray_random.c +518 -0
- data/ext/carray_scatter.c +357 -0
- data/ext/carray_slab.c +1219 -0
- data/ext/carray_slab.h +84 -0
- data/ext/carray_sort.c +829 -0
- data/ext/carray_sort_kernel.c +620 -0
- data/ext/carray_struct.c +695 -0
- data/ext/carray_test.c +343 -229
- data/ext/carray_undef.c +34 -17
- data/ext/carray_utils.c +175 -74
- data/ext/extconf.rb +216 -55
- data/ext/mk_call_cfunc.rb +480 -0
- data/ext/mkkernel.rb +8842 -0
- data/ext/ruby_carray.c +202 -101
- data/ext/version.h +4 -14
- data/ext/version.rb +5 -13
- data/lib/carray/arrow_tensor.rb +401 -0
- data/lib/carray/attribute.rb +166 -0
- data/lib/carray/autoload_carray.rb +220 -0
- data/lib/carray/autoload_method_extension.rb +44 -0
- data/lib/carray/axis_group.rb +711 -0
- data/lib/carray/basics.rb +481 -0
- data/lib/carray/bincount_nd.rb +358 -0
- data/lib/carray/block_iterator.rb +604 -0
- data/lib/carray/boolean_reduce.rb +109 -0
- data/lib/carray/categorical.rb +561 -0
- data/lib/carray/categorical_iterator.rb +1062 -0
- data/lib/carray/complex.rb +150 -0
- data/lib/carray/conditional.rb +216 -0
- data/lib/carray/const_string.rb +228 -0
- data/lib/carray/construct.rb +139 -328
- data/lib/carray/core_extensions.rb +240 -0
- data/lib/carray/data_type_extension.rb +233 -0
- data/lib/carray/fixlen_string.rb +95 -0
- data/lib/carray/frame/concat.rb +132 -0
- data/lib/carray/frame/convert.rb +95 -0
- data/lib/carray/frame/csv_parser.rb +211 -0
- data/lib/carray/frame/frame.rb +649 -0
- data/lib/carray/frame/group.rb +186 -0
- data/lib/carray/frame/io.rb +164 -0
- data/lib/carray/frame/join.rb +248 -0
- data/lib/carray/frame/records.rb +99 -0
- data/lib/carray/frame/sort.rb +113 -0
- data/lib/carray/frame/verbs.rb +299 -0
- data/lib/carray/frame.rb +16 -0
- data/lib/carray/histogram.rb +512 -0
- data/lib/carray/inspect.rb +37 -20
- data/lib/carray/iterator.rb +57 -349
- data/lib/carray/lazy.rb +889 -0
- data/lib/carray/mask_gap_fill.rb +200 -0
- data/lib/carray/math.rb +78 -342
- data/lib/carray/meld_reduce.rb +289 -0
- data/lib/carray/methods/align_addr.rb +116 -0
- data/lib/carray/methods/bin.rb +128 -0
- data/lib/carray/methods/bincount.rb +87 -0
- data/lib/carray/methods/bit_string.rb +92 -0
- data/lib/carray/methods/broadcast.rb +63 -0
- data/lib/carray/methods/choose.rb +39 -0
- data/lib/carray/methods/composition.rb +280 -0
- data/lib/carray/methods/gather_nd.rb +206 -0
- data/lib/carray/methods/index.rb +39 -0
- data/lib/carray/methods/insert_block.rb +99 -0
- data/lib/carray/methods/is_in.rb +141 -0
- data/lib/carray/methods/join.rb +90 -0
- data/lib/carray/methods/locate_addr.rb +47 -0
- data/lib/carray/methods/mask_duplicates.rb +41 -0
- data/lib/carray/methods/meshgrid.rb +91 -0
- data/lib/carray/methods/mode.rb +126 -0
- data/lib/carray/methods/nunique.rb +46 -0
- data/lib/carray/methods/resize.rb +56 -0
- data/lib/carray/methods/snap.rb +156 -0
- data/lib/carray/methods/string_format.rb +57 -0
- data/lib/carray/methods/unique.rb +47 -0
- data/lib/carray/methods/value_counts.rb +71 -0
- data/lib/carray/mkmf.rb +124 -101
- data/lib/carray/runtime.rb +108 -0
- data/lib/carray/serialize.rb +478 -167
- data/lib/carray/slab_iterator.rb +292 -0
- data/lib/carray/stack.rb +291 -0
- data/lib/carray/string.rb +56 -180
- data/lib/carray/string_operation_extension.rb +289 -0
- data/lib/carray/struct.rb +335 -323
- data/lib/carray/struct_builder.rb +697 -0
- data/lib/carray/table.rb +41 -2
- data/lib/carray/time.rb +2255 -38
- data/lib/carray/window_iterator.rb +655 -0
- data/lib/carray.rb +55 -57
- metadata +163 -130
- data/Rakefile +0 -51
- data/TODO.md +0 -18
- data/ext/ca_iter_block.c +0 -257
- data/ext/ca_iter_dimension.c +0 -299
- data/ext/ca_iter_window.c +0 -214
- data/ext/ca_obj_mapping.c +0 -644
- data/ext/carray_iterator.c +0 -641
- data/ext/carray_math.rb +0 -850
- data/ext/carray_numeric.c +0 -259
- data/ext/carray_sort_addr.c +0 -254
- data/ext/carray_stat.c +0 -2100
- data/ext/carray_stat_proc.rb +0 -1999
- data/ext/mkmath.rb +0 -741
- data/ext/ruby_ccomplex.c +0 -509
- data/ext/ruby_float_func.c +0 -86
- data/lib/carray/array.rb +0 -8
- data/lib/carray/autoload/autoload_base.rb +0 -19
- data/lib/carray/autoload/autoload_gem_cairo.rb +0 -9
- data/lib/carray/autoload/autoload_gem_ffi.rb +0 -9
- data/lib/carray/autoload/autoload_gem_gnuplot.rb +0 -2
- data/lib/carray/autoload/autoload_gem_io_csv.rb +0 -14
- data/lib/carray/autoload/autoload_gem_io_pg.rb +0 -6
- data/lib/carray/autoload/autoload_gem_io_sqlite3.rb +0 -12
- data/lib/carray/autoload/autoload_gem_narray.rb +0 -10
- data/lib/carray/autoload/autoload_gem_numo_narray.rb +0 -15
- data/lib/carray/autoload/autoload_gem_opencv.rb +0 -16
- data/lib/carray/autoload/autoload_gem_random.rb +0 -8
- data/lib/carray/autoload/autoload_gem_rmagick.rb +0 -23
- data/lib/carray/autoload/autoload_gem_zimg.rb +0 -3
- data/lib/carray/autoload/autoload_io_imagemagick.rb +0 -6
- data/lib/carray/autoload/autoload_math_histogram.rb +0 -5
- data/lib/carray/autoload/autoload_math_recurrence.rb +0 -6
- data/lib/carray/autoload/autoload_object_iterator.rb +0 -1
- data/lib/carray/autoload/autoload_object_link.rb +0 -1
- data/lib/carray/autoload/autoload_object_pack.rb +0 -2
- data/lib/carray/autoload.rb +0 -141
- data/lib/carray/basic.rb +0 -191
- data/lib/carray/broadcast.rb +0 -101
- data/lib/carray/compose.rb +0 -315
- data/lib/carray/convert.rb +0 -115
- data/lib/carray/info.rb +0 -110
- data/lib/carray/io/imagemagick.rb +0 -235
- data/lib/carray/mask.rb +0 -102
- data/lib/carray/math/histogram.rb +0 -177
- data/lib/carray/math/recurrence.rb +0 -93
- data/lib/carray/object/ca_obj_iterator.rb +0 -50
- data/lib/carray/object/ca_obj_link.rb +0 -50
- data/lib/carray/object/ca_obj_pack.rb +0 -99
- data/lib/carray/obsolete.rb +0 -256
- data/lib/carray/ordering.rb +0 -181
- data/lib/carray/testing.rb +0 -51
- data/lib/carray/transform.rb +0 -109
- data/misc/Methods.ja.md +0 -182
- data/misc/NOTE +0 -51
- data/spec/Classes/CABitfield_spec.rb +0 -58
- data/spec/Classes/CABlockIterator_spec.rb +0 -114
- data/spec/Classes/CABlock_spec.rb +0 -205
- data/spec/Classes/CAField_spec.rb +0 -39
- data/spec/Classes/CAGrid_spec.rb +0 -75
- data/spec/Classes/CAMap_spec.rb +0 -0
- data/spec/Classes/CAMapping_spec.rb +0 -105
- data/spec/Classes/CAObject_attribute_spec.rb +0 -33
- data/spec/Classes/CAObject_spec.rb +0 -33
- data/spec/Classes/CARefer_spec.rb +0 -93
- data/spec/Classes/CARepeat_spec.rb +0 -65
- data/spec/Classes/CASelect_spec.rb +0 -22
- data/spec/Classes/CAShift_spec.rb +0 -16
- data/spec/Classes/CAStruct_spec.rb +0 -71
- data/spec/Classes/CATranspose_spec.rb +0 -60
- data/spec/Classes/CAUnboudRepeat_spec.rb +0 -102
- data/spec/Classes/CAWindow_spec.rb +0 -54
- data/spec/Classes/CAWrap_spec.rb +0 -8
- data/spec/Classes/CArray_spec.rb +0 -184
- data/spec/Classes/CScalar_spec.rb +0 -55
- data/spec/Classes/ex1.rb +0 -46
- data/spec/Features/feature_130_spec.rb +0 -19
- data/spec/Features/feature_attributes_spec.rb +0 -280
- data/spec/Features/feature_boolean_spec.rb +0 -98
- data/spec/Features/feature_broadcast.rb +0 -116
- data/spec/Features/feature_cast_function.rb +0 -19
- data/spec/Features/feature_cast_spec.rb +0 -33
- data/spec/Features/feature_class_spec.rb +0 -84
- data/spec/Features/feature_complex_spec.rb +0 -42
- data/spec/Features/feature_composite_spec.rb +0 -124
- data/spec/Features/feature_convert_spec.rb +0 -46
- data/spec/Features/feature_copy_spec.rb +0 -123
- data/spec/Features/feature_creation_spec.rb +0 -84
- data/spec/Features/feature_element_spec.rb +0 -144
- data/spec/Features/feature_extream_spec.rb +0 -54
- data/spec/Features/feature_generate_spec.rb +0 -74
- data/spec/Features/feature_index_spec.rb +0 -69
- data/spec/Features/feature_mask_spec.rb +0 -580
- data/spec/Features/feature_math_spec.rb +0 -97
- data/spec/Features/feature_order_spec.rb +0 -146
- data/spec/Features/feature_ref_store_spec.rb +0 -209
- data/spec/Features/feature_serialization_spec.rb +0 -125
- data/spec/Features/feature_stat_spec.rb +0 -397
- data/spec/Features/feature_virtual_spec.rb +0 -48
- data/spec/Features/method_eq_spec.rb +0 -81
- data/spec/Features/method_is_nan_spec.rb +0 -12
- data/spec/Features/method_map_spec.rb +0 -54
- data/spec/Features/method_max_with.rb +0 -20
- data/spec/Features/method_min_with.rb +0 -19
- data/spec/Features/method_ne_spec.rb +0 -18
- data/spec/Features/method_project_spec.rb +0 -188
- data/spec/Features/method_ref_spec.rb +0 -27
- data/spec/Features/method_round_spec.rb +0 -11
- data/spec/Features/method_s_linspace_spec.rb +0 -48
- data/spec/Features/method_s_span_spec.rb +0 -14
- data/spec/Features/method_seq_spec.rb +0 -47
- data/spec/Features/method_sort_with.rb +0 -43
- data/spec/Features/method_sorted_with.rb +0 -29
- data/spec/Features/method_span_spec.rb +0 -42
- data/spec/Features/method_wrap_readonly_spec.rb +0 -43
- data/spec/UnitTest/test_CAVirtual.rb +0 -214
- data/spec/spec_all.rb +0 -10
- data/utils/ca_ase.rb +0 -21
- data/utils/ca_methods.rb +0 -15
- data/utils/cast_checker.rb +0 -30
- data/utils/convert_test.rb +0 -73
- data/utils/extract_yard.rb +0 -22
- data/utils/guess_shape.rb +0 -76
- data/utils/monkey_patch_methods.rb +0 -62
- data/utils/remove_resource_fork.sh +0 -5
|
@@ -0,0 +1,1062 @@
|
|
|
1
|
+
# ----------------------------------------------------------------------------
|
|
2
|
+
#
|
|
3
|
+
# carray/categorical_iterator.rb
|
|
4
|
+
#
|
|
5
|
+
# CACategoricalIterator — a per-category reduction dispatcher. `cat` (a
|
|
6
|
+
# CACategorical) brings the equivalence-class classification; `value` is the
|
|
7
|
+
# payload. `value.group_by_category(cat)` lays `value` out as a
|
|
8
|
+
# category-contiguous eager copy by gathering it through `cat`'s cached
|
|
9
|
+
# grouping plan (the counting sort lives on the categorical, built once and
|
|
10
|
+
# shared across every payload column) and offers per-category reductions off
|
|
11
|
+
# `cat.reduceat_index` as segment boundaries:
|
|
12
|
+
#
|
|
13
|
+
# grp = value.group_by_category(cat)
|
|
14
|
+
# grp.max # per-category max
|
|
15
|
+
# grp.median # order statistics share the same surface
|
|
16
|
+
# grp.stddev
|
|
17
|
+
#
|
|
18
|
+
# This is the consumer of the sort_addr / reduceat_index foundation on
|
|
19
|
+
# CACategorical. Order statistics (median / percentile / quantile) cannot be
|
|
20
|
+
# scattered — they need every value of a group held together — so the values
|
|
21
|
+
# are materialized into contiguous blocks once, and every reduction (monoid or
|
|
22
|
+
# order-stat) then folds those held blocks.
|
|
23
|
+
#
|
|
24
|
+
# Each group is delegated to the same-named CArray reduction over that group's
|
|
25
|
+
# members, so a group result equals `CArray#<reduction>` for those members and
|
|
26
|
+
# the mask contract (empty / all-masked -> identity for sum/prod, UNDEF for
|
|
27
|
+
# ratios) carries through unchanged. Results are length-k CArrays aligned to
|
|
28
|
+
# `cat.labels`; undefined slots are MASKED cells (never magic floats). Output
|
|
29
|
+
# dtype and the empty / all-masked answer per method:
|
|
30
|
+
#
|
|
31
|
+
# elements -> int64, classified cells (incl. masked)
|
|
32
|
+
# count / count_not_masked / count_masked / count(v) -> int64
|
|
33
|
+
# sum -> value dtype, empty/all-masked = 0 (identity)
|
|
34
|
+
# prod -> float64, empty/all-masked = 1 (identity)
|
|
35
|
+
# max / min -> value dtype, empty/all-masked = MASKED
|
|
36
|
+
# mean -> float64, empty/all-masked = MASKED
|
|
37
|
+
# median / percentile -> float64, empty/all-masked = MASKED
|
|
38
|
+
# variance / stddev (sample, ddof=1) -> float64, empty/all-masked = MASKED,
|
|
39
|
+
# single value = 0.0 (n=1 contract)
|
|
40
|
+
# all / any -> boolean (boolean value dtype only)
|
|
41
|
+
# labels -> cat.labels
|
|
42
|
+
#
|
|
43
|
+
# Generic iteration (the escape hatch for statistics not in the named surface),
|
|
44
|
+
# matching each_slab / reduce_slab: each { |members| ... } yields per category,
|
|
45
|
+
# reduce { |members| ... } / reduce(init) { |acc, e| ... } folds each category
|
|
46
|
+
# to one value (length-k). map (an element-wise group-wise transform back to
|
|
47
|
+
# the source shape) is a later pass -- use reduce for aggregation.
|
|
48
|
+
#
|
|
49
|
+
# prod / all / any / count(v) are per-group fallbacks (they delegate to the
|
|
50
|
+
# CArray reduction per category); a fused reduceat for them is a later pass.
|
|
51
|
+
#
|
|
52
|
+
# Names (group_by_category / CACategoricalIterator) are provisional; the
|
|
53
|
+
# contract (grouped copy + offsets + labels + per-category reduction) is the
|
|
54
|
+
# ground truth.
|
|
55
|
+
#
|
|
56
|
+
# The per-column gather and the segmented reduction both run in Ruby; the
|
|
57
|
+
# counting sort itself (the dominant cost) is a C kernel cached on the
|
|
58
|
+
# categorical, so a wide aggregate pays it once.
|
|
59
|
+
#
|
|
60
|
+
# ----------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
require "carray"
|
|
63
|
+
|
|
64
|
+
# A CAIterator over the categories of a CACategorical. CAIterator is the
|
|
65
|
+
# family base (the built-in iterators like CAWindowIterator / CABlockIterator
|
|
66
|
+
# are defined in C); a Ruby `Foo < CAIterator` supplies its own behaviour and
|
|
67
|
+
# does not lean on the base machinery. Like CASlabIterator, this class defines
|
|
68
|
+
# its own `each` (over the k categories, yielding each category's member slice),
|
|
69
|
+
# which drives the inherited Enumerable surface; the reduction methods (sum /
|
|
70
|
+
# mean / median / ...) aggregate the groups into length-k arrays. The kernels
|
|
71
|
+
# are per-category slices of an eager, category-contiguous grouped copy. This
|
|
72
|
+
# supersedes the older CAClassIterator.
|
|
73
|
+
class CACategoricalIterator < CAIterator
|
|
74
|
+
|
|
75
|
+
# value : the payload CArray to reduce, one cell per categorical cell.
|
|
76
|
+
# cat : the CACategorical carrying the classification.
|
|
77
|
+
#
|
|
78
|
+
# Lays the value out category-contiguous by GATHERING it through the
|
|
79
|
+
# categorical's cached grouping plan (the counting sort lives on `cat`, built
|
|
80
|
+
# once and shared by every payload column and iterator — see
|
|
81
|
+
# CACategorical's grouping-plan note). The plan gives the segment STARTS
|
|
82
|
+
# (reduceat_index) and the group-major permutation (perm[slot] = source index
|
|
83
|
+
# at that grouped slot, = the valid prefix of sort_addr); gathering value
|
|
84
|
+
# through perm is the only per-column work. Excluded cells (masked or
|
|
85
|
+
# out-of-vocabulary code) are absent from perm, so they never join a group;
|
|
86
|
+
# the value mask rides the gather into the grouped copy.
|
|
87
|
+
def initialize (value, cat)
|
|
88
|
+
@cat = cat
|
|
89
|
+
@labels = cat.labels
|
|
90
|
+
@k = cat.labels.size
|
|
91
|
+
@value = value # source, kept for #cumsum etc.
|
|
92
|
+
@src_shape = value.shape # output shape for #map
|
|
93
|
+
@ndim = 1 # 1-D iterator over k categories
|
|
94
|
+
@shape = [@k]
|
|
95
|
+
|
|
96
|
+
if value.elements == cat.elements
|
|
97
|
+
# Flat classifier path (backward compat): cat classifies every cell of
|
|
98
|
+
# value one-to-one, so eager counting-sort gather is meaningful. This is
|
|
99
|
+
# what all no-axis reductions consume — case C (all cells collapse into k
|
|
100
|
+
# buckets) plus case B interpreted flatly.
|
|
101
|
+
# category_sizes IS the per-group cell counts (what #elements returns);
|
|
102
|
+
# the segment STARTS are its cached exclusive prefix scan (cat.reduceat_
|
|
103
|
+
# index): offsets[c] = sum of counts[0...c]. Both come off the shared
|
|
104
|
+
# plan, so the counting sort is not repeated here.
|
|
105
|
+
@elements = cat.category_sizes.int64
|
|
106
|
+
nvalid = @elements.sum
|
|
107
|
+
@offsets = cat.reduceat_index # cached segment STARTS (int64[k])
|
|
108
|
+
# Group-major source indices = the valid prefix of the cached sort_addr.
|
|
109
|
+
# With no classified cell the prefix is empty (and slicing a length-0
|
|
110
|
+
# sort_addr would be out of range), so take the empty permutation directly.
|
|
111
|
+
@perm = nvalid > 0 ? cat.sort_addr[0...nvalid] : CArray.int64(0)
|
|
112
|
+
@codes = cat.codes.reshape(cat.elements) # flat codes view (map re-walk / weights)
|
|
113
|
+
# Gather value into category-contiguous order via the cached permutation
|
|
114
|
+
# and materialise (the reduceat kernels read the grouped buffer's raw ptr,
|
|
115
|
+
# so it must be a contiguous entity, not the selection view); the value
|
|
116
|
+
# mask rides the gather. Payload-dependent, so this is the only part
|
|
117
|
+
# rebuilt per column. With no classified cell (empty / all-excluded) there
|
|
118
|
+
# is nothing to gather — an empty index into an empty source is out of
|
|
119
|
+
# range — so build the empty grouped buffer directly.
|
|
120
|
+
@grouped = nvalid > 0 ? value.reshape(value.elements)[@perm].copy
|
|
121
|
+
: CArray.new(value.data_type, [0])
|
|
122
|
+
@empty = CArray.new(@grouped.data_type, [0])
|
|
123
|
+
else
|
|
124
|
+
# Shape mismatch: only per-fiber axis: dispatch could still work. With a
|
|
125
|
+
# 1-D value there is no fiber structure to broadcast into, so a mismatch
|
|
126
|
+
# is unrecoverable (preserves the old strict check). For higher-rank
|
|
127
|
+
# value, defer validation to reduce time — check only that cat.ndim fits
|
|
128
|
+
# one of the 3 axis: cases (§2.2 of PROPOSAL_CATEGORICAL_REDUCE_AXIS);
|
|
129
|
+
# any no-axis reduce called on this iterator will surface the mismatch
|
|
130
|
+
# because @grouped stays undefined.
|
|
131
|
+
if value.ndim == 1 ||
|
|
132
|
+
! [1, value.ndim - 1, value.ndim].include?(cat.ndim)
|
|
133
|
+
raise ArgumentError,
|
|
134
|
+
"group_by_category: value.elements (#{value.elements}) != " \
|
|
135
|
+
"cat.elements (#{cat.elements})" +
|
|
136
|
+
(value.ndim == 1 ? "" :
|
|
137
|
+
". For per-fiber reduce use `.sum(axis: k)`; cat.ndim=" \
|
|
138
|
+
"#{cat.ndim} must be 1 (case A), #{value.ndim} (case B), " \
|
|
139
|
+
"or #{value.ndim - 1} (band-only) for h.ndim=#{value.ndim}.")
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
self
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# @overload each { |members| ... }
|
|
146
|
+
# Yields each category's members (a CArray slice of the grouped copy, in
|
|
147
|
+
# {#labels} order; an empty category yields an empty array). Without a
|
|
148
|
+
# block, returns an Enumerator. This is the own iteration that drives the
|
|
149
|
+
# inherited Enumerable methods (map / count / to_a / ...); it does not use
|
|
150
|
+
# the CAIterator base each / kernel_at_addr path.
|
|
151
|
+
# @yieldparam members [CArray]
|
|
152
|
+
# @return [Enumerator, self]
|
|
153
|
+
def each
|
|
154
|
+
return to_enum(:each) unless block_given?
|
|
155
|
+
@k.times { |c| yield group_slice(c) }
|
|
156
|
+
self
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# @overload labels
|
|
160
|
+
# Returns the category vocabulary the results are aligned to.
|
|
161
|
+
# @return [Array]
|
|
162
|
+
attr_reader :labels
|
|
163
|
+
|
|
164
|
+
# @overload ngroups
|
|
165
|
+
# Returns the number of groups (= `labels.size`); the length of every
|
|
166
|
+
# per-group result CArray the reductions return.
|
|
167
|
+
# @return [Integer]
|
|
168
|
+
def ngroups
|
|
169
|
+
@k
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# @overload elements
|
|
173
|
+
# Returns per-group cell counts (classified cells, including value-masked
|
|
174
|
+
# ones; = `cat.category_sizes`), a length-ngroups CArray aligned to
|
|
175
|
+
# {#labels}. The CAIterator count-family member — `CArray#elements`
|
|
176
|
+
# (structural, mask-independent) lifted per group.
|
|
177
|
+
# @return [CArray]
|
|
178
|
+
def elements
|
|
179
|
+
@elements
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Group-vocabulary alias for {#elements}; reads naturally next to
|
|
183
|
+
# {#ngroups} and mirrors `CACategorical#category_sizes`.
|
|
184
|
+
alias group_sizes elements
|
|
185
|
+
|
|
186
|
+
# @overload inspect
|
|
187
|
+
# Returns a compact one-line summary — the group count, the label
|
|
188
|
+
# vocabulary, and the per-group cell counts — instead of dumping the
|
|
189
|
+
# internal grouped/value/codes buffers.
|
|
190
|
+
# @return [String]
|
|
191
|
+
def inspect
|
|
192
|
+
"#<#{self.class} ngroups=#{@k} labels=#{@labels.inspect} " \
|
|
193
|
+
"elements=#{@elements.to_a.inspect}>"
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# @overload count_not_masked
|
|
197
|
+
# Returns the per-category count of present (non-masked) values as int64
|
|
198
|
+
# — the denominator the value reductions actually divide by. Equals
|
|
199
|
+
# {#elements} unless the value carries a mask. A count is always defined,
|
|
200
|
+
# so an empty category is `0` (never masked).
|
|
201
|
+
# @return [CArray]
|
|
202
|
+
# @overload count_not_masked(axis:)
|
|
203
|
+
# Per-fiber per-category count of present (non-masked) values along `axis`
|
|
204
|
+
# (int64, shape [K, ...band]). Empty cells are `0`.
|
|
205
|
+
# @param axis [Integer]
|
|
206
|
+
# @return [CArray]
|
|
207
|
+
def count_not_masked(axis: nil)
|
|
208
|
+
return axis_moments(axis)[:count] if axis
|
|
209
|
+
m = moments
|
|
210
|
+
m ? m[:count] : per_category(CA_INT64) { |s| s.count_not_masked }
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# @overload count(v = <none>)
|
|
214
|
+
# Per-category count, mirroring `CArray#count` per group. No argument
|
|
215
|
+
# returns {#count_not_masked} (present cells); `count(UNDEF)` returns
|
|
216
|
+
# {#count_masked}; `count(v)` counts cells whose value equals `v`.
|
|
217
|
+
# @return [CArray] length-k int64, aligned to {#labels}
|
|
218
|
+
# @overload count(axis:)
|
|
219
|
+
# No-arg + axis: = per-fiber per-category count_not_masked (shape [K, ...band]).
|
|
220
|
+
# `count(v, axis:)` (value equality) and `count(UNDEF, axis:)` are deferred
|
|
221
|
+
# to Phase 3 of PROPOSAL_CATEGORICAL_REDUCE_AXIS.
|
|
222
|
+
# @param axis [Integer]
|
|
223
|
+
# @return [CArray]
|
|
224
|
+
def count (*args, axis: nil)
|
|
225
|
+
if axis
|
|
226
|
+
return count_not_masked(axis: axis) if args.empty?
|
|
227
|
+
raise NotImplementedError,
|
|
228
|
+
"CACategoricalIterator#count(v, axis:) not yet implemented — " \
|
|
229
|
+
"value-equality count with axis: deferred to Phase 3 of " \
|
|
230
|
+
"PROPOSAL_CATEGORICAL_REDUCE_AXIS."
|
|
231
|
+
end
|
|
232
|
+
return count_not_masked if args.empty?
|
|
233
|
+
# Delegate per group to CArray#count (handles count(UNDEF) -> masked count and
|
|
234
|
+
# count(v) alike, with core's exact dtype equality). The group slice is a
|
|
235
|
+
# CABlock, whose own #count is the block geometry accessor, so dispatch
|
|
236
|
+
# CArray#count explicitly. (Not fused: a value-equality reduceat would have
|
|
237
|
+
# to reproduce core's cross-type / out-of-range equality exactly.)
|
|
238
|
+
cnt = CArray.instance_method(:count)
|
|
239
|
+
per_category(CA_INT64) { |s| cnt.bind_call(s, *args) }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# @overload count_masked
|
|
243
|
+
# Returns the per-category count of masked (missing) values as int64.
|
|
244
|
+
# Empty categories are `0`.
|
|
245
|
+
# @return [CArray]
|
|
246
|
+
# @overload count_masked(axis:)
|
|
247
|
+
# Deferred to Phase 3 of PROPOSAL_CATEGORICAL_REDUCE_AXIS (needs a separate
|
|
248
|
+
# "assigned count" scatter that counts cells regardless of value mask).
|
|
249
|
+
# @param axis [Integer]
|
|
250
|
+
# @return [CArray]
|
|
251
|
+
def count_masked(axis: nil)
|
|
252
|
+
if axis
|
|
253
|
+
raise NotImplementedError,
|
|
254
|
+
"CACategoricalIterator#count_masked(axis:) not yet implemented — " \
|
|
255
|
+
"deferred to Phase 3 of PROPOSAL_CATEGORICAL_REDUCE_AXIS " \
|
|
256
|
+
"(needs a separate value-mask-only scatter kernel)."
|
|
257
|
+
end
|
|
258
|
+
m = moments
|
|
259
|
+
m ? @elements - m[:count] : per_category(CA_INT64) { |s| s.count_masked }
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# @overload sum
|
|
263
|
+
# Returns per-category sums in the value dtype. An empty or fully-masked
|
|
264
|
+
# category sums the empty set, which is the additive identity `0`
|
|
265
|
+
# (unmasked) — the same contract as `CArray#sum` on an empty / all-masked
|
|
266
|
+
# array.
|
|
267
|
+
# @return [CArray]
|
|
268
|
+
# @overload sum(axis:)
|
|
269
|
+
# Returns per-category sums per fiber along `axis`. Cat may be 1-D (case
|
|
270
|
+
# A, broadcasts across band axes), same rank as source (case B, per-fiber
|
|
271
|
+
# independent classifier), or one rank less (band-only, constant along
|
|
272
|
+
# reduce axis). Output shape = `[K, ...source.shape without axis]`. See
|
|
273
|
+
# PROPOSAL_CATEGORICAL_REDUCE_AXIS.
|
|
274
|
+
# @param axis [Integer] reduce axis of the source value.
|
|
275
|
+
# @return [CArray]
|
|
276
|
+
def sum(axis: nil)
|
|
277
|
+
return axis_sum(axis) if axis
|
|
278
|
+
m = moments
|
|
279
|
+
return per_category(@grouped.data_type) { |s| s.sum } unless m
|
|
280
|
+
out = CArray.new(@grouped.data_type, [@k])
|
|
281
|
+
out[] = m[:sum] # cast float64 sums -> value dtype (empty -> 0)
|
|
282
|
+
out
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# @overload max
|
|
286
|
+
# Returns per-category maxima in the value dtype. Empty categories are
|
|
287
|
+
# MASKED.
|
|
288
|
+
# @return [CArray]
|
|
289
|
+
# @overload max(axis:)
|
|
290
|
+
# Per-fiber per-category maxima along `axis` (h dtype, masked where empty).
|
|
291
|
+
# @param axis [Integer]
|
|
292
|
+
# @return [CArray]
|
|
293
|
+
def max(axis: nil)
|
|
294
|
+
return axis_moments(axis)[:max] if axis
|
|
295
|
+
m = moments
|
|
296
|
+
m ? m[:max] : per_category(@grouped.data_type) { |s| s.max }
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# @overload min
|
|
300
|
+
# Returns per-category minima in the value dtype. Empty categories are
|
|
301
|
+
# MASKED.
|
|
302
|
+
# @return [CArray]
|
|
303
|
+
# @overload min(axis:)
|
|
304
|
+
# Per-fiber per-category minima along `axis` (h dtype, masked where empty).
|
|
305
|
+
# @param axis [Integer]
|
|
306
|
+
# @return [CArray]
|
|
307
|
+
def min(axis: nil)
|
|
308
|
+
return axis_moments(axis)[:min] if axis
|
|
309
|
+
m = moments
|
|
310
|
+
m ? m[:min] : per_category(@grouped.data_type) { |s| s.min }
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# @overload mean
|
|
314
|
+
# Returns per-category means as float64. Empty categories are MASKED.
|
|
315
|
+
# @return [CArray]
|
|
316
|
+
# @overload mean(axis:)
|
|
317
|
+
# Per-fiber per-category means (float64, empty group cells MASKED).
|
|
318
|
+
# @param axis [Integer]
|
|
319
|
+
# @return [CArray]
|
|
320
|
+
def mean(axis: nil)
|
|
321
|
+
return axis_mean(axis) if axis
|
|
322
|
+
m = moments
|
|
323
|
+
return per_category(CA_FLOAT64) { |s| s.mean } unless m
|
|
324
|
+
cnt = m[:count]
|
|
325
|
+
out = m[:sum] / cnt.float64 # count 0 -> NaN, masked next
|
|
326
|
+
out[cnt.eq(0)] = UNDEF # empty / all-masked category -> MASKED
|
|
327
|
+
out
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# @overload median
|
|
331
|
+
# Returns per-category medians as float64. Empty categories are MASKED.
|
|
332
|
+
# @return [CArray]
|
|
333
|
+
def median(axis: nil)
|
|
334
|
+
axis_order_stat_defer!(:median) if axis
|
|
335
|
+
percentile(50.0)
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# @overload percentile(p)
|
|
339
|
+
# Returns the per-category `p`-th percentile as float64 (`p` in 0..100,
|
|
340
|
+
# `:linear` interpolation, matching `CArray#percentile`). Empty categories
|
|
341
|
+
# are MASKED. Order statistics need every value of a group held together —
|
|
342
|
+
# this is the reduceat that only the eager grouped copy can serve.
|
|
343
|
+
# @param p [Numeric] percentile in 0..100.
|
|
344
|
+
# @return [CArray]
|
|
345
|
+
def percentile (p, axis: nil)
|
|
346
|
+
axis_order_stat_defer!(:percentile) if axis
|
|
347
|
+
unless MONOID_TYPES.include?(@grouped.data_type)
|
|
348
|
+
return per_category(CA_FLOAT64) { |s| s.percentile(p) }
|
|
349
|
+
end
|
|
350
|
+
out = CArray.float64(@k)
|
|
351
|
+
@grouped.send(:__reduceat_percentile__, @offsets, p.to_f, out)
|
|
352
|
+
out
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# @overload quantile
|
|
356
|
+
# Returns the per-category five-number summary `[min, Q1, median, Q3, max]`
|
|
357
|
+
# as five length-k float64 CArrays (matching `CArray#quantile`): the
|
|
358
|
+
# percentiles at 0 / 25 / 50 / 75 / 100. Empty / all-masked categories are
|
|
359
|
+
# MASKED. For a single fraction q in 0..1 use `percentile(q * 100)`.
|
|
360
|
+
# @return [Array<CArray>]
|
|
361
|
+
def quantile
|
|
362
|
+
unless MONOID_TYPES.include?(@grouped.data_type)
|
|
363
|
+
return [0, 25, 50, 75, 100].map { |p| percentile(p) }
|
|
364
|
+
end
|
|
365
|
+
outs = Array.new(5) { CArray.float64(@k) }
|
|
366
|
+
@grouped.send(:__reduceat_quantile__, @offsets, *outs)
|
|
367
|
+
outs
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
# @overload variance
|
|
371
|
+
# Returns per-category SAMPLE variance (ddof=1) as float64. Matches
|
|
372
|
+
# `CArray#variance` per group: an empty or fully-masked category is MASKED,
|
|
373
|
+
# a single-value category is `0.0` (CArray's n=1 contract), n>=2 is the
|
|
374
|
+
# sample variance.
|
|
375
|
+
# @return [CArray]
|
|
376
|
+
def variance(axis: nil)
|
|
377
|
+
return axis_variance_family(axis, :variance) if axis
|
|
378
|
+
m = moments
|
|
379
|
+
return per_category(CA_FLOAT64) { |s| s.variance } unless m
|
|
380
|
+
cnt = m[:count]
|
|
381
|
+
means = m[:sum] / cnt.float64 # per-segment mean (garbage where count 0/1,
|
|
382
|
+
out = CArray.float64(@k) # ignored by the kernel's n<2 guards)
|
|
383
|
+
@grouped.send(:__reduceat_variance__, @offsets, means, cnt, out)
|
|
384
|
+
out
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# @overload stddev
|
|
388
|
+
# Returns per-category SAMPLE standard deviation (ddof=1) as float64.
|
|
389
|
+
# Matches `CArray#stddev` per group (empty / all-masked MASKED,
|
|
390
|
+
# single-value `0.0`).
|
|
391
|
+
# @return [CArray]
|
|
392
|
+
def stddev(axis: nil)
|
|
393
|
+
return axis_variance_family(axis, :stddev) if axis
|
|
394
|
+
m = moments
|
|
395
|
+
return per_category(CA_FLOAT64) { |s| s.stddev } unless m
|
|
396
|
+
variance.sqrt # sqrt propagates the n=0 mask
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# @overload prod
|
|
400
|
+
# Returns per-category products as float64 (matching `CArray#prod`). An
|
|
401
|
+
# empty / fully-masked category is `1.0` (the multiplicative identity).
|
|
402
|
+
# Single-pass reduceat for numeric values; per-group fallback otherwise.
|
|
403
|
+
# @return [CArray]
|
|
404
|
+
# @overload prod(axis:)
|
|
405
|
+
# Per-fiber per-category products (float64, shape [K, ...band]). Empty
|
|
406
|
+
# group cells `1.0` (identity).
|
|
407
|
+
# @param axis [Integer]
|
|
408
|
+
# @return [CArray]
|
|
409
|
+
def prod(axis: nil)
|
|
410
|
+
return axis_prod(axis) if axis
|
|
411
|
+
return per_category(CA_FLOAT64) { |s| s.prod } unless MONOID_TYPES.include?(@grouped.data_type)
|
|
412
|
+
out = CArray.float64(@k)
|
|
413
|
+
@grouped.send(:__reduceat_prod__, @offsets, out)
|
|
414
|
+
out
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
# @overload all
|
|
418
|
+
# Returns the per-category `all` as boolean (matching `CArray#all`): true
|
|
419
|
+
# iff every present value is truthy (empty category -> true, vacuously).
|
|
420
|
+
# The value dtype must be boolean, as for `CArray#all`.
|
|
421
|
+
# @return [CArray]
|
|
422
|
+
def all
|
|
423
|
+
aa = all_any
|
|
424
|
+
aa ? aa[:all] : per_category(CA_BOOLEAN) { |s| s.all }
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
# @overload any
|
|
428
|
+
# Returns the per-category `any` as boolean (matching `CArray#any`): true
|
|
429
|
+
# iff some present value is truthy (empty category -> false). The value
|
|
430
|
+
# dtype must be boolean, as for `CArray#any`.
|
|
431
|
+
# @return [CArray]
|
|
432
|
+
def any
|
|
433
|
+
aa = all_any
|
|
434
|
+
aa ? aa[:any] : per_category(CA_BOOLEAN) { |s| s.any }
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
# ---- tier 2 (fused / population / position) ------------------------------
|
|
438
|
+
|
|
439
|
+
# @overload minmax
|
|
440
|
+
# Returns the per-category `[min, max]` pair (each a length-k CArray in the
|
|
441
|
+
# value dtype; empty categories MASKED), matching `CArray#minmax`. Both come
|
|
442
|
+
# from the single cached moments pass.
|
|
443
|
+
# @return [Array<CArray>]
|
|
444
|
+
# @overload minmax(axis:)
|
|
445
|
+
# Per-fiber `[min_ca, max_ca]` along `axis` (each shape [K, ...band], h dtype,
|
|
446
|
+
# empty group cells MASKED). Ruby Array of two CArrays, not stacked.
|
|
447
|
+
# @param axis [Integer]
|
|
448
|
+
# @return [Array<CArray>]
|
|
449
|
+
def minmax(axis: nil)
|
|
450
|
+
return [min(axis: axis), max(axis: axis)] if axis
|
|
451
|
+
[min, max]
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
# @overload variancep
|
|
455
|
+
# Per-category POPULATION variance (ddof=0) as float64, matching
|
|
456
|
+
# `CArray#variancep`: empty / all-masked -> MASKED, single value -> 0.0.
|
|
457
|
+
# Derived from the sample variance (variancep = variance * (n-1) / n), so it
|
|
458
|
+
# reuses the centred two-pass kernel with no extra walk.
|
|
459
|
+
# @return [CArray]
|
|
460
|
+
def variancep(axis: nil)
|
|
461
|
+
return axis_variance_family(axis, :variancep) if axis
|
|
462
|
+
m = moments
|
|
463
|
+
return per_category(CA_FLOAT64) { |s| s.variancep } unless m
|
|
464
|
+
cnt = m[:count]
|
|
465
|
+
vp = variance * (cnt - 1).float64 / cnt.float64
|
|
466
|
+
vp[cnt.eq(0)] = UNDEF # empty / all-masked stays masked
|
|
467
|
+
vp
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
# @overload stddevp
|
|
471
|
+
# Per-category POPULATION standard deviation (ddof=0) as float64.
|
|
472
|
+
# @return [CArray]
|
|
473
|
+
# @overload stddevp(axis:)
|
|
474
|
+
# Per-fiber per-category population stddev (float64, empty group cells MASKED).
|
|
475
|
+
# @param axis [Integer]
|
|
476
|
+
# @return [CArray]
|
|
477
|
+
def stddevp(axis: nil)
|
|
478
|
+
return axis_variance_family(axis, :stddevp) if axis
|
|
479
|
+
m = moments
|
|
480
|
+
return per_category(CA_FLOAT64) { |s| s.stddevp } unless m
|
|
481
|
+
variancep.sqrt
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
# @overload min_index
|
|
485
|
+
# Per-category group-local index of the minimum — the position within the
|
|
486
|
+
# category's members (source order) — matching `CArray#min_index` per group.
|
|
487
|
+
# Empty / all-masked categories are MASKED. Single-pass fused reduceat for
|
|
488
|
+
# numeric values; per-group fallback otherwise.
|
|
489
|
+
# @return [CArray] length-k int64
|
|
490
|
+
def min_index
|
|
491
|
+
am = arg_minmax
|
|
492
|
+
am ? am[:min] : per_category(CA_INT64) { |s| s.min_index }
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
# @overload max_index
|
|
496
|
+
# Per-category group-local index of the maximum. See {#min_index}.
|
|
497
|
+
# @return [CArray] length-k int64
|
|
498
|
+
def max_index
|
|
499
|
+
am = arg_minmax
|
|
500
|
+
am ? am[:max] : per_category(CA_INT64) { |s| s.max_index }
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
# @overload min_addr
|
|
504
|
+
# Per-category flat source address of the minimum — which cell of the source
|
|
505
|
+
# value holds it, matching `CArray#min_addr` per group. Unlike {#min_index}
|
|
506
|
+
# (the group-local rank) this indexes back into the original array
|
|
507
|
+
# (`value.reshape(value.elements)[grp.min_addr]`). Empty categories MASKED.
|
|
508
|
+
# @return [CArray] length-k int64
|
|
509
|
+
def min_addr
|
|
510
|
+
group_addr(min_index)
|
|
511
|
+
end
|
|
512
|
+
|
|
513
|
+
# @overload max_addr
|
|
514
|
+
# Per-category flat source address of the maximum. See {#min_addr}.
|
|
515
|
+
# @return [CArray] length-k int64
|
|
516
|
+
def max_addr
|
|
517
|
+
group_addr(max_index)
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
# @overload sort_addr
|
|
521
|
+
# Per-category sort by flat source address. Returns a length-nvalid
|
|
522
|
+
# (= `elements.sum`) int64 CArray of the flat SOURCE addresses that sort each
|
|
523
|
+
# category's members, in group-major order: segment `c` holds category `c`'s
|
|
524
|
+
# source addresses in ascending-value order, segments concatenated in
|
|
525
|
+
# {#labels} order. So `value.reshape(value.elements)[grp.sort_addr]` yields
|
|
526
|
+
# the values grouped and sorted within each group, and splitting by the
|
|
527
|
+
# {#elements} prefix sum gives per-group. Excluded cells (in no category) are
|
|
528
|
+
# omitted. A masked value sorts to the tail of its segment (as `CArray#sort`
|
|
529
|
+
# sends masked cells to the end), so with a mask the first address is the
|
|
530
|
+
# minimum but the last is the masked cell, not the maximum.
|
|
531
|
+
#
|
|
532
|
+
# Unlike {#min_index} / {#max_index} (group-local rank), this indexes back
|
|
533
|
+
# into the original array. There is no group-local sort surface: a
|
|
534
|
+
# group-local rank order is weak (the grouped copy is already
|
|
535
|
+
# category-contiguous), so only the source-address form is offered, mirroring
|
|
536
|
+
# {#min_addr} vs the skipped group-local min_index-into-source.
|
|
537
|
+
# @return [CArray] length-nvalid int64
|
|
538
|
+
def sort_addr
|
|
539
|
+
out = CArray.int64(@grouped.elements)
|
|
540
|
+
@k.times do |c|
|
|
541
|
+
lo = @offsets[c]
|
|
542
|
+
hi = (c + 1 < @k) ? @offsets[c + 1] : @grouped.elements
|
|
543
|
+
next unless hi > lo
|
|
544
|
+
# View-local sort order of the segment (0..size-1), lifted to grouped
|
|
545
|
+
# slots, then mapped back to source addresses via perm.
|
|
546
|
+
out[lo...hi] = perm[@grouped[lo...hi].sort_addr + lo]
|
|
547
|
+
end
|
|
548
|
+
out
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
# @overload wsum(weights)
|
|
552
|
+
# Per-category weighted sum as float64, matching `CArray#wsum`. `weights` is
|
|
553
|
+
# a per-cell weight CArray in the source order (same elements as the value).
|
|
554
|
+
# Empty / all-masked category -> 0.0 (the additive identity). A cell is
|
|
555
|
+
# skipped iff its value OR its weight is masked (core's contract).
|
|
556
|
+
# @param weights [CArray]
|
|
557
|
+
# @return [CArray]
|
|
558
|
+
# @overload wsum(weights, axis:)
|
|
559
|
+
# Per-fiber per-category weighted sum along `axis`. `weights` must have
|
|
560
|
+
# shape == source.shape (rev3 requires explicit broadcast; wrap 1-D or
|
|
561
|
+
# band-shape weights via `.broadcast_to(*source.shape)` at the call site).
|
|
562
|
+
# Empty group cell → `0.0` (identity). Mask contract: cell contributes iff
|
|
563
|
+
# value AND weight are present.
|
|
564
|
+
# @param weights [CArray]
|
|
565
|
+
# @param axis [Integer]
|
|
566
|
+
# @return [CArray]
|
|
567
|
+
def wsum (weights, axis: nil)
|
|
568
|
+
return axis_wsum_wmean(weights, axis)[0] if axis
|
|
569
|
+
wg = scatter_weights(weights)
|
|
570
|
+
return kernel_weighted(wg)[0] if MONOID_TYPES.include?(@grouped.data_type)
|
|
571
|
+
fold_weighted(wg, 0.0) { |v, ws| v.wsum(ws) }
|
|
572
|
+
end
|
|
573
|
+
|
|
574
|
+
# @overload wmean(weights)
|
|
575
|
+
# Per-category weighted mean as float64, matching `CArray#wmean`. Empty
|
|
576
|
+
# category -> MASKED; a present category whose weights sum to zero -> NaN
|
|
577
|
+
# (core's 0/0 contract).
|
|
578
|
+
# @param weights [CArray]
|
|
579
|
+
# @return [CArray]
|
|
580
|
+
# @overload wmean(weights, axis:)
|
|
581
|
+
# Per-fiber per-category weighted mean along `axis`. Same weights-shape
|
|
582
|
+
# contract as {#wsum} (weights.shape == source.shape). Empty cell → MASKED;
|
|
583
|
+
# a present cell whose weights sum to zero → NaN (0/0 core contract).
|
|
584
|
+
# @param weights [CArray]
|
|
585
|
+
# @param axis [Integer]
|
|
586
|
+
# @return [CArray]
|
|
587
|
+
def wmean (weights, axis: nil)
|
|
588
|
+
return axis_wsum_wmean(weights, axis)[1] if axis
|
|
589
|
+
wg = scatter_weights(weights)
|
|
590
|
+
return kernel_weighted(wg)[1] if MONOID_TYPES.include?(@grouped.data_type)
|
|
591
|
+
fold_weighted(wg, UNDEF) { |v, ws| v.wmean(ws) }
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
# @overload reduce { |members| ... }
|
|
595
|
+
# Custom per-category reduction (the escape hatch for statistics not in the
|
|
596
|
+
# named surface), mirroring `CArray#reduce_slab`. The block receives each
|
|
597
|
+
# category's members (a CArray) and returns one value per category.
|
|
598
|
+
# @yieldparam members [CArray]
|
|
599
|
+
# @return [CArray] length-k, aligned to {#labels}
|
|
600
|
+
# @overload reduce(init) { |acc, elem| ... }
|
|
601
|
+
# Per-category fiber fold: each category's members are folded element by
|
|
602
|
+
# element starting from `init`.
|
|
603
|
+
# @param init [Object] initial accumulator.
|
|
604
|
+
# @return [CArray] length-k
|
|
605
|
+
def reduce (*args, data_type: nil, &blk)
|
|
606
|
+
raise LocalJumpError, "no block given (yield)" unless blk
|
|
607
|
+
dt = data_type || CA_OBJECT
|
|
608
|
+
if args.empty?
|
|
609
|
+
per_category(dt) { |s| blk.call(s) }
|
|
610
|
+
else
|
|
611
|
+
init = args[0]
|
|
612
|
+
per_category(dt) { |s|
|
|
613
|
+
acc = init
|
|
614
|
+
s.each { |e| acc = blk.call(acc, e) }
|
|
615
|
+
acc
|
|
616
|
+
}
|
|
617
|
+
end
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# @overload map(data_type: nil) { |members| ... }
|
|
621
|
+
# Group-wise element-wise transform, mirroring `CArray#map_slab`. The block
|
|
622
|
+
# receives each category's members and returns either a same-length CArray
|
|
623
|
+
# (scattered back cell for cell) or a scalar (broadcast over the group's
|
|
624
|
+
# cells). Returns a NEW CArray shaped like the source `value`; the original
|
|
625
|
+
# is not modified (`value[] = grp.map { ... }` for in-place). Excluded cells
|
|
626
|
+
# (in no category) are UNDEF in the result.
|
|
627
|
+
# @yieldparam members [CArray]
|
|
628
|
+
# @return [CArray] shaped like the source value
|
|
629
|
+
def map (data_type: nil)
|
|
630
|
+
raise LocalJumpError, "no block given (yield)" unless block_given?
|
|
631
|
+
dt = data_type || @grouped.data_type
|
|
632
|
+
# Apply the block per category, assembled in grouped (category-contiguous)
|
|
633
|
+
# order: a same-length result scatters cell for cell, a scalar broadcasts.
|
|
634
|
+
transformed = CArray.new(dt, [@grouped.elements])
|
|
635
|
+
@k.times do |c|
|
|
636
|
+
lo = @offsets[c]
|
|
637
|
+
hi = (c + 1 < @k) ? @offsets[c + 1] : @grouped.elements
|
|
638
|
+
transformed[lo...hi] = yield(@grouped[lo...hi]) if hi > lo
|
|
639
|
+
end
|
|
640
|
+
# Scatter back to source positions via the permutation (grouped-order source
|
|
641
|
+
# indices). Excluded cells are absent from perm and stay UNDEF.
|
|
642
|
+
out = CArray.new(dt, @src_shape)
|
|
643
|
+
out[] = UNDEF
|
|
644
|
+
out.reshape(@codes.elements)[perm] = transformed
|
|
645
|
+
out
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
# ---- segment scan: within-category running statistics ------------------
|
|
649
|
+
#
|
|
650
|
+
# The per-element-emit siblings of the reductions: unlike a reduction (which
|
|
651
|
+
# collapses each category to one value) a scan preserves the source shape,
|
|
652
|
+
# each cell holding its category's running statistic up to and including that
|
|
653
|
+
# cell, in source (row-major) order. A category is a partition (each cell is
|
|
654
|
+
# in exactly one category), so the running value is single-valued. The flat
|
|
655
|
+
# categorical grouping is the one-band case of the axis-group scan, so each
|
|
656
|
+
# routes straight through the fused C kernel __axis_group_scan__ (the same one
|
|
657
|
+
# CAGroupIterator drives) with the whole source as a single grouped axis and
|
|
658
|
+
# the categorical's codes as the single bundle -- which yields SOURCE-ORDER
|
|
659
|
+
# output directly, so no counting-sort inverse permutation is needed.
|
|
660
|
+
# Excluded (out-of-vocabulary / masked-code) and source-masked cells join no
|
|
661
|
+
# running total and are UNDEF. Mirroring the reductions (sum / mean), a scan
|
|
662
|
+
# takes no axis argument. cumsum / cumprod -> float64, cummax / cummin
|
|
663
|
+
# preserve the value dtype, cumcount -> int64 (1-based within-category
|
|
664
|
+
# ordinal); an object value dtype is carried by the kernel's object branch.
|
|
665
|
+
|
|
666
|
+
# @overload cumsum
|
|
667
|
+
# Per-category inclusive running sum (float64), source-shaped.
|
|
668
|
+
# @return [CArray]
|
|
669
|
+
# @overload cumprod
|
|
670
|
+
# Per-category inclusive running product (float64), source-shaped.
|
|
671
|
+
# @return [CArray]
|
|
672
|
+
# @overload cummax
|
|
673
|
+
# Per-category inclusive running maximum (value dtype), source-shaped.
|
|
674
|
+
# @return [CArray]
|
|
675
|
+
# @overload cummin
|
|
676
|
+
# Per-category inclusive running minimum (value dtype), source-shaped.
|
|
677
|
+
# @return [CArray]
|
|
678
|
+
# @overload cumcount
|
|
679
|
+
# Per-category 1-based within-category ordinal (int64), source-shaped.
|
|
680
|
+
# @return [CArray]
|
|
681
|
+
[:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
|
|
682
|
+
define_method(op) { scan(op) }
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
private
|
|
686
|
+
|
|
687
|
+
# Axis-aware moments (count / sum / min / max) — computed once per axis via
|
|
688
|
+
# the fused per-fiber scatter-reduce C kernel and cached (matches the flat
|
|
689
|
+
# #moments caching in spirit: pay one kernel per {iterator, axis} pair, share
|
|
690
|
+
# across sum / mean / min / max / minmax / count* consumers). Returns
|
|
691
|
+
# `{count: <int64>, sum: <float64>, min: <h dtype masked>, max: <h dtype masked>}`,
|
|
692
|
+
# all shape [K, ...band].
|
|
693
|
+
def axis_moments (axis)
|
|
694
|
+
@axis_moments_cache ||= {}
|
|
695
|
+
cached = @axis_moments_cache[axis]
|
|
696
|
+
return cached if cached
|
|
697
|
+
h = @value
|
|
698
|
+
unless axis.is_a?(Integer) && axis >= 0 && axis < h.ndim
|
|
699
|
+
raise ArgumentError,
|
|
700
|
+
"group_by_category.<reduce>(axis: #{axis.inspect}): axis must be an " \
|
|
701
|
+
"Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
702
|
+
end
|
|
703
|
+
codes_h_shape = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
704
|
+
band = h.shape.dup; band.delete_at(axis)
|
|
705
|
+
out_shape = [@k] + band
|
|
706
|
+
counts = CArray.int64(*out_shape)
|
|
707
|
+
sums = CArray.float64(*out_shape)
|
|
708
|
+
mins = CArray.new(h.data_type, out_shape)
|
|
709
|
+
maxs = CArray.new(h.data_type, out_shape)
|
|
710
|
+
h.__send__(:__fiber_scatter_moments__, codes_h_shape, axis, @k,
|
|
711
|
+
counts, sums, mins, maxs)
|
|
712
|
+
@axis_moments_cache[axis] = {count: counts, sum: sums, min: mins, max: maxs}
|
|
713
|
+
end
|
|
714
|
+
|
|
715
|
+
# Axis-aware sum: from moments, cast float64 sums to h dtype so empty-group
|
|
716
|
+
# identity 0 rides (matching flat #sum).
|
|
717
|
+
def axis_sum (axis)
|
|
718
|
+
m = axis_moments(axis)
|
|
719
|
+
out = CArray.new(@value.data_type, m[:sum].shape)
|
|
720
|
+
out[] = m[:sum]
|
|
721
|
+
out
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
# Axis-aware mean: sums / counts (float64); empty group cells (count=0) MASKED.
|
|
725
|
+
# Matches flat #mean per fiber.
|
|
726
|
+
def axis_mean (axis)
|
|
727
|
+
m = axis_moments(axis)
|
|
728
|
+
cnt = m[:count]
|
|
729
|
+
out = m[:sum] / cnt.float64 # count 0 -> NaN
|
|
730
|
+
out[cnt.eq(0)] = UNDEF # empty / all-masked -> MASKED
|
|
731
|
+
out
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
# Axis-aware variance / stddev / variancep / stddevp — Ruby-level per-c mask
|
|
735
|
+
# then delegate to the source's own axis-aware kernel. Order (median /
|
|
736
|
+
# percentile / quantile) is genuinely order-statistical (needs sort per
|
|
737
|
+
# group), and remains deferred; the variance family is only a centred
|
|
738
|
+
# two-pass numeric aggregate, so this loop hits the same ε-close two-pass
|
|
739
|
+
# kernel per (group, axis) that CArray#variance uses, no new C needed.
|
|
740
|
+
#
|
|
741
|
+
# Cost: K axis-reductions over an h-shaped local (most cells masked away for
|
|
742
|
+
# each c) — bounded by K, typically small. A fused per-fiber variance
|
|
743
|
+
# kernel is a natural follow-on if bench demands it.
|
|
744
|
+
def axis_variance_family (axis, op)
|
|
745
|
+
h = @value
|
|
746
|
+
unless axis.is_a?(Integer) && axis >= 0 && axis < h.ndim
|
|
747
|
+
raise ArgumentError,
|
|
748
|
+
"group_by_category.#{op}(axis: #{axis.inspect}): axis must be an " \
|
|
749
|
+
"Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
750
|
+
end
|
|
751
|
+
full_c = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
752
|
+
band = h.shape.dup; band.delete_at(axis)
|
|
753
|
+
out = CArray.float64(*([@k] + band))
|
|
754
|
+
slot_idx = [nil] + [nil] * band.size # placeholder; c fills slot 0
|
|
755
|
+
codes_bad = full_c.has_mask? ? full_c.is_masked : nil
|
|
756
|
+
@k.times do |c|
|
|
757
|
+
h_local = h.copy
|
|
758
|
+
# Boolean of cells that DO belong to group c (with codes present). On
|
|
759
|
+
# any masked codes cell the codes.eq(c) result carries UNDEF, which
|
|
760
|
+
# naturally reads as "not in group c" for our exclusion purpose.
|
|
761
|
+
in_c = full_c.eq(c)
|
|
762
|
+
exclude = in_c.not
|
|
763
|
+
exclude = exclude | codes_bad if codes_bad
|
|
764
|
+
h_local[exclude] = UNDEF
|
|
765
|
+
slice = h_local.__send__(op, axis: axis) # float64, band shape, mask carries n<contract
|
|
766
|
+
slot_idx[0] = c
|
|
767
|
+
out[*slot_idx] = slice
|
|
768
|
+
end
|
|
769
|
+
out
|
|
770
|
+
end
|
|
771
|
+
|
|
772
|
+
# Axis-aware wsum + wmean fused (single kernel call, both outputs). Returns
|
|
773
|
+
# [wsum_ca, wmean_ca]. Weights must match source shape exactly (explicit
|
|
774
|
+
# broadcast on the call site for 1-D or band-shape weights). A cell
|
|
775
|
+
# contributes iff its value AND its weight are present.
|
|
776
|
+
def axis_wsum_wmean (weights, axis)
|
|
777
|
+
h = @value
|
|
778
|
+
unless axis.is_a?(Integer) && axis >= 0 && axis < h.ndim
|
|
779
|
+
raise ArgumentError,
|
|
780
|
+
"group_by_category.wsum/wmean(axis: #{axis.inspect}): axis must " \
|
|
781
|
+
"be an Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
782
|
+
end
|
|
783
|
+
unless weights.is_a?(CArray) && weights.shape == h.shape
|
|
784
|
+
raise ArgumentError,
|
|
785
|
+
"group_by_category.wsum/wmean(axis: #{axis}): weights.shape " \
|
|
786
|
+
"#{weights.respond_to?(:shape) ? weights.shape.inspect : weights.class} " \
|
|
787
|
+
"must equal source.shape #{h.shape.inspect}. Wrap 1-D / band-shape " \
|
|
788
|
+
"weights via `.broadcast_to(*source.shape)` before passing."
|
|
789
|
+
end
|
|
790
|
+
codes_h_shape = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
791
|
+
weights_f64 = weights.data_type == CA_FLOAT64 ? weights : weights.float64
|
|
792
|
+
band = h.shape.dup; band.delete_at(axis)
|
|
793
|
+
ws_out = CArray.float64(*([@k] + band))
|
|
794
|
+
wm_out = CArray.float64(*([@k] + band))
|
|
795
|
+
h.__send__(:__fiber_scatter_wsum_wmean__, codes_h_shape, weights_f64,
|
|
796
|
+
axis, @k, ws_out, wm_out)
|
|
797
|
+
[ws_out, wm_out]
|
|
798
|
+
end
|
|
799
|
+
|
|
800
|
+
# Axis-aware prod: dedicated kernel (identity 1.0, separate from moments to
|
|
801
|
+
# avoid conflating with sum's zero-identity memset).
|
|
802
|
+
def axis_prod (axis)
|
|
803
|
+
h = @value
|
|
804
|
+
unless axis.is_a?(Integer) && axis >= 0 && axis < h.ndim
|
|
805
|
+
raise ArgumentError,
|
|
806
|
+
"group_by_category.prod(axis: #{axis.inspect}): axis must be an " \
|
|
807
|
+
"Integer in [0, #{h.ndim}) for source h with shape #{h.shape}"
|
|
808
|
+
end
|
|
809
|
+
codes_h_shape = resolve_axis_codes(@cat.codes, h.shape, axis)
|
|
810
|
+
band = h.shape.dup; band.delete_at(axis)
|
|
811
|
+
out = CArray.float64(*([@k] + band))
|
|
812
|
+
h.__send__(:__fiber_scatter_prod__, codes_h_shape, axis, @k, out)
|
|
813
|
+
out
|
|
814
|
+
end
|
|
815
|
+
|
|
816
|
+
# Broadcast `codes` to `h_shape` per PROPOSAL §2.2 3-case positional rule.
|
|
817
|
+
# Returns a broadcast view of codes at h_shape (or codes itself for case B).
|
|
818
|
+
# Raises ArgumentError with a message that enumerates all 3 accepted shapes.
|
|
819
|
+
def resolve_axis_codes (codes, h_shape, axis)
|
|
820
|
+
ndim = h_shape.size
|
|
821
|
+
band = h_shape.dup; band.delete_at(axis)
|
|
822
|
+
case codes.ndim
|
|
823
|
+
when 1
|
|
824
|
+
unless codes.shape == [h_shape[axis]]
|
|
825
|
+
axis_shape_mismatch!(codes.shape, h_shape, axis, band)
|
|
826
|
+
end
|
|
827
|
+
view_shape = Array.new(ndim, 1); view_shape[axis] = h_shape[axis]
|
|
828
|
+
codes.reshape(*view_shape).broadcast_to(*h_shape)
|
|
829
|
+
when ndim
|
|
830
|
+
unless codes.shape == h_shape
|
|
831
|
+
axis_shape_mismatch!(codes.shape, h_shape, axis, band)
|
|
832
|
+
end
|
|
833
|
+
codes
|
|
834
|
+
when ndim - 1
|
|
835
|
+
unless codes.shape == band
|
|
836
|
+
axis_shape_mismatch!(codes.shape, h_shape, axis, band)
|
|
837
|
+
end
|
|
838
|
+
view_shape = h_shape.dup; view_shape[axis] = 1
|
|
839
|
+
codes.reshape(*view_shape).broadcast_to(*h_shape)
|
|
840
|
+
else
|
|
841
|
+
axis_shape_mismatch!(codes.shape, h_shape, axis, band)
|
|
842
|
+
end
|
|
843
|
+
end
|
|
844
|
+
|
|
845
|
+
def axis_shape_mismatch! (cat_shape, h_shape, axis, band)
|
|
846
|
+
raise ArgumentError,
|
|
847
|
+
"group_by_category.sum(axis: #{axis}): cat.shape=#{cat_shape.inspect} " \
|
|
848
|
+
"does not fit any of the 3 accepted forms for h.shape=#{h_shape.inspect}: " \
|
|
849
|
+
"case A cat.shape=[#{h_shape[axis]}], " \
|
|
850
|
+
"case B cat.shape=#{h_shape.inspect}, " \
|
|
851
|
+
"band-only cat.shape=#{band.inspect}."
|
|
852
|
+
end
|
|
853
|
+
|
|
854
|
+
# Order-stat axis: is deferred to Phase 4 (per-fiber counting-sort C kernel).
|
|
855
|
+
# Called from median / percentile / variance / stddev when axis: is given.
|
|
856
|
+
def axis_order_stat_defer! (op)
|
|
857
|
+
raise NotImplementedError,
|
|
858
|
+
"CACategoricalIterator##{op}(axis:) not yet implemented — order " \
|
|
859
|
+
"statistics deferred to Phase 4 of PROPOSAL_CATEGORICAL_REDUCE_AXIS " \
|
|
860
|
+
"(needs per-fiber counting-sort C kernel, tracked separately)."
|
|
861
|
+
end
|
|
862
|
+
|
|
863
|
+
# Drive a segment scan through the axis-group scan kernel: the whole value as
|
|
864
|
+
# one grouped axis, the flat codes as the single bundle. The kernel emits in
|
|
865
|
+
# source order, so the flat result reshapes straight back to the source shape.
|
|
866
|
+
def scan (op)
|
|
867
|
+
@value.reshape(@value.elements)
|
|
868
|
+
.__axis_group_scan__([0], [[@codes, @k, [0]]], op)
|
|
869
|
+
.reshape(*@src_shape)
|
|
870
|
+
end
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
# Permutation: perm[slot] = the source index whose value sits at that grouped
|
|
874
|
+
# slot. This is the valid prefix of the categorical's cached sort_addr, sliced
|
|
875
|
+
# at construction (the same counting sort that lays out @grouped), so #map /
|
|
876
|
+
# #sort_addr / the *_addr reductions read it for free.
|
|
877
|
+
def perm
|
|
878
|
+
@perm
|
|
879
|
+
end
|
|
880
|
+
|
|
881
|
+
# Lay a per-cell weight array out in category-contiguous order (same layout as
|
|
882
|
+
# @grouped), so wsum / wmean can pair each group's values with its weights.
|
|
883
|
+
# Weights are coerced to float64; the same counting-sort scatter propagates
|
|
884
|
+
# the weight mask and skips excluded cells, so wg lines up with @grouped.
|
|
885
|
+
def scatter_weights (weights)
|
|
886
|
+
unless weights.elements == @codes.elements
|
|
887
|
+
raise ArgumentError,
|
|
888
|
+
"wsum/wmean: weights.elements (#{weights.elements}) != " \
|
|
889
|
+
"value.elements (#{@codes.elements})"
|
|
890
|
+
end
|
|
891
|
+
wf = weights.float64
|
|
892
|
+
wg = CArray.float64(@grouped.elements)
|
|
893
|
+
@codes.send(:__categorical_scatter__, wf.reshape(wf.elements),
|
|
894
|
+
@offsets.copy, wg, @k)
|
|
895
|
+
wg
|
|
896
|
+
end
|
|
897
|
+
|
|
898
|
+
# Map a per-category group-local index to the flat source address via the
|
|
899
|
+
# permutation (grouped slot -> source index). The min/max sits at grouped slot
|
|
900
|
+
# offsets[c] + local_index[c]; perm carries it back to the source. Empty
|
|
901
|
+
# categories (masked local index) stay masked.
|
|
902
|
+
def group_addr (local_index)
|
|
903
|
+
out = CArray.int64(@k)
|
|
904
|
+
@k.times do |c|
|
|
905
|
+
out[c] = local_index.is_masked[c] ? UNDEF
|
|
906
|
+
: perm[@offsets[c] + local_index[c]]
|
|
907
|
+
end
|
|
908
|
+
out
|
|
909
|
+
end
|
|
910
|
+
|
|
911
|
+
# Fused per-segment weighted sum + weighted mean (one C pass over the grouped
|
|
912
|
+
# copy, weights in group order). Returns [wsum, wmean]; wmean is masked where a
|
|
913
|
+
# segment has no present (value AND weight) pair. Numeric value dtypes only.
|
|
914
|
+
def kernel_weighted (wg)
|
|
915
|
+
ws = CArray.float64(@k)
|
|
916
|
+
wm = CArray.float64(@k)
|
|
917
|
+
@grouped.send(:__reduceat_wsum_wmean__, @offsets, wg, ws, wm)
|
|
918
|
+
[ws, wm]
|
|
919
|
+
end
|
|
920
|
+
|
|
921
|
+
# Per-group weighted fallback for non-numeric value dtypes (complex): delegate
|
|
922
|
+
# each group to CArray#wsum / #wmean. Empty segments take the given identity.
|
|
923
|
+
def fold_weighted (wg, empty)
|
|
924
|
+
out = CArray.float64(@k)
|
|
925
|
+
@k.times do |c|
|
|
926
|
+
lo = @offsets[c]
|
|
927
|
+
hi = (c + 1 < @k) ? @offsets[c + 1] : @grouped.elements
|
|
928
|
+
out[c] = hi > lo ? yield(@grouped[lo...hi], wg[lo...hi]) : empty
|
|
929
|
+
end
|
|
930
|
+
out
|
|
931
|
+
end
|
|
932
|
+
|
|
933
|
+
# The members of category `c` as a CArray slice of the grouped copy. An empty
|
|
934
|
+
# category (zero-width segment) yields the shared empty array — a zero-length
|
|
935
|
+
# slice cannot be taken directly, and an empty array carries the same reduction
|
|
936
|
+
# contract we want (identity for sum, UNDEF for ratios).
|
|
937
|
+
def group_slice (c)
|
|
938
|
+
lo = @offsets[c]
|
|
939
|
+
hi = (c + 1 < @k) ? @offsets[c + 1] : @grouped.elements
|
|
940
|
+
hi > lo ? @grouped[lo...hi] : @empty
|
|
941
|
+
end
|
|
942
|
+
|
|
943
|
+
# Single-pass reduceat moments (count / sum / min / max per category), computed
|
|
944
|
+
# once over the grouped copy and cached — the whole point of the eager copy is
|
|
945
|
+
# that one scatter is followed by cheap single-pass reductions with no
|
|
946
|
+
# per-segment views. Nil for a non-numeric value dtype (complex / object /
|
|
947
|
+
# bool), where the monoid reductions fall back to per_category.
|
|
948
|
+
# numeric value dtypes the C moments kernel handles (int8..float64); bool /
|
|
949
|
+
# complex / object fall back to per_category.
|
|
950
|
+
MONOID_TYPES = %i[int8 uint8 int16 uint16 int32 uint32
|
|
951
|
+
int64 uint64 float32 float64].freeze
|
|
952
|
+
|
|
953
|
+
def moments
|
|
954
|
+
return @moments if defined?(@moments)
|
|
955
|
+
@moments =
|
|
956
|
+
if MONOID_TYPES.include?(@grouped.data_type)
|
|
957
|
+
dt = @grouped.data_type
|
|
958
|
+
counts = CArray.int64(@k)
|
|
959
|
+
sums = CArray.float64(@k)
|
|
960
|
+
mins = CArray.new(dt, [@k])
|
|
961
|
+
maxs = CArray.new(dt, [@k])
|
|
962
|
+
@grouped.send(:__reduceat_moments__, @offsets, counts, sums, mins, maxs)
|
|
963
|
+
{ count: counts, sum: sums, min: mins, max: maxs }
|
|
964
|
+
end
|
|
965
|
+
end
|
|
966
|
+
|
|
967
|
+
# Single-pass fused group-local argmin / argmax (min_index / max_index),
|
|
968
|
+
# cached. Nil for a non-numeric value dtype (fall back to per_category).
|
|
969
|
+
def arg_minmax
|
|
970
|
+
return @arg_minmax if defined?(@arg_minmax)
|
|
971
|
+
@arg_minmax =
|
|
972
|
+
if MONOID_TYPES.include?(@grouped.data_type)
|
|
973
|
+
mn = CArray.int64(@k)
|
|
974
|
+
mx = CArray.int64(@k)
|
|
975
|
+
@grouped.send(:__reduceat_argminmax__, @offsets, mn, mx)
|
|
976
|
+
{ min: mn, max: mx }
|
|
977
|
+
end
|
|
978
|
+
end
|
|
979
|
+
|
|
980
|
+
# Single-pass fused per-category boolean all / any, cached. Nil unless the
|
|
981
|
+
# value dtype is boolean (fall back to per_category, which raises like
|
|
982
|
+
# CArray#all on a non-boolean).
|
|
983
|
+
def all_any
|
|
984
|
+
return @all_any if defined?(@all_any)
|
|
985
|
+
@all_any =
|
|
986
|
+
if @grouped.data_type == CA_BOOLEAN
|
|
987
|
+
a = CArray.boolean(@k)
|
|
988
|
+
o = CArray.boolean(@k)
|
|
989
|
+
@grouped.send(:__reduceat_all_any__, @offsets, a, o)
|
|
990
|
+
{ all: a, any: o }
|
|
991
|
+
end
|
|
992
|
+
end
|
|
993
|
+
|
|
994
|
+
# Build a length-k typed output by folding each category's members with the
|
|
995
|
+
# given reduction block. Fallback path (order statistics, and monoids on a
|
|
996
|
+
# non-numeric value dtype): each group is delegated to the same CArray
|
|
997
|
+
# reduction, so the per-group result matches `CArray#<reduction>` over that
|
|
998
|
+
# group's members — the mask carries the "insufficient present data" contract
|
|
999
|
+
# for free (an all-masked group reduces like an empty one; identity-bearing
|
|
1000
|
+
# reductions return their identity, ratios return UNDEF; see ext ERI).
|
|
1001
|
+
def per_category (data_type)
|
|
1002
|
+
out = CArray.new(data_type, [@k])
|
|
1003
|
+
@k.times { |c| out[c] = yield(group_slice(c)) }
|
|
1004
|
+
out
|
|
1005
|
+
end
|
|
1006
|
+
end
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
class CArray
|
|
1010
|
+
# @overload group_by_category(cat)
|
|
1011
|
+
# Returns a {CACategoricalIterator} that reduces `self` (the payload)
|
|
1012
|
+
# per category of `cat`. Requires `self.elements == cat.elements`.
|
|
1013
|
+
# @param cat [CACategorical] the classifier.
|
|
1014
|
+
# @return [CACategoricalIterator]
|
|
1015
|
+
# @raise [ArgumentError] when element counts differ.
|
|
1016
|
+
def group_by_category (cat)
|
|
1017
|
+
CACategoricalIterator.new(self, cat)
|
|
1018
|
+
end
|
|
1019
|
+
|
|
1020
|
+
# @overload group_by_run
|
|
1021
|
+
# Segments `self` into maximal runs of consecutive non-masked cells and
|
|
1022
|
+
# returns a {CACategoricalIterator} that reduces each run as one category,
|
|
1023
|
+
# ordered by position. The run boundary is the mask: a masked cell belongs
|
|
1024
|
+
# to no run and breaks any run across it. State what separates runs (the
|
|
1025
|
+
# "background") by masking before the call — e.g. `ca.mask_where(:le, 0)`
|
|
1026
|
+
# makes non-positive cells background without mutating `ca`. A series with
|
|
1027
|
+
# no present cell yields zero groups rather than raising. 1-D only.
|
|
1028
|
+
#
|
|
1029
|
+
# ```ruby
|
|
1030
|
+
# prec = CA_DOUBLE([1,2,2,2,0,0,0,2,1,2,0,0,0,3,2,3,2,1,0,0,0])
|
|
1031
|
+
# grp = prec.mask_where(:le, 0).group_by_run
|
|
1032
|
+
# grp.sum # => [7.0, 5.0, 11.0] per-run accumulation
|
|
1033
|
+
# grp.count # => [4, 3, 5] per-run length
|
|
1034
|
+
# grp.each { |members| ... } # each run as a CArray
|
|
1035
|
+
# ```
|
|
1036
|
+
#
|
|
1037
|
+
# The run categories are labelled by their 0-based run index, so
|
|
1038
|
+
# `grp.labels` is `[0, 1, ...]` in position order.
|
|
1039
|
+
#
|
|
1040
|
+
# @return [CACategoricalIterator] one category per run, in order.
|
|
1041
|
+
# @raise [RuntimeError] when `self` is not 1-D.
|
|
1042
|
+
def group_by_run
|
|
1043
|
+
raise "group_by_run: 1-D only (got #{ndim}-D)" unless ndim == 1
|
|
1044
|
+
if elements == 0
|
|
1045
|
+
code = CArray.int64(0)
|
|
1046
|
+
else
|
|
1047
|
+
present = is_not_masked
|
|
1048
|
+
edge = present & present.shift(1).not # rising edge = run start
|
|
1049
|
+
# feed cumsum via a zero-copy int8 reinterpret of the 1-byte booleans
|
|
1050
|
+
# rather than widening to int64; cumsum promotes to float64, so the
|
|
1051
|
+
# running count never overflows int8.
|
|
1052
|
+
code = edge.refer(:int8).cumsum.int64 - 1 # 0-based run index per cell
|
|
1053
|
+
code[present.not] = UNDEF # masked cells join no run
|
|
1054
|
+
end
|
|
1055
|
+
# categorize turns the dense run indices into the run categories: it derives
|
|
1056
|
+
# the label vocabulary and folds an all-masked (dry) series to zero groups
|
|
1057
|
+
# on its own, so no explicit run count is needed here. code is monotonic (a
|
|
1058
|
+
# cumsum), so categorize's first-appearance order is already run order and
|
|
1059
|
+
# sort_labels would be a no-op.
|
|
1060
|
+
group_by_category(code.categorize)
|
|
1061
|
+
end
|
|
1062
|
+
end
|