carray 3.0.1 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +824 -0
- data/README.md +5 -6
- data/carray.gemspec +1 -1
- data/ext/ca_axis_dispatch.c +258 -4
- data/ext/ca_axis_group.c +368 -184
- data/ext/ca_binop_dispatch.c +54 -32
- data/ext/ca_binop_dispatch.h +1 -0
- data/ext/ca_categorical_iterator.c +90 -41
- data/ext/ca_compare.c +8 -2
- data/ext/ca_for_buffer.h +21 -68
- data/ext/ca_group_iter.c +27 -11
- data/ext/ca_kernel_iterator.c +960 -245
- data/ext/ca_kernel_iterator.h +352 -181
- data/ext/ca_monop_dispatch.c +4 -4
- data/ext/ca_monop_dispatch.h +4 -0
- data/ext/ca_obj_array.c +194 -42
- data/ext/ca_obj_bincmp.c +152 -121
- data/ext/ca_obj_binop.c +107 -210
- data/ext/ca_obj_bitarray.c +37 -24
- data/ext/ca_obj_bitfield.c +95 -62
- data/ext/ca_obj_block.c +5 -4
- data/ext/ca_obj_byte_swap.c +40 -40
- data/ext/ca_obj_const_string.c +180 -50
- data/ext/ca_obj_face.c +428 -62
- data/ext/ca_obj_face.h +107 -47
- data/ext/ca_obj_fake.c +21 -50
- data/ext/ca_obj_farray.c +1 -0
- data/ext/ca_obj_field.c +16 -3
- data/ext/ca_obj_fixlen_string.c +39 -26
- data/ext/ca_obj_grid.c +181 -124
- data/ext/ca_obj_meld.c +235 -55
- data/ext/ca_obj_moncmp.c +61 -61
- data/ext/ca_obj_monop.c +125 -107
- data/ext/ca_obj_object.c +551 -233
- data/ext/ca_obj_record.c +51 -16
- data/ext/ca_obj_reduce.c +5 -8
- data/ext/ca_obj_refer.c +117 -33
- data/ext/ca_obj_remap.c +57 -17
- data/ext/ca_obj_repeat.c +21 -5
- data/ext/ca_obj_roll.c +115 -11
- data/ext/ca_obj_select.c +88 -75
- data/ext/ca_obj_select_axis.c +58 -60
- data/ext/ca_obj_shift.c +22 -102
- data/ext/ca_obj_stack.c +223 -123
- data/ext/ca_obj_stride.c +267 -35
- data/ext/ca_obj_string.c +25 -22
- data/ext/ca_obj_tile.c +32 -16
- data/ext/ca_obj_time.c +23 -25
- data/ext/ca_obj_timedelta.c +21 -23
- data/ext/ca_obj_transpose.c +12 -3
- data/ext/ca_obj_triop.c +110 -117
- data/ext/ca_obj_window.c +157 -29
- data/ext/ca_op_ipower.c +25 -13
- data/ext/ca_op_powi.h +57 -30
- data/ext/ca_pow5_table.h +656 -0
- data/ext/ca_rng_normal.h +42 -0
- data/ext/ca_rng_xoshiro256pp.h +105 -0
- data/ext/ca_sort_kernels.h +70 -0
- data/ext/ca_sweep_engine.c +385 -198
- data/ext/ca_sweep_engine.h +31 -23
- data/ext/ca_triop_dispatch.c +4 -1
- data/ext/ca_triop_dispatch.h +5 -2
- data/ext/caframe_csv_reader.c +463 -0
- data/ext/carray.h +90 -28
- data/ext/carray_access.c +487 -231
- data/ext/carray_address_basis.c +731 -0
- data/ext/carray_attribute.c +45 -3
- data/ext/carray_bincount.c +32 -2
- data/ext/carray_broadcast.c +49 -17
- data/ext/carray_call_cfunc.c +667 -511
- data/ext/carray_cast.c +1556 -247
- data/ext/carray_cast_func.rb +34 -21
- data/ext/carray_conversion.c +197 -73
- data/ext/carray_copy.c +84 -38
- data/ext/carray_core.c +591 -119
- data/ext/carray_count.c +148 -96
- data/ext/carray_element.c +18 -27
- data/ext/carray_factorize.c +604 -291
- data/ext/carray_generate.c +76 -40
- data/ext/carray_histogram.c +152 -94
- data/ext/carray_hold.c +2 -1
- data/ext/carray_index_classifier.c +63 -96
- data/ext/carray_internal.h +400 -1
- data/ext/carray_lazy.c +834 -79
- data/ext/carray_loop.c +88 -66
- data/ext/carray_mask.c +705 -324
- data/ext/carray_math_kernel.h +1 -0
- data/ext/carray_median_percentile.c +263 -63
- data/ext/carray_memory_view.c +254 -248
- data/ext/carray_operator.c +1093 -363
- data/ext/carray_order.c +133 -69
- data/ext/carray_partition.c +102 -56
- data/ext/carray_random.c +582 -204
- data/ext/carray_scatter.c +308 -65
- data/ext/carray_segment.c +257 -0
- data/ext/carray_slab.c +183 -70
- data/ext/carray_sort.c +235 -240
- data/ext/carray_sort_kernel.c +8 -1
- data/ext/carray_struct.c +99 -21
- data/ext/carray_test.c +167 -64
- data/ext/carray_time_text.c +1182 -0
- data/ext/carray_utils.c +363 -98
- data/ext/extconf.rb +27 -2
- data/ext/mk_call_cfunc.rb +103 -118
- data/ext/mk_pow5_table.rb +35 -0
- data/ext/mkkernel.rb +1349 -851
- data/ext/ruby_carray.c +17 -2
- data/ext/version.h +4 -4
- data/ext/version.rb +4 -2
- data/lib/carray/attribute.rb +39 -0
- data/lib/carray/autoload_carray.rb +24 -3
- data/lib/carray/autoload_method_extension.rb +18 -0
- data/lib/carray/axis_group.rb +245 -20
- data/lib/carray/basics.rb +95 -25
- data/lib/carray/bincount_nd.rb +6 -0
- data/lib/carray/block_iterator.rb +200 -54
- data/lib/carray/boolean_reduce.rb +21 -29
- data/lib/carray/categorical.rb +244 -59
- data/lib/carray/categorical_iterator.rb +391 -757
- data/lib/carray/complex.rb +36 -29
- data/lib/carray/conditional.rb +94 -7
- data/lib/carray/const_string.rb +177 -27
- data/lib/carray/construct.rb +40 -0
- data/lib/carray/data_type_extension.rb +3 -0
- data/lib/carray/data_type_limits.rb +91 -0
- data/lib/carray/fixlen_string.rb +1 -1
- data/lib/carray/frame/concat.rb +157 -6
- data/lib/carray/frame/convert.rb +13 -7
- data/lib/carray/frame/csv_parser.rb +590 -76
- data/lib/carray/frame/describe.rb +79 -0
- data/lib/carray/frame/frame.rb +205 -35
- data/lib/carray/frame/group.rb +199 -40
- data/lib/carray/frame/io.rb +452 -60
- data/lib/carray/frame/join.rb +134 -46
- data/lib/carray/frame/records.rb +54 -19
- data/lib/carray/frame/reshape.rb +448 -0
- data/lib/carray/frame/sort.rb +26 -2
- data/lib/carray/frame/verbs.rb +726 -56
- data/lib/carray/frame.rb +2 -0
- data/lib/carray/fuse_source.rb +111 -2
- data/lib/carray/fusion.rb +229 -38
- data/lib/carray/inspect.rb +71 -20
- data/lib/carray/iterator.rb +157 -2
- data/lib/carray/lazy.rb +54 -67
- data/lib/carray/mask_gap_fill.rb +66 -50
- data/lib/carray/math.rb +32 -19
- data/lib/carray/meld_reduce.rb +40 -7
- data/lib/carray/methods/align_addr.rb +8 -4
- data/lib/carray/methods/bincount.rb +2 -2
- data/lib/carray/methods/bit_string.rb +14 -3
- data/lib/carray/methods/composition.rb +16 -10
- data/lib/carray/methods/discovery_along.rb +74 -0
- data/lib/carray/methods/factorize.rb +50 -0
- data/lib/carray/methods/gather_nd.rb +13 -2
- data/lib/carray/methods/insert_block.rb +9 -14
- data/lib/carray/methods/is_in.rb +130 -6
- data/lib/carray/methods/join.rb +1 -1
- data/lib/carray/methods/locate_addr.rb +235 -21
- data/lib/carray/methods/mask_duplicates.rb +35 -1
- data/lib/carray/methods/mode.rb +18 -1
- data/lib/carray/methods/nunique.rb +22 -1
- data/lib/carray/methods/repeat.rb +91 -0
- data/lib/carray/methods/resize.rb +21 -23
- data/lib/carray/methods/snap.rb +34 -12
- data/lib/carray/methods/unique.rb +52 -3
- data/lib/carray/record_order.rb +208 -0
- data/lib/carray/rng.rb +86 -0
- data/lib/carray/runtime.rb +14 -11
- data/lib/carray/segment_iterator.rb +990 -0
- data/lib/carray/serialize.rb +197 -27
- data/lib/carray/slab_iterator.rb +109 -40
- data/lib/carray/stack.rb +10 -4
- data/lib/carray/string_operation_extension.rb +9 -4
- data/lib/carray/struct.rb +142 -51
- data/lib/carray/struct_builder.rb +110 -14
- data/lib/carray/table.rb +2 -2
- data/lib/carray/time.rb +535 -105
- data/lib/carray/window_iterator.rb +259 -82
- data/lib/carray.rb +4 -2
- data/yard-stubs/ca_obj_array.rb +4 -4
- data/yard-stubs/ca_obj_bitarray.rb +7 -0
- data/yard-stubs/ca_obj_bitfield.rb +23 -12
- data/yard-stubs/ca_obj_block.rb +2 -7
- data/yard-stubs/ca_obj_fake.rb +1 -1
- data/yard-stubs/ca_obj_field.rb +1 -1
- data/yard-stubs/ca_obj_refer.rb +3 -3
- data/yard-stubs/ca_obj_roll.rb +0 -1
- data/yard-stubs/ca_obj_stride.rb +10 -7
- data/yard-stubs/ca_obj_transpose.rb +1 -1
- data/yard-stubs/ca_obj_window.rb +11 -4
- data/yard-stubs/carray_access.rb +6 -5
- data/yard-stubs/carray_broadcast.rb +2 -2
- data/yard-stubs/carray_cast.rb +78 -43
- data/yard-stubs/carray_conversion.rb +13 -1
- data/yard-stubs/carray_copy.rb +8 -3
- data/yard-stubs/carray_core.rb +3 -83
- data/yard-stubs/carray_count.rb +16 -2
- data/yard-stubs/carray_lazy.rb +210 -1
- data/yard-stubs/carray_loop.rb +4 -1
- data/yard-stubs/carray_mask.rb +34 -10
- data/yard-stubs/carray_math.rb +1499 -4
- data/yard-stubs/carray_median_percentile.rb +26 -8
- data/yard-stubs/carray_order.rb +43 -82
- data/yard-stubs/carray_scatter.rb +10 -3
- data/yard-stubs/carray_segment.rb +72 -0
- data/yard-stubs/carray_slab.rb +11 -8
- data/yard-stubs/carray_sort.rb +50 -8
- data/yard-stubs/carray_test.rb +10 -0
- data/yard-stubs/carray_utils.rb +0 -26
- metadata +19 -19
- data/ext/ca_for_each_element.h +0 -239
- data/ext/carray_build_flags.h +0 -3
- data/ext/carray_config.h +0 -28
- data/ext/carray_kernels_bincmp.c +0 -4446
- data/ext/carray_kernels_binop.c +0 -11001
- data/ext/carray_kernels_init.c +0 -1131
- data/ext/carray_kernels_map.c +0 -3467
- data/ext/carray_kernels_moncmp.c +0 -2097
- data/ext/carray_kernels_monop.c +0 -18313
- data/ext/carray_kernels_reduce_aggregate.c +0 -25837
- data/ext/carray_kernels_reduce_boolean.c +0 -330
- data/ext/carray_kernels_reduce_cumulative.c +0 -14593
- data/ext/carray_kernels_reduce_extreme.c +0 -16948
- data/ext/carray_kernels_reduce_variance.c +0 -3910
- data/ext/carray_kernels_scan.c +0 -3693
- data/ext/carray_kernels_search.c +0 -32138
- data/ext/carray_kernels_sort.c +0 -10626
- data/ext/carray_kernels_triop.c +0 -1392
data/lib/carray/methods/join.rb
CHANGED
|
@@ -66,7 +66,7 @@ class CArray
|
|
|
66
66
|
t = transpose(*order)
|
|
67
67
|
end
|
|
68
68
|
inner = t.shape[-1]
|
|
69
|
-
outer_n = t.
|
|
69
|
+
outer_n = t.shape[0...-1].inject(1, :*) # 1 when ndim == 1
|
|
70
70
|
flat = t.reshape(outer_n, inner)
|
|
71
71
|
|
|
72
72
|
strings = Array.new(outer_n) { |i| flat[i, nil].to_a.join(sep_str) }
|
|
@@ -1,52 +1,266 @@
|
|
|
1
|
+
require "carray/methods/is_in" # fixlen_widened_with, values_in_face, values_for_lookup
|
|
2
|
+
|
|
1
3
|
class CArray
|
|
2
4
|
|
|
3
|
-
#
|
|
4
|
-
# in yard-stubs/carray_order.rb (grouped with the search family).
|
|
5
|
+
# @!group Sorting and searching
|
|
5
6
|
|
|
7
|
+
# @overload locate_addr(ref)
|
|
8
|
+
# Returns, for each element of `self`, the flat address into `ref`
|
|
9
|
+
# where the value first occurs, or `UNDEF` where it is not present.
|
|
10
|
+
# Builds a value-to-first-address map from `ref` in one pass (an
|
|
11
|
+
# open-addressing hash, the same substrate as {#unique} /
|
|
12
|
+
# {#value_counts}), then probes each element of `self`; no sort,
|
|
13
|
+
# peak memory `O(distinct ref values)`.
|
|
14
|
+
#
|
|
15
|
+
# The return is `ref`'s flat address (0 to `ref.elements - 1`), so a
|
|
16
|
+
# multi-dimensional `ref` still yields a `self`-shaped result of flat
|
|
17
|
+
# addresses; downstream reads (`ref[addr]`, `model_var[addr]`, ...)
|
|
18
|
+
# apply it as a flat gather.
|
|
19
|
+
#
|
|
20
|
+
# Works on numeric, `CA_OBJECT`, and `CA_FIXLEN` values, matching the
|
|
21
|
+
# value-hash discovery family: numeric follows `==` with all NaN
|
|
22
|
+
# collapsed to one value and `-0.0 == +0.0`; object follows Ruby
|
|
23
|
+
# `hash` / `eql?` with Float NaN collapsed; fixlen follows byte
|
|
24
|
+
# equality. When `ref` holds duplicate values the returned address is
|
|
25
|
+
# the earliest (appearance-order) occurrence. Masked cells of `ref` do
|
|
26
|
+
# not enter the map but still occupy their flat address; masked cells
|
|
27
|
+
# of `self` are `UNDEF` in the result.
|
|
28
|
+
#
|
|
29
|
+
# `self` and `ref` are compared at their common type, negotiated by
|
|
30
|
+
# `CArray.result_type` — a fractional query against an integer `ref`
|
|
31
|
+
# is compared at the promoted type rather than truncated, so `1.5` no
|
|
32
|
+
# longer matches `1`, and integers of mixed sign are matched by value
|
|
33
|
+
# (`-1` never matches `255`). Cross-family input (numeric against
|
|
34
|
+
# fixlen) raises.
|
|
35
|
+
#
|
|
36
|
+
# A time key (`CATime`) reads values that are not time arrays -- Time,
|
|
37
|
+
# Date or String objects, in an Array, an `:object` array or a string
|
|
38
|
+
# array -- as instants, on either side of the lookup, as {#is_in} does.
|
|
39
|
+
#
|
|
40
|
+
# Typical use is time-axis lookup: compute the address once against a
|
|
41
|
+
# reference axis, then reuse it to gather from many `ref`-shaped
|
|
42
|
+
# variables without repeating the lookup.
|
|
43
|
+
#
|
|
44
|
+
# @param ref [CArray, Array, Range] reference values to match against;
|
|
45
|
+
# any shape (used as flat). An Array or Range is coerced with `to_ca`
|
|
46
|
+
# and so lands in `:object`, where numeric matching follows Ruby
|
|
47
|
+
# `eql?` (`2.0` does not match `2`); pass a CArray to match in a
|
|
48
|
+
# numeric lane.
|
|
49
|
+
# @return [CArray] `:int64` flat addresses into `ref`, same shape as
|
|
50
|
+
# `self`; unmatched cells are masked.
|
|
51
|
+
# @raise [CArray::DataTypeError] when `self` and `ref` have no common data type.
|
|
6
52
|
def locate_addr (ref)
|
|
53
|
+
# A time key (any Face that reconciles operands) against values that do
|
|
54
|
+
# not -- Time, Date or String objects, a string column -- reads them as
|
|
55
|
+
# values of its kind, whichever side they are on.
|
|
56
|
+
if ref.is_a?(CArray) && ref.class == self.class &&
|
|
57
|
+
respond_to?(:in_common_unit_with) && ref.unit != unit
|
|
58
|
+
a, b = in_common_unit_with(ref)
|
|
59
|
+
return a.locate_addr(b)
|
|
60
|
+
end
|
|
61
|
+
if respond_to?(:to_comparable) && !ref.respond_to?(:to_comparable)
|
|
62
|
+
ref = values_in_face(self, ref)
|
|
63
|
+
return CArray.int64(*shape).fill(UNDEF) unless ref
|
|
64
|
+
elsif ref.respond_to?(:to_comparable) && !respond_to?(:to_comparable)
|
|
65
|
+
q = values_in_face(ref, self)
|
|
66
|
+
return CArray.int64(*shape).fill(UNDEF) unless q
|
|
67
|
+
return q.locate_addr(ref)
|
|
68
|
+
end
|
|
69
|
+
unless ref.is_a?(CArray) || ref.respond_to?(:to_ca)
|
|
70
|
+
raise TypeError, "locate_addr: the reference is an array of values (got #{ref.class})"
|
|
71
|
+
end
|
|
7
72
|
ref = ref.to_ca unless ref.is_a?(CArray)
|
|
73
|
+
ref = values_for_lookup(ref)
|
|
74
|
+
if data_type == CA_FIXLEN && ref.data_type == CA_FIXLEN && bytes != ref.bytes
|
|
75
|
+
a, b = fixlen_widened_with(ref)
|
|
76
|
+
return a.locate_addr(b) unless a.equal?(self) && b.equal?(ref)
|
|
77
|
+
end
|
|
8
78
|
# Put self and ref in a common lane via the single-source promotion rule
|
|
9
79
|
# (CArray.result_type), so a fractional query against an int ref is compared
|
|
10
80
|
# at the promoted type instead of truncating (1.5 no longer matches 1).
|
|
11
81
|
# to_type is elementwise and order-preserving, so the addresses stay valid
|
|
12
|
-
# indices into ref. result_type raises for cross-family input
|
|
13
|
-
|
|
82
|
+
# indices into ref. result_type raises for cross-family input; an integer
|
|
83
|
+
# pair of mixed sign is matched by value (see value_match_type).
|
|
84
|
+
t = value_match_type(CArray.result_type(self, ref), ref.data_type)
|
|
14
85
|
q = (data_type == t) ? self : to_type(t)
|
|
15
86
|
r = (ref.data_type == t) ? ref : ref.to_type(t)
|
|
16
87
|
q.send(:__locate_addr__, r)
|
|
17
88
|
end
|
|
18
89
|
|
|
90
|
+
# @overload locate_nearest_addr(ref, direction: :round, tolerance: nil)
|
|
91
|
+
# Returns, for each element of `self`, the flat address into `ref` of
|
|
92
|
+
# the nearest reference value. Continuous sibling of {#locate_addr}:
|
|
93
|
+
# each cell is placed among the sorted reference values by comparing
|
|
94
|
+
# it with them, so an integer key is exact at any size and an infinite
|
|
95
|
+
# one keeps its place. `:round` takes the nearer neighbour, the upper
|
|
96
|
+
# one on a tie.
|
|
97
|
+
#
|
|
98
|
+
# Outside `ref`'s span the direction still decides: `:floor` matches
|
|
99
|
+
# a cell after the last value to that last value, `:ceil` matches a
|
|
100
|
+
# cell before the first value to that first value, and `:round`
|
|
101
|
+
# matches either to the nearer end. The other side has no match
|
|
102
|
+
# (`:floor` before the first value, `:ceil` after the last) and is
|
|
103
|
+
# masked. Use `tolerance:` to refuse matches that are too far away.
|
|
104
|
+
#
|
|
105
|
+
# `tolerance:` (default `nil`) sets a maximum accepted absolute
|
|
106
|
+
# distance between `self[i]` and its matched `ref` value. When
|
|
107
|
+
# `|ref[addr] - self[i]| > tolerance`, the result cell is masked. Use
|
|
108
|
+
# for accuracy-controlled matching (e.g. "an observation snaps to a
|
|
109
|
+
# time step only if within N seconds").
|
|
110
|
+
#
|
|
111
|
+
# `ref` need not be given in ascending order: it is sorted internally
|
|
112
|
+
# and the returned addresses are mapped back to positions in `ref` as
|
|
113
|
+
# passed. A masked or NaN `ref` value matches nothing. Two time keys
|
|
114
|
+
# of different units are compared in the finer of the two.
|
|
115
|
+
# With one value present in `ref`, that value is the match for
|
|
116
|
+
# every cell the direction allows (`:floor` at or after it, `:ceil`
|
|
117
|
+
# at or before it, `:round` all). With none, every cell is masked;
|
|
118
|
+
# an empty `self` gives an empty result. A masked or NaN cell of
|
|
119
|
+
# `self` is always masked.
|
|
120
|
+
#
|
|
121
|
+
# @param ref [CArray] 1-D reference grid to match against.
|
|
122
|
+
# @param direction [Symbol] `:round`, `:floor`, or `:ceil` — rounding
|
|
123
|
+
# applied to the fractional position.
|
|
124
|
+
# @param tolerance [Numeric, String, CATimedelta::Element, nil] maximum
|
|
125
|
+
# accepted `|self - ref|` distance; cells beyond this are masked.
|
|
126
|
+
# For a time key it is a duration: a String such as `"10 minutes"` or
|
|
127
|
+
# a CATimedelta value (a bare number is refused, since its length
|
|
128
|
+
# would depend on the key's unit). `nil` disables the check.
|
|
129
|
+
# @return [CArray] `:int64` flat addresses into `ref`, same shape as
|
|
130
|
+
# `self`; cells with no match in the direction and beyond-tolerance
|
|
131
|
+
# cells are masked.
|
|
132
|
+
# @raise [ArgumentError] when `direction` is not one of the accepted
|
|
133
|
+
# symbols, or `tolerance` is not a duration for a time key.
|
|
19
134
|
def locate_nearest_addr (ref, direction: :round, tolerance: nil)
|
|
20
135
|
unless [:round, :floor, :ceil].include?(direction)
|
|
21
136
|
raise ArgumentError,
|
|
22
137
|
"locate_nearest_addr: direction must be :round / :floor / " \
|
|
23
138
|
":ceil (got #{direction.inspect})"
|
|
24
139
|
end
|
|
140
|
+
return CArray.int64(*shape) if elements == 0
|
|
141
|
+
# A masked reference value matches nothing, as in locate_addr: sorting
|
|
142
|
+
# puts the masked ones last, and only the others are searched. A NaN
|
|
143
|
+
# one has no place among the others either; it sorts after them.
|
|
25
144
|
ri = ref.sort_addr
|
|
145
|
+
ri = ri[0...ref.count_not_masked] if ref.has_mask?
|
|
146
|
+
if ref.float? && ri.elements > 0
|
|
147
|
+
ri = ri[0...(ri.elements - ref[ri].is_nan.count(true))]
|
|
148
|
+
end
|
|
149
|
+
if ri.elements == 0
|
|
150
|
+
out = CArray.int64(*shape)
|
|
151
|
+
out[] = UNDEF
|
|
152
|
+
return out
|
|
153
|
+
end
|
|
26
154
|
rs = ref[ri]
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
fill = CArray.float64(*shape)
|
|
35
|
-
fill[] = sec.nil? ? UNDEF : sec
|
|
36
|
-
sec = fill
|
|
37
|
-
end
|
|
38
|
-
masked = sec.mask_invalid
|
|
155
|
+
n = rs.elements
|
|
156
|
+
rk, qk = nearest_keys(rs, self)
|
|
157
|
+
# The position of each cell among the sorted references is counted, not
|
|
158
|
+
# interpolated: :floor takes the last reference at or below it, :ceil the
|
|
159
|
+
# first at or above it.
|
|
160
|
+
# Counting keeps an integer key exact past 2**53 and an infinite one in
|
|
161
|
+
# its place, where an interpolated position would not.
|
|
39
162
|
si = case direction
|
|
40
|
-
when :
|
|
41
|
-
when :
|
|
42
|
-
when :
|
|
43
|
-
end
|
|
163
|
+
when :floor then nearest_count_below(rk, qk, true) - 1
|
|
164
|
+
when :ceil then nearest_count_below(rk, qk, false)
|
|
165
|
+
when :round then nearest_round(rk, qk, nearest_count_below(rk, qk, false))
|
|
166
|
+
end
|
|
167
|
+
si[si.lt(0) | si.ge(n)] = UNDEF
|
|
168
|
+
si[qk.is_nan.strip_mask(false)] = UNDEF if qk.float?
|
|
169
|
+
si = si.reshape(*shape)
|
|
170
|
+
si[is_masked] = UNDEF if has_mask?
|
|
44
171
|
idx = ri.project(si)
|
|
45
172
|
if tolerance
|
|
46
173
|
dist = (ref.project(idx) - self).abs
|
|
47
|
-
idx[dist > tolerance] = UNDEF
|
|
174
|
+
idx[dist > nearest_tolerance(tolerance, dist)] = UNDEF
|
|
48
175
|
end
|
|
49
176
|
idx
|
|
50
177
|
end
|
|
51
178
|
|
|
179
|
+
# The keys locate_nearest_addr orders by: the storage of the sorted
|
|
180
|
+
# reference and of the query, in one numeric type. A Face key is first
|
|
181
|
+
# brought into the reference's space (two time keys of different units
|
|
182
|
+
# into the finer of the two, which holds both exactly); plain keys meet at
|
|
183
|
+
# the type CArray.result_type gives, an integer pair of mixed sign at one
|
|
184
|
+
# that holds both.
|
|
185
|
+
private def nearest_keys(rs, q)
|
|
186
|
+
if rs.face? || q.face?
|
|
187
|
+
if rs.respond_to?(:to_unit) && q.class == rs.class && q.unit != rs.unit
|
|
188
|
+
u = CATimeUnitAlgebra.finer(rs.unit, q.unit)
|
|
189
|
+
rs = rs.to_unit(u)
|
|
190
|
+
q = q.to_unit(u)
|
|
191
|
+
end
|
|
192
|
+
q = rs.__send__(:__reconcile_operand__, q, "locate_nearest_addr")
|
|
193
|
+
rs = rs.__send__(:__face_storage__)
|
|
194
|
+
end
|
|
195
|
+
t = value_match_type(CArray.result_type(q, rs), rs.data_type)
|
|
196
|
+
rk = (rs.data_type == t) ? rs : rs.to_type(t)
|
|
197
|
+
qk = (q.data_type == t) ? q : q.to_type(t)
|
|
198
|
+
[rk, qk.flatten]
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# For each query key, how many of the sorted reference keys are below it
|
|
202
|
+
# (inclusive: at or below). A numeric key is binary-searched in C; an
|
|
203
|
+
# object key is sorted together with the references, a tie putting the
|
|
204
|
+
# reference first when it counts, and counted against the references met
|
|
205
|
+
# before it.
|
|
206
|
+
private def nearest_count_below(rk, qk, inclusive)
|
|
207
|
+
return rk.send(:__count_below__, qk, inclusive) unless rk.data_type == CA_OBJECT
|
|
208
|
+
n, m = rk.elements, qk.elements
|
|
209
|
+
keys = CArray.concatenate([rk, qk])
|
|
210
|
+
tag = CArray.int8(n + m)
|
|
211
|
+
tag[0...n] = inclusive ? 0 : 1
|
|
212
|
+
tag[n...n+m] = inclusive ? 1 : 0
|
|
213
|
+
perm = CArray.sort_addr(keys, tag)
|
|
214
|
+
seen = perm.lt(n).cumsum.int64
|
|
215
|
+
query = perm.ge(n)
|
|
216
|
+
count = CArray.int64(m)
|
|
217
|
+
count[perm[query] - n] = seen[query]
|
|
218
|
+
count
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# :round -- an exact match takes its first reference; otherwise the
|
|
222
|
+
# nearer of the two neighbours, the upper one on a tie. +lt+ counts the
|
|
223
|
+
# references below each key, so the upper neighbour is at lt and an exact
|
|
224
|
+
# match is the reference there.
|
|
225
|
+
private def nearest_round(rk, qk, lt)
|
|
226
|
+
lo = rk.project(lt - 1)
|
|
227
|
+
hi = rk.project(lt)
|
|
228
|
+
upper = (hi - qk).le(qk - lo).strip_mask(false)
|
|
229
|
+
upper[lo.is_masked] = true if lo.has_mask?
|
|
230
|
+
upper[hi.is_masked] = false if hi.has_mask?
|
|
231
|
+
upper[hi.eq(qk).strip_mask(false)] = true
|
|
232
|
+
si = lt - 1
|
|
233
|
+
si[upper] = lt[upper]
|
|
234
|
+
si
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# The tolerance in the units of the distance. A time key measures its
|
|
238
|
+
# distance as a CATimedelta, so its tolerance is a duration: a String in
|
|
239
|
+
# the spelling CArray.time takes for a unit ("10 minutes"), or a
|
|
240
|
+
# CATimedelta value. A bare number would mean a different length in each
|
|
241
|
+
# unit, so it is refused there; any other key takes a number.
|
|
242
|
+
private def nearest_tolerance(tolerance, dist)
|
|
243
|
+
unless dist.is_a?(CATimedelta)
|
|
244
|
+
if tolerance.is_a?(String)
|
|
245
|
+
raise ArgumentError,
|
|
246
|
+
"tolerance: #{tolerance.inspect} is a duration, but the key is " \
|
|
247
|
+
"#{data_type_name}; give a number"
|
|
248
|
+
end
|
|
249
|
+
return tolerance
|
|
250
|
+
end
|
|
251
|
+
case tolerance
|
|
252
|
+
when String
|
|
253
|
+
res = CATime::Resolution.parse(tolerance)
|
|
254
|
+
CATimedelta::Element.new(res.count, res.base)
|
|
255
|
+
when CATimedelta::Element
|
|
256
|
+
tolerance
|
|
257
|
+
else
|
|
258
|
+
raise ArgumentError,
|
|
259
|
+
"tolerance: a time key needs a duration such as \"10 minutes\" " \
|
|
260
|
+
"or a CATimedelta value (got #{tolerance.inspect})"
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# @!endgroup
|
|
265
|
+
|
|
52
266
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require "carray/methods/discovery_along"
|
|
2
|
+
|
|
1
3
|
class CArray
|
|
2
4
|
|
|
3
5
|
# @overload mask_duplicates(axis: nil)
|
|
@@ -23,7 +25,39 @@ class CArray
|
|
|
23
25
|
# `nil` uses flatten order.
|
|
24
26
|
# @return [CArray] shape-preserving copy of `self` with
|
|
25
27
|
# duplicates masked.
|
|
26
|
-
|
|
28
|
+
# @overload mask_duplicates(along: k)
|
|
29
|
+
# Returns a shape-preserving copy of `self` with every cell of a
|
|
30
|
+
# duplicated **sub-array** masked, comparing whole sub-arrays rather
|
|
31
|
+
# than cells: `along: k` names the axis whose index enumerates them,
|
|
32
|
+
# so `z.mask_duplicates(along: 0)` masks each repeated row of a 2-D
|
|
33
|
+
# array and keeps the first occurrence.
|
|
34
|
+
#
|
|
35
|
+
# Note the contrast with `axis:`, which names the axis a *fiber runs
|
|
36
|
+
# along* and marks repeated values inside each fiber. The two cannot
|
|
37
|
+
# be given together.
|
|
38
|
+
#
|
|
39
|
+
# A sub-array holding a masked cell does not participate: it is
|
|
40
|
+
# neither judged a duplicate nor able to make a later one, and its
|
|
41
|
+
# cells keep the mask they came with. `object` arrays are refused,
|
|
42
|
+
# because their cells hold Ruby references.
|
|
43
|
+
#
|
|
44
|
+
# @param along [Integer] axis whose index enumerates the sub-arrays.
|
|
45
|
+
# @return [CArray] shape-preserving copy of `self` with duplicated
|
|
46
|
+
# sub-arrays masked.
|
|
47
|
+
def mask_duplicates (axis: nil, along: nil)
|
|
48
|
+
reject_axis_with_along(axis, along, "mask_duplicates")
|
|
49
|
+
if along
|
|
50
|
+
keys = fibers_as_cells(along, "mask_duplicates")
|
|
51
|
+
# A sub-array that did not participate (it held a masked cell) is
|
|
52
|
+
# masked in `keys` already; only the ones that did and repeated
|
|
53
|
+
# are duplicates here.
|
|
54
|
+
repeated = keys.mask_duplicates.is_masked & keys.is_not_masked
|
|
55
|
+
args = [:_] * ndim
|
|
56
|
+
args[normalize_axis(along, "mask_duplicates")] = nil
|
|
57
|
+
spread = CArray.boolean(*shape)
|
|
58
|
+
spread[] = repeated[*args]
|
|
59
|
+
return mask_where(spread)
|
|
60
|
+
end
|
|
27
61
|
dup =
|
|
28
62
|
if axis.nil?
|
|
29
63
|
# One seen-set over the flattened array, then restore shape.
|
data/lib/carray/methods/mode.rb
CHANGED
|
@@ -65,12 +65,25 @@ class CArray
|
|
|
65
65
|
# Only meaningful for discrete / binned data: raw float is almost all
|
|
66
66
|
# unique, so every value is modal and the per-axis Array grows to the
|
|
67
67
|
# fiber length. Bin first. Same NaN / mask semantics as {#is_mode}.
|
|
68
|
+
# A complex array raises {CArray::DataTypeError}, since the modes
|
|
69
|
+
# are listed in ascending order; {#is_mode} marks its modal cells.
|
|
68
70
|
#
|
|
69
71
|
# @param axis [Integer, nil] axis to take the mode along; `nil` uses
|
|
70
72
|
# the whole array.
|
|
71
73
|
# @return [CArray, Array<CArray>] 1-D CArray (flat) or an Array of
|
|
72
74
|
# reduced CArrays, one per mode rank (per-axis).
|
|
73
75
|
def mode (axis: nil)
|
|
76
|
+
# An error from the methods mode is made of (sort, ...) names mode.
|
|
77
|
+
CArray.__answer_as__(:mode) { mode_answered(axis) }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private def mode_answered (axis)
|
|
81
|
+
if complex?
|
|
82
|
+
raise CArray::DataTypeError,
|
|
83
|
+
"mode: the modes are listed in ascending order, and a " \
|
|
84
|
+
"#{data_type_name} value has no order (complex numbers are not " \
|
|
85
|
+
"ordered); is_mode marks the cells that hold a mode"
|
|
86
|
+
end
|
|
74
87
|
return __mode_flat if axis.nil?
|
|
75
88
|
k = normalize_axis(axis, "mode")
|
|
76
89
|
|
|
@@ -120,7 +133,11 @@ class CArray
|
|
|
120
133
|
def __mode_flat
|
|
121
134
|
values, counts = value_counts
|
|
122
135
|
return values if values.elements == 0
|
|
123
|
-
values[counts.eq(counts.max)]
|
|
136
|
+
modes = values[counts.eq(counts.max)]
|
|
137
|
+
# Records are sorted only by the members their struct declares in
|
|
138
|
+
# order_by:; without one the modes stay in first-appearance order.
|
|
139
|
+
return modes if is_a?(CARecord) && data_class.order_by.nil?
|
|
140
|
+
modes.sort
|
|
124
141
|
end
|
|
125
142
|
|
|
126
143
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require "carray/methods/discovery_along"
|
|
2
|
+
|
|
1
3
|
class CArray
|
|
2
4
|
|
|
3
5
|
# @overload nunique(axis: nil, keep_axis: false)
|
|
@@ -29,7 +31,26 @@ class CArray
|
|
|
29
31
|
# axis as a length-1 axis instead of dropping it.
|
|
30
32
|
# @return [Integer, CArray] Integer for `axis: nil`, otherwise a
|
|
31
33
|
# reduced `CA_INT64` CArray.
|
|
32
|
-
|
|
34
|
+
# @overload nunique(along: k)
|
|
35
|
+
# Returns how many distinct **sub-arrays** `self` holds, comparing
|
|
36
|
+
# whole sub-arrays rather than cells: `along: k` names the axis
|
|
37
|
+
# whose index enumerates them, so `z.nunique(along: 0)` counts the
|
|
38
|
+
# distinct rows of a 2-D array. This is `len(np.unique(z, axis=k))`.
|
|
39
|
+
#
|
|
40
|
+
# Note the contrast with `axis:`, which names the axis a *fiber runs
|
|
41
|
+
# along* and counts the distinct values inside each fiber -- one
|
|
42
|
+
# count per fiber, not one count for the array. The two cannot be
|
|
43
|
+
# given together.
|
|
44
|
+
#
|
|
45
|
+
# Sub-arrays holding a masked cell do not participate and are not
|
|
46
|
+
# counted. `object` arrays are refused, because their cells hold
|
|
47
|
+
# Ruby references.
|
|
48
|
+
#
|
|
49
|
+
# @param along [Integer] axis whose index enumerates the sub-arrays.
|
|
50
|
+
# @return [Integer] the number of distinct sub-arrays.
|
|
51
|
+
def nunique (axis: nil, keep_axis: false, along: nil)
|
|
52
|
+
reject_axis_with_along(axis, along, "nunique")
|
|
53
|
+
return fibers_as_cells(along, "nunique").nunique if along
|
|
33
54
|
# Per-fiber single-pass seen-set hash (C __nunique__), one lane per data type
|
|
34
55
|
# family (numeric widen / NaN collapse, object rb_hash + rb_eql, fixlen
|
|
35
56
|
# byte-hash + memcmp). Masked cells are skipped; the accumulator is a no-op
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
class CArray
|
|
2
|
+
|
|
3
|
+
# @overload repeat(count, axis: nil)
|
|
4
|
+
# @overload repeat(counts, axis: nil)
|
|
5
|
+
# Returns each element (or each sub-array) of `self` laid down
|
|
6
|
+
# `count` times in place, one after another.
|
|
7
|
+
#
|
|
8
|
+
# With an Integer, every element is repeated the same number of
|
|
9
|
+
# times. With an array of counts -- one per element, or one per
|
|
10
|
+
# sub-array when `axis:` is given -- each is repeated its own number
|
|
11
|
+
# of times, and a count of 0 drops that one:
|
|
12
|
+
#
|
|
13
|
+
# v = CA_INT([10, 20, 30])
|
|
14
|
+
# v.repeat(2) # => [ 10, 10, 20, 20, 30, 30 ]
|
|
15
|
+
# v.repeat([3, 1, 2]) # => [ 10, 10, 10, 20, 30, 30 ]
|
|
16
|
+
# v.repeat([2, 0, 1]) # => [ 10, 10, 30 ]
|
|
17
|
+
#
|
|
18
|
+
# This is `np.repeat`, and it is not {#tile}: `repeat` puts the
|
|
19
|
+
# copies of one element next to each other, `tile` lays the whole
|
|
20
|
+
# array down again -- `v.tile(2)` is `[10, 20, 30, 10, 20, 30]`.
|
|
21
|
+
#
|
|
22
|
+
# `axis: k` repeats the sub-arrays enumerated by axis `k` rather
|
|
23
|
+
# than the cells, so a 2-D array repeats whole rows and the result
|
|
24
|
+
# keeps its shape apart from that axis:
|
|
25
|
+
#
|
|
26
|
+
# t = CA_INT([[0, 1], [2, 3], [4, 5]])
|
|
27
|
+
# t.repeat([2, 1, 1], axis: 0)
|
|
28
|
+
# # => [ [ 0, 1 ],
|
|
29
|
+
# # [ 0, 1 ],
|
|
30
|
+
# # [ 2, 3 ],
|
|
31
|
+
# # [ 4, 5 ] ]
|
|
32
|
+
#
|
|
33
|
+
# Without `axis:` a multi-dimensional receiver is taken in flatten
|
|
34
|
+
# (row-major) order and the result is 1-D, as `np.repeat` does.
|
|
35
|
+
#
|
|
36
|
+
# The result is a **view** of `self` -- the same element named as
|
|
37
|
+
# many times as it was repeated -- so nothing is copied and writing
|
|
38
|
+
# through it reaches `self`, at every position that names the cell
|
|
39
|
+
# written. Take a `copy` when that is not wanted.
|
|
40
|
+
#
|
|
41
|
+
# Counts must be non-negative, and there must be exactly as many as
|
|
42
|
+
# there are elements (or sub-arrays along `axis:`). A masked count
|
|
43
|
+
# is refused: it names no number of repetitions.
|
|
44
|
+
#
|
|
45
|
+
# @param count [Integer, CArray, Array] how many times to repeat --
|
|
46
|
+
# one number for all, or one per element / sub-array.
|
|
47
|
+
# @param axis [Integer, nil] axis whose index enumerates the
|
|
48
|
+
# sub-arrays to repeat; `nil` repeats cells in flatten order.
|
|
49
|
+
# @return [CArray] the repeated elements, as a view of `self`.
|
|
50
|
+
def repeat (count, axis: nil)
|
|
51
|
+
along = axis.nil? ? nil : normalize_axis(axis, "repeat")
|
|
52
|
+
width = along ? shape[along] : elements
|
|
53
|
+
|
|
54
|
+
counts =
|
|
55
|
+
case count
|
|
56
|
+
when Integer
|
|
57
|
+
if count < 0
|
|
58
|
+
raise ArgumentError, "repeat: count must not be negative (got #{count})"
|
|
59
|
+
end
|
|
60
|
+
CArray.int64(width).fill(count)
|
|
61
|
+
else
|
|
62
|
+
given = count.to_ca.int64
|
|
63
|
+
unless given.elements == width
|
|
64
|
+
raise ArgumentError,
|
|
65
|
+
"repeat: #{given.elements} counts for #{width} " \
|
|
66
|
+
"#{along ? "sub-arrays along axis #{along}" : 'elements'}"
|
|
67
|
+
end
|
|
68
|
+
if given.has_mask? and given.count_masked > 0
|
|
69
|
+
raise ArgumentError,
|
|
70
|
+
"repeat: a masked count names no number of repetitions"
|
|
71
|
+
end
|
|
72
|
+
if given.lt(0).any
|
|
73
|
+
raise ArgumentError, "repeat: counts must not be negative"
|
|
74
|
+
end
|
|
75
|
+
given.flatten
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
source = along ? self : (ndim > 1 ? flatten : self)
|
|
79
|
+
source[*gather_args(CArray.segment_index(lengths: counts), along)]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# One index argument per axis: the gathered index on the axis being
|
|
83
|
+
# repeated, whole axes everywhere else.
|
|
84
|
+
private def gather_args (index, along)
|
|
85
|
+
return [index] unless along
|
|
86
|
+
args = [nil] * ndim
|
|
87
|
+
args[along] = index
|
|
88
|
+
args
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
end
|
|
@@ -4,8 +4,8 @@ class CArray
|
|
|
4
4
|
# Returns `self` resized to `newdim`. The original data is
|
|
5
5
|
# placed at offset 0 (positive size) or right-aligned (negative
|
|
6
6
|
# size); the new area outside the original region is filled
|
|
7
|
-
# with `fill_value`, or
|
|
8
|
-
#
|
|
7
|
+
# with `fill_value`, or left as zero storage when it is not given
|
|
8
|
+
# (0 for a numeric array). Pass `UNDEF` to mask the new area.
|
|
9
9
|
#
|
|
10
10
|
# Each entry of `newdim` is: `nil` to keep the current
|
|
11
11
|
# `shape[i]`, a positive Integer for a new size with the
|
|
@@ -13,15 +13,19 @@ class CArray
|
|
|
13
13
|
# `d` for a new size `|d|` with the original right-aligned.
|
|
14
14
|
#
|
|
15
15
|
# Works on Face arrays (CARecord / CATime / ...) and
|
|
16
|
-
# plain fixlen: the
|
|
17
|
-
#
|
|
16
|
+
# plain fixlen: the result is the same Face, and `fill_value` is a
|
|
17
|
+
# value of it (a Time for a time array).
|
|
18
18
|
#
|
|
19
19
|
# @param newdim [Array<Integer, nil>] new shape spec, one entry
|
|
20
20
|
# per axis.
|
|
21
|
-
# @param fill_value [Object] value for the extended area
|
|
21
|
+
# @param fill_value [Object] value for the extended area (zero
|
|
22
|
+
# storage when not given).
|
|
22
23
|
# @return [CArray] resized copy.
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
NO_FILL_VALUE = ::Object.new.freeze
|
|
25
|
+
private_constant :NO_FILL_VALUE
|
|
26
|
+
|
|
27
|
+
def resize (*newdim, fill_value: NO_FILL_VALUE)
|
|
28
|
+
raise ArgumentError, "ndim mismatch" if newdim.size != ndim
|
|
25
29
|
offset = Array.new(ndim, 0)
|
|
26
30
|
newdim = newdim.each_with_index.map do |d, i|
|
|
27
31
|
case d
|
|
@@ -32,24 +36,18 @@ class CArray
|
|
|
32
36
|
offset[i] = size - shape[i] if d < 0
|
|
33
37
|
size
|
|
34
38
|
else
|
|
35
|
-
raise "invalid dimension size"
|
|
39
|
+
raise ArgumentError, "invalid dimension size"
|
|
36
40
|
end
|
|
37
41
|
end
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if dt != :fixlen || fill_value.equal?(UNDEF) || fill_value.is_a?(String)
|
|
48
|
-
out[] = fill_value
|
|
49
|
-
end
|
|
50
|
-
out.mask.paste(offset, src.false) if out.has_mask?
|
|
51
|
-
out.paste(offset, src)
|
|
52
|
-
out = out.face_lift(face_parent) if face_parent
|
|
42
|
+
# The original cells are copied as storage, so a read-only Face (a
|
|
43
|
+
# const string, a categorical) resizes too. The new area starts as zero
|
|
44
|
+
# storage; a given fill_value is written as a value of the array, which a
|
|
45
|
+
# read-only Face refuses.
|
|
46
|
+
out = __blank_like__(newdim)
|
|
47
|
+
out_storage = out.__send__(:__face_storage__)
|
|
48
|
+
out[] = fill_value unless fill_value.equal?(NO_FILL_VALUE)
|
|
49
|
+
out_storage.mask.paste(offset, self.false) if out_storage.has_mask?
|
|
50
|
+
out_storage.paste(offset, __face_storage__)
|
|
53
51
|
out
|
|
54
52
|
end
|
|
55
53
|
|