carray 3.0.1 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +824 -0
  3. data/README.md +5 -6
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +258 -4
  6. data/ext/ca_axis_group.c +368 -184
  7. data/ext/ca_binop_dispatch.c +54 -32
  8. data/ext/ca_binop_dispatch.h +1 -0
  9. data/ext/ca_categorical_iterator.c +90 -41
  10. data/ext/ca_compare.c +8 -2
  11. data/ext/ca_for_buffer.h +21 -68
  12. data/ext/ca_group_iter.c +27 -11
  13. data/ext/ca_kernel_iterator.c +960 -245
  14. data/ext/ca_kernel_iterator.h +352 -181
  15. data/ext/ca_monop_dispatch.c +4 -4
  16. data/ext/ca_monop_dispatch.h +4 -0
  17. data/ext/ca_obj_array.c +194 -42
  18. data/ext/ca_obj_bincmp.c +152 -121
  19. data/ext/ca_obj_binop.c +107 -210
  20. data/ext/ca_obj_bitarray.c +37 -24
  21. data/ext/ca_obj_bitfield.c +95 -62
  22. data/ext/ca_obj_block.c +5 -4
  23. data/ext/ca_obj_byte_swap.c +40 -40
  24. data/ext/ca_obj_const_string.c +180 -50
  25. data/ext/ca_obj_face.c +428 -62
  26. data/ext/ca_obj_face.h +107 -47
  27. data/ext/ca_obj_fake.c +21 -50
  28. data/ext/ca_obj_farray.c +1 -0
  29. data/ext/ca_obj_field.c +16 -3
  30. data/ext/ca_obj_fixlen_string.c +39 -26
  31. data/ext/ca_obj_grid.c +181 -124
  32. data/ext/ca_obj_meld.c +235 -55
  33. data/ext/ca_obj_moncmp.c +61 -61
  34. data/ext/ca_obj_monop.c +125 -107
  35. data/ext/ca_obj_object.c +551 -233
  36. data/ext/ca_obj_record.c +51 -16
  37. data/ext/ca_obj_reduce.c +5 -8
  38. data/ext/ca_obj_refer.c +117 -33
  39. data/ext/ca_obj_remap.c +57 -17
  40. data/ext/ca_obj_repeat.c +21 -5
  41. data/ext/ca_obj_roll.c +115 -11
  42. data/ext/ca_obj_select.c +88 -75
  43. data/ext/ca_obj_select_axis.c +58 -60
  44. data/ext/ca_obj_shift.c +22 -102
  45. data/ext/ca_obj_stack.c +223 -123
  46. data/ext/ca_obj_stride.c +267 -35
  47. data/ext/ca_obj_string.c +25 -22
  48. data/ext/ca_obj_tile.c +32 -16
  49. data/ext/ca_obj_time.c +23 -25
  50. data/ext/ca_obj_timedelta.c +21 -23
  51. data/ext/ca_obj_transpose.c +12 -3
  52. data/ext/ca_obj_triop.c +110 -117
  53. data/ext/ca_obj_window.c +157 -29
  54. data/ext/ca_op_ipower.c +25 -13
  55. data/ext/ca_op_powi.h +57 -30
  56. data/ext/ca_pow5_table.h +656 -0
  57. data/ext/ca_rng_normal.h +42 -0
  58. data/ext/ca_rng_xoshiro256pp.h +105 -0
  59. data/ext/ca_sort_kernels.h +70 -0
  60. data/ext/ca_sweep_engine.c +385 -198
  61. data/ext/ca_sweep_engine.h +31 -23
  62. data/ext/ca_triop_dispatch.c +4 -1
  63. data/ext/ca_triop_dispatch.h +5 -2
  64. data/ext/caframe_csv_reader.c +463 -0
  65. data/ext/carray.h +90 -28
  66. data/ext/carray_access.c +487 -231
  67. data/ext/carray_address_basis.c +731 -0
  68. data/ext/carray_attribute.c +45 -3
  69. data/ext/carray_bincount.c +32 -2
  70. data/ext/carray_broadcast.c +49 -17
  71. data/ext/carray_call_cfunc.c +667 -511
  72. data/ext/carray_cast.c +1556 -247
  73. data/ext/carray_cast_func.rb +34 -21
  74. data/ext/carray_conversion.c +197 -73
  75. data/ext/carray_copy.c +84 -38
  76. data/ext/carray_core.c +591 -119
  77. data/ext/carray_count.c +148 -96
  78. data/ext/carray_element.c +18 -27
  79. data/ext/carray_factorize.c +604 -291
  80. data/ext/carray_generate.c +76 -40
  81. data/ext/carray_histogram.c +152 -94
  82. data/ext/carray_hold.c +2 -1
  83. data/ext/carray_index_classifier.c +63 -96
  84. data/ext/carray_internal.h +400 -1
  85. data/ext/carray_lazy.c +834 -79
  86. data/ext/carray_loop.c +88 -66
  87. data/ext/carray_mask.c +705 -324
  88. data/ext/carray_math_kernel.h +1 -0
  89. data/ext/carray_median_percentile.c +263 -63
  90. data/ext/carray_memory_view.c +254 -248
  91. data/ext/carray_operator.c +1093 -363
  92. data/ext/carray_order.c +133 -69
  93. data/ext/carray_partition.c +102 -56
  94. data/ext/carray_random.c +582 -204
  95. data/ext/carray_scatter.c +308 -65
  96. data/ext/carray_segment.c +257 -0
  97. data/ext/carray_slab.c +183 -70
  98. data/ext/carray_sort.c +235 -240
  99. data/ext/carray_sort_kernel.c +8 -1
  100. data/ext/carray_struct.c +99 -21
  101. data/ext/carray_test.c +167 -64
  102. data/ext/carray_time_text.c +1182 -0
  103. data/ext/carray_utils.c +363 -98
  104. data/ext/extconf.rb +27 -2
  105. data/ext/mk_call_cfunc.rb +103 -118
  106. data/ext/mk_pow5_table.rb +35 -0
  107. data/ext/mkkernel.rb +1349 -851
  108. data/ext/ruby_carray.c +17 -2
  109. data/ext/version.h +4 -4
  110. data/ext/version.rb +4 -2
  111. data/lib/carray/attribute.rb +39 -0
  112. data/lib/carray/autoload_carray.rb +24 -3
  113. data/lib/carray/autoload_method_extension.rb +18 -0
  114. data/lib/carray/axis_group.rb +245 -20
  115. data/lib/carray/basics.rb +95 -25
  116. data/lib/carray/bincount_nd.rb +6 -0
  117. data/lib/carray/block_iterator.rb +200 -54
  118. data/lib/carray/boolean_reduce.rb +21 -29
  119. data/lib/carray/categorical.rb +244 -59
  120. data/lib/carray/categorical_iterator.rb +391 -757
  121. data/lib/carray/complex.rb +36 -29
  122. data/lib/carray/conditional.rb +94 -7
  123. data/lib/carray/const_string.rb +177 -27
  124. data/lib/carray/construct.rb +40 -0
  125. data/lib/carray/data_type_extension.rb +3 -0
  126. data/lib/carray/data_type_limits.rb +91 -0
  127. data/lib/carray/fixlen_string.rb +1 -1
  128. data/lib/carray/frame/concat.rb +157 -6
  129. data/lib/carray/frame/convert.rb +13 -7
  130. data/lib/carray/frame/csv_parser.rb +590 -76
  131. data/lib/carray/frame/describe.rb +79 -0
  132. data/lib/carray/frame/frame.rb +205 -35
  133. data/lib/carray/frame/group.rb +199 -40
  134. data/lib/carray/frame/io.rb +452 -60
  135. data/lib/carray/frame/join.rb +134 -46
  136. data/lib/carray/frame/records.rb +54 -19
  137. data/lib/carray/frame/reshape.rb +448 -0
  138. data/lib/carray/frame/sort.rb +26 -2
  139. data/lib/carray/frame/verbs.rb +726 -56
  140. data/lib/carray/frame.rb +2 -0
  141. data/lib/carray/fuse_source.rb +111 -2
  142. data/lib/carray/fusion.rb +229 -38
  143. data/lib/carray/inspect.rb +71 -20
  144. data/lib/carray/iterator.rb +157 -2
  145. data/lib/carray/lazy.rb +54 -67
  146. data/lib/carray/mask_gap_fill.rb +66 -50
  147. data/lib/carray/math.rb +32 -19
  148. data/lib/carray/meld_reduce.rb +40 -7
  149. data/lib/carray/methods/align_addr.rb +8 -4
  150. data/lib/carray/methods/bincount.rb +2 -2
  151. data/lib/carray/methods/bit_string.rb +14 -3
  152. data/lib/carray/methods/composition.rb +16 -10
  153. data/lib/carray/methods/discovery_along.rb +74 -0
  154. data/lib/carray/methods/factorize.rb +50 -0
  155. data/lib/carray/methods/gather_nd.rb +13 -2
  156. data/lib/carray/methods/insert_block.rb +9 -14
  157. data/lib/carray/methods/is_in.rb +130 -6
  158. data/lib/carray/methods/join.rb +1 -1
  159. data/lib/carray/methods/locate_addr.rb +235 -21
  160. data/lib/carray/methods/mask_duplicates.rb +35 -1
  161. data/lib/carray/methods/mode.rb +18 -1
  162. data/lib/carray/methods/nunique.rb +22 -1
  163. data/lib/carray/methods/repeat.rb +91 -0
  164. data/lib/carray/methods/resize.rb +21 -23
  165. data/lib/carray/methods/snap.rb +34 -12
  166. data/lib/carray/methods/unique.rb +52 -3
  167. data/lib/carray/record_order.rb +208 -0
  168. data/lib/carray/rng.rb +86 -0
  169. data/lib/carray/runtime.rb +14 -11
  170. data/lib/carray/segment_iterator.rb +990 -0
  171. data/lib/carray/serialize.rb +197 -27
  172. data/lib/carray/slab_iterator.rb +109 -40
  173. data/lib/carray/stack.rb +10 -4
  174. data/lib/carray/string_operation_extension.rb +9 -4
  175. data/lib/carray/struct.rb +142 -51
  176. data/lib/carray/struct_builder.rb +110 -14
  177. data/lib/carray/table.rb +2 -2
  178. data/lib/carray/time.rb +535 -105
  179. data/lib/carray/window_iterator.rb +259 -82
  180. data/lib/carray.rb +4 -2
  181. data/yard-stubs/ca_obj_array.rb +4 -4
  182. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  183. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  184. data/yard-stubs/ca_obj_block.rb +2 -7
  185. data/yard-stubs/ca_obj_fake.rb +1 -1
  186. data/yard-stubs/ca_obj_field.rb +1 -1
  187. data/yard-stubs/ca_obj_refer.rb +3 -3
  188. data/yard-stubs/ca_obj_roll.rb +0 -1
  189. data/yard-stubs/ca_obj_stride.rb +10 -7
  190. data/yard-stubs/ca_obj_transpose.rb +1 -1
  191. data/yard-stubs/ca_obj_window.rb +11 -4
  192. data/yard-stubs/carray_access.rb +6 -5
  193. data/yard-stubs/carray_broadcast.rb +2 -2
  194. data/yard-stubs/carray_cast.rb +78 -43
  195. data/yard-stubs/carray_conversion.rb +13 -1
  196. data/yard-stubs/carray_copy.rb +8 -3
  197. data/yard-stubs/carray_core.rb +3 -83
  198. data/yard-stubs/carray_count.rb +16 -2
  199. data/yard-stubs/carray_lazy.rb +210 -1
  200. data/yard-stubs/carray_loop.rb +4 -1
  201. data/yard-stubs/carray_mask.rb +34 -10
  202. data/yard-stubs/carray_math.rb +1499 -4
  203. data/yard-stubs/carray_median_percentile.rb +26 -8
  204. data/yard-stubs/carray_order.rb +43 -82
  205. data/yard-stubs/carray_scatter.rb +10 -3
  206. data/yard-stubs/carray_segment.rb +72 -0
  207. data/yard-stubs/carray_slab.rb +11 -8
  208. data/yard-stubs/carray_sort.rb +50 -8
  209. data/yard-stubs/carray_test.rb +10 -0
  210. data/yard-stubs/carray_utils.rb +0 -26
  211. metadata +19 -19
  212. data/ext/ca_for_each_element.h +0 -239
  213. data/ext/carray_build_flags.h +0 -3
  214. data/ext/carray_config.h +0 -28
  215. data/ext/carray_kernels_bincmp.c +0 -4446
  216. data/ext/carray_kernels_binop.c +0 -11001
  217. data/ext/carray_kernels_init.c +0 -1131
  218. data/ext/carray_kernels_map.c +0 -3467
  219. data/ext/carray_kernels_moncmp.c +0 -2097
  220. data/ext/carray_kernels_monop.c +0 -18313
  221. data/ext/carray_kernels_reduce_aggregate.c +0 -25837
  222. data/ext/carray_kernels_reduce_boolean.c +0 -330
  223. data/ext/carray_kernels_reduce_cumulative.c +0 -14593
  224. data/ext/carray_kernels_reduce_extreme.c +0 -16948
  225. data/ext/carray_kernels_reduce_variance.c +0 -3910
  226. data/ext/carray_kernels_scan.c +0 -3693
  227. data/ext/carray_kernels_search.c +0 -32138
  228. data/ext/carray_kernels_sort.c +0 -10626
  229. data/ext/carray_kernels_triop.c +0 -1392
@@ -66,7 +66,7 @@ class CArray
66
66
  t = transpose(*order)
67
67
  end
68
68
  inner = t.shape[-1]
69
- outer_n = t.elements / inner # 1 when ndim == 1
69
+ outer_n = t.shape[0...-1].inject(1, :*) # 1 when ndim == 1
70
70
  flat = t.reshape(outer_n, inner)
71
71
 
72
72
  strings = Array.new(outer_n) { |i| flat[i, nil].to_a.join(sep_str) }
@@ -1,52 +1,266 @@
1
+ require "carray/methods/is_in" # fixlen_widened_with, values_in_face, values_for_lookup
2
+
1
3
  class CArray
2
4
 
3
- # User-facing YARD docs for #locate_addr and #locate_nearest_addr live
4
- # in yard-stubs/carray_order.rb (grouped with the search family).
5
+ # @!group Sorting and searching
5
6
 
7
+ # @overload locate_addr(ref)
8
+ # Returns, for each element of `self`, the flat address into `ref`
9
+ # where the value first occurs, or `UNDEF` where it is not present.
10
+ # Builds a value-to-first-address map from `ref` in one pass (an
11
+ # open-addressing hash, the same substrate as {#unique} /
12
+ # {#value_counts}), then probes each element of `self`; no sort,
13
+ # peak memory `O(distinct ref values)`.
14
+ #
15
+ # The return is `ref`'s flat address (0 to `ref.elements - 1`), so a
16
+ # multi-dimensional `ref` still yields a `self`-shaped result of flat
17
+ # addresses; downstream reads (`ref[addr]`, `model_var[addr]`, ...)
18
+ # apply it as a flat gather.
19
+ #
20
+ # Works on numeric, `CA_OBJECT`, and `CA_FIXLEN` values, matching the
21
+ # value-hash discovery family: numeric follows `==` with all NaN
22
+ # collapsed to one value and `-0.0 == +0.0`; object follows Ruby
23
+ # `hash` / `eql?` with Float NaN collapsed; fixlen follows byte
24
+ # equality. When `ref` holds duplicate values the returned address is
25
+ # the earliest (appearance-order) occurrence. Masked cells of `ref` do
26
+ # not enter the map but still occupy their flat address; masked cells
27
+ # of `self` are `UNDEF` in the result.
28
+ #
29
+ # `self` and `ref` are compared at their common type, negotiated by
30
+ # `CArray.result_type` — a fractional query against an integer `ref`
31
+ # is compared at the promoted type rather than truncated, so `1.5` no
32
+ # longer matches `1`, and integers of mixed sign are matched by value
33
+ # (`-1` never matches `255`). Cross-family input (numeric against
34
+ # fixlen) raises.
35
+ #
36
+ # A time key (`CATime`) reads values that are not time arrays -- Time,
37
+ # Date or String objects, in an Array, an `:object` array or a string
38
+ # array -- as instants, on either side of the lookup, as {#is_in} does.
39
+ #
40
+ # Typical use is time-axis lookup: compute the address once against a
41
+ # reference axis, then reuse it to gather from many `ref`-shaped
42
+ # variables without repeating the lookup.
43
+ #
44
+ # @param ref [CArray, Array, Range] reference values to match against;
45
+ # any shape (used as flat). An Array or Range is coerced with `to_ca`
46
+ # and so lands in `:object`, where numeric matching follows Ruby
47
+ # `eql?` (`2.0` does not match `2`); pass a CArray to match in a
48
+ # numeric lane.
49
+ # @return [CArray] `:int64` flat addresses into `ref`, same shape as
50
+ # `self`; unmatched cells are masked.
51
+ # @raise [CArray::DataTypeError] when `self` and `ref` have no common data type.
6
52
  def locate_addr (ref)
53
+ # A time key (any Face that reconciles operands) against values that do
54
+ # not -- Time, Date or String objects, a string column -- reads them as
55
+ # values of its kind, whichever side they are on.
56
+ if ref.is_a?(CArray) && ref.class == self.class &&
57
+ respond_to?(:in_common_unit_with) && ref.unit != unit
58
+ a, b = in_common_unit_with(ref)
59
+ return a.locate_addr(b)
60
+ end
61
+ if respond_to?(:to_comparable) && !ref.respond_to?(:to_comparable)
62
+ ref = values_in_face(self, ref)
63
+ return CArray.int64(*shape).fill(UNDEF) unless ref
64
+ elsif ref.respond_to?(:to_comparable) && !respond_to?(:to_comparable)
65
+ q = values_in_face(ref, self)
66
+ return CArray.int64(*shape).fill(UNDEF) unless q
67
+ return q.locate_addr(ref)
68
+ end
69
+ unless ref.is_a?(CArray) || ref.respond_to?(:to_ca)
70
+ raise TypeError, "locate_addr: the reference is an array of values (got #{ref.class})"
71
+ end
7
72
  ref = ref.to_ca unless ref.is_a?(CArray)
73
+ ref = values_for_lookup(ref)
74
+ if data_type == CA_FIXLEN && ref.data_type == CA_FIXLEN && bytes != ref.bytes
75
+ a, b = fixlen_widened_with(ref)
76
+ return a.locate_addr(b) unless a.equal?(self) && b.equal?(ref)
77
+ end
8
78
  # Put self and ref in a common lane via the single-source promotion rule
9
79
  # (CArray.result_type), so a fractional query against an int ref is compared
10
80
  # at the promoted type instead of truncating (1.5 no longer matches 1).
11
81
  # to_type is elementwise and order-preserving, so the addresses stay valid
12
- # indices into ref. result_type raises for cross-family input.
13
- t = CArray.result_type(self, ref)
82
+ # indices into ref. result_type raises for cross-family input; an integer
83
+ # pair of mixed sign is matched by value (see value_match_type).
84
+ t = value_match_type(CArray.result_type(self, ref), ref.data_type)
14
85
  q = (data_type == t) ? self : to_type(t)
15
86
  r = (ref.data_type == t) ? ref : ref.to_type(t)
16
87
  q.send(:__locate_addr__, r)
17
88
  end
18
89
 
90
+ # @overload locate_nearest_addr(ref, direction: :round, tolerance: nil)
91
+ # Returns, for each element of `self`, the flat address into `ref` of
92
+ # the nearest reference value. Continuous sibling of {#locate_addr}:
93
+ # each cell is placed among the sorted reference values by comparing
94
+ # it with them, so an integer key is exact at any size and an infinite
95
+ # one keeps its place. `:round` takes the nearer neighbour, the upper
96
+ # one on a tie.
97
+ #
98
+ # Outside `ref`'s span the direction still decides: `:floor` matches
99
+ # a cell after the last value to that last value, `:ceil` matches a
100
+ # cell before the first value to that first value, and `:round`
101
+ # matches either to the nearer end. The other side has no match
102
+ # (`:floor` before the first value, `:ceil` after the last) and is
103
+ # masked. Use `tolerance:` to refuse matches that are too far away.
104
+ #
105
+ # `tolerance:` (default `nil`) sets a maximum accepted absolute
106
+ # distance between `self[i]` and its matched `ref` value. When
107
+ # `|ref[addr] - self[i]| > tolerance`, the result cell is masked. Use
108
+ # for accuracy-controlled matching (e.g. "an observation snaps to a
109
+ # time step only if within N seconds").
110
+ #
111
+ # `ref` need not be given in ascending order: it is sorted internally
112
+ # and the returned addresses are mapped back to positions in `ref` as
113
+ # passed. A masked or NaN `ref` value matches nothing. Two time keys
114
+ # of different units are compared in the finer of the two.
115
+ # With one value present in `ref`, that value is the match for
116
+ # every cell the direction allows (`:floor` at or after it, `:ceil`
117
+ # at or before it, `:round` all). With none, every cell is masked;
118
+ # an empty `self` gives an empty result. A masked or NaN cell of
119
+ # `self` is always masked.
120
+ #
121
+ # @param ref [CArray] 1-D reference grid to match against.
122
+ # @param direction [Symbol] `:round`, `:floor`, or `:ceil` — rounding
123
+ # applied to the fractional position.
124
+ # @param tolerance [Numeric, String, CATimedelta::Element, nil] maximum
125
+ # accepted `|self - ref|` distance; cells beyond this are masked.
126
+ # For a time key it is a duration: a String such as `"10 minutes"` or
127
+ # a CATimedelta value (a bare number is refused, since its length
128
+ # would depend on the key's unit). `nil` disables the check.
129
+ # @return [CArray] `:int64` flat addresses into `ref`, same shape as
130
+ # `self`; cells with no match in the direction and beyond-tolerance
131
+ # cells are masked.
132
+ # @raise [ArgumentError] when `direction` is not one of the accepted
133
+ # symbols, or `tolerance` is not a duration for a time key.
19
134
  def locate_nearest_addr (ref, direction: :round, tolerance: nil)
20
135
  unless [:round, :floor, :ceil].include?(direction)
21
136
  raise ArgumentError,
22
137
  "locate_nearest_addr: direction must be :round / :floor / " \
23
138
  ":ceil (got #{direction.inspect})"
24
139
  end
140
+ return CArray.int64(*shape) if elements == 0
141
+ # A masked reference value matches nothing, as in locate_addr: sorting
142
+ # puts the masked ones last, and only the others are searched. A NaN
143
+ # one has no place among the others either; it sorts after them.
25
144
  ri = ref.sort_addr
145
+ ri = ri[0...ref.count_not_masked] if ref.has_mask?
146
+ if ref.float? && ri.elements > 0
147
+ ri = ri[0...(ri.elements - ref[ri].is_nan.count(true))]
148
+ end
149
+ if ri.elements == 0
150
+ out = CArray.int64(*shape)
151
+ out[] = UNDEF
152
+ return out
153
+ end
26
154
  rs = ref[ri]
27
- sec = rs.linear_section(self)
28
- unless sec.is_a?(CArray)
29
- # A single-element (scalar-like) self makes linear_section collapse to
30
- # its scalar-query path, which returns a bare Float (or nil when out of
31
- # range) instead of a CArray. Rebuild a self-shaped float64 CArray so
32
- # the mask_invalid -> direction -> project pipeline stays array-valued
33
- # and the returned addr array matches self's shape.
34
- fill = CArray.float64(*shape)
35
- fill[] = sec.nil? ? UNDEF : sec
36
- sec = fill
37
- end
38
- masked = sec.mask_invalid
155
+ n = rs.elements
156
+ rk, qk = nearest_keys(rs, self)
157
+ # The position of each cell among the sorted references is counted, not
158
+ # interpolated: :floor takes the last reference at or below it, :ceil the
159
+ # first at or above it.
160
+ # Counting keeps an integer key exact past 2**53 and an infinite one in
161
+ # its place, where an interpolated position would not.
39
162
  si = case direction
40
- when :round then masked.round
41
- when :floor then masked.floor
42
- when :ceil then masked.ceil
43
- end.int64
163
+ when :floor then nearest_count_below(rk, qk, true) - 1
164
+ when :ceil then nearest_count_below(rk, qk, false)
165
+ when :round then nearest_round(rk, qk, nearest_count_below(rk, qk, false))
166
+ end
167
+ si[si.lt(0) | si.ge(n)] = UNDEF
168
+ si[qk.is_nan.strip_mask(false)] = UNDEF if qk.float?
169
+ si = si.reshape(*shape)
170
+ si[is_masked] = UNDEF if has_mask?
44
171
  idx = ri.project(si)
45
172
  if tolerance
46
173
  dist = (ref.project(idx) - self).abs
47
- idx[dist > tolerance] = UNDEF
174
+ idx[dist > nearest_tolerance(tolerance, dist)] = UNDEF
48
175
  end
49
176
  idx
50
177
  end
51
178
 
179
+ # The keys locate_nearest_addr orders by: the storage of the sorted
180
+ # reference and of the query, in one numeric type. A Face key is first
181
+ # brought into the reference's space (two time keys of different units
182
+ # into the finer of the two, which holds both exactly); plain keys meet at
183
+ # the type CArray.result_type gives, an integer pair of mixed sign at one
184
+ # that holds both.
185
+ private def nearest_keys(rs, q)
186
+ if rs.face? || q.face?
187
+ if rs.respond_to?(:to_unit) && q.class == rs.class && q.unit != rs.unit
188
+ u = CATimeUnitAlgebra.finer(rs.unit, q.unit)
189
+ rs = rs.to_unit(u)
190
+ q = q.to_unit(u)
191
+ end
192
+ q = rs.__send__(:__reconcile_operand__, q, "locate_nearest_addr")
193
+ rs = rs.__send__(:__face_storage__)
194
+ end
195
+ t = value_match_type(CArray.result_type(q, rs), rs.data_type)
196
+ rk = (rs.data_type == t) ? rs : rs.to_type(t)
197
+ qk = (q.data_type == t) ? q : q.to_type(t)
198
+ [rk, qk.flatten]
199
+ end
200
+
201
+ # For each query key, how many of the sorted reference keys are below it
202
+ # (inclusive: at or below). A numeric key is binary-searched in C; an
203
+ # object key is sorted together with the references, a tie putting the
204
+ # reference first when it counts, and counted against the references met
205
+ # before it.
206
+ private def nearest_count_below(rk, qk, inclusive)
207
+ return rk.send(:__count_below__, qk, inclusive) unless rk.data_type == CA_OBJECT
208
+ n, m = rk.elements, qk.elements
209
+ keys = CArray.concatenate([rk, qk])
210
+ tag = CArray.int8(n + m)
211
+ tag[0...n] = inclusive ? 0 : 1
212
+ tag[n...n+m] = inclusive ? 1 : 0
213
+ perm = CArray.sort_addr(keys, tag)
214
+ seen = perm.lt(n).cumsum.int64
215
+ query = perm.ge(n)
216
+ count = CArray.int64(m)
217
+ count[perm[query] - n] = seen[query]
218
+ count
219
+ end
220
+
221
+ # :round -- an exact match takes its first reference; otherwise the
222
+ # nearer of the two neighbours, the upper one on a tie. +lt+ counts the
223
+ # references below each key, so the upper neighbour is at lt and an exact
224
+ # match is the reference there.
225
+ private def nearest_round(rk, qk, lt)
226
+ lo = rk.project(lt - 1)
227
+ hi = rk.project(lt)
228
+ upper = (hi - qk).le(qk - lo).strip_mask(false)
229
+ upper[lo.is_masked] = true if lo.has_mask?
230
+ upper[hi.is_masked] = false if hi.has_mask?
231
+ upper[hi.eq(qk).strip_mask(false)] = true
232
+ si = lt - 1
233
+ si[upper] = lt[upper]
234
+ si
235
+ end
236
+
237
+ # The tolerance in the units of the distance. A time key measures its
238
+ # distance as a CATimedelta, so its tolerance is a duration: a String in
239
+ # the spelling CArray.time takes for a unit ("10 minutes"), or a
240
+ # CATimedelta value. A bare number would mean a different length in each
241
+ # unit, so it is refused there; any other key takes a number.
242
+ private def nearest_tolerance(tolerance, dist)
243
+ unless dist.is_a?(CATimedelta)
244
+ if tolerance.is_a?(String)
245
+ raise ArgumentError,
246
+ "tolerance: #{tolerance.inspect} is a duration, but the key is " \
247
+ "#{data_type_name}; give a number"
248
+ end
249
+ return tolerance
250
+ end
251
+ case tolerance
252
+ when String
253
+ res = CATime::Resolution.parse(tolerance)
254
+ CATimedelta::Element.new(res.count, res.base)
255
+ when CATimedelta::Element
256
+ tolerance
257
+ else
258
+ raise ArgumentError,
259
+ "tolerance: a time key needs a duration such as \"10 minutes\" " \
260
+ "or a CATimedelta value (got #{tolerance.inspect})"
261
+ end
262
+ end
263
+
264
+ # @!endgroup
265
+
52
266
  end
@@ -1,3 +1,5 @@
1
+ require "carray/methods/discovery_along"
2
+
1
3
  class CArray
2
4
 
3
5
  # @overload mask_duplicates(axis: nil)
@@ -23,7 +25,39 @@ class CArray
23
25
  # `nil` uses flatten order.
24
26
  # @return [CArray] shape-preserving copy of `self` with
25
27
  # duplicates masked.
26
- def mask_duplicates (axis: nil)
28
+ # @overload mask_duplicates(along: k)
29
+ # Returns a shape-preserving copy of `self` with every cell of a
30
+ # duplicated **sub-array** masked, comparing whole sub-arrays rather
31
+ # than cells: `along: k` names the axis whose index enumerates them,
32
+ # so `z.mask_duplicates(along: 0)` masks each repeated row of a 2-D
33
+ # array and keeps the first occurrence.
34
+ #
35
+ # Note the contrast with `axis:`, which names the axis a *fiber runs
36
+ # along* and marks repeated values inside each fiber. The two cannot
37
+ # be given together.
38
+ #
39
+ # A sub-array holding a masked cell does not participate: it is
40
+ # neither judged a duplicate nor able to make a later one, and its
41
+ # cells keep the mask they came with. `object` arrays are refused,
42
+ # because their cells hold Ruby references.
43
+ #
44
+ # @param along [Integer] axis whose index enumerates the sub-arrays.
45
+ # @return [CArray] shape-preserving copy of `self` with duplicated
46
+ # sub-arrays masked.
47
+ def mask_duplicates (axis: nil, along: nil)
48
+ reject_axis_with_along(axis, along, "mask_duplicates")
49
+ if along
50
+ keys = fibers_as_cells(along, "mask_duplicates")
51
+ # A sub-array that did not participate (it held a masked cell) is
52
+ # masked in `keys` already; only the ones that did and repeated
53
+ # are duplicates here.
54
+ repeated = keys.mask_duplicates.is_masked & keys.is_not_masked
55
+ args = [:_] * ndim
56
+ args[normalize_axis(along, "mask_duplicates")] = nil
57
+ spread = CArray.boolean(*shape)
58
+ spread[] = repeated[*args]
59
+ return mask_where(spread)
60
+ end
27
61
  dup =
28
62
  if axis.nil?
29
63
  # One seen-set over the flattened array, then restore shape.
@@ -65,12 +65,25 @@ class CArray
65
65
  # Only meaningful for discrete / binned data: raw float is almost all
66
66
  # unique, so every value is modal and the per-axis Array grows to the
67
67
  # fiber length. Bin first. Same NaN / mask semantics as {#is_mode}.
68
+ # A complex array raises {CArray::DataTypeError}, since the modes
69
+ # are listed in ascending order; {#is_mode} marks its modal cells.
68
70
  #
69
71
  # @param axis [Integer, nil] axis to take the mode along; `nil` uses
70
72
  # the whole array.
71
73
  # @return [CArray, Array<CArray>] 1-D CArray (flat) or an Array of
72
74
  # reduced CArrays, one per mode rank (per-axis).
73
75
  def mode (axis: nil)
76
+ # An error from the methods mode is made of (sort, ...) names mode.
77
+ CArray.__answer_as__(:mode) { mode_answered(axis) }
78
+ end
79
+
80
+ private def mode_answered (axis)
81
+ if complex?
82
+ raise CArray::DataTypeError,
83
+ "mode: the modes are listed in ascending order, and a " \
84
+ "#{data_type_name} value has no order (complex numbers are not " \
85
+ "ordered); is_mode marks the cells that hold a mode"
86
+ end
74
87
  return __mode_flat if axis.nil?
75
88
  k = normalize_axis(axis, "mode")
76
89
 
@@ -120,7 +133,11 @@ class CArray
120
133
  def __mode_flat
121
134
  values, counts = value_counts
122
135
  return values if values.elements == 0
123
- values[counts.eq(counts.max)].sort
136
+ modes = values[counts.eq(counts.max)]
137
+ # Records are sorted only by the members their struct declares in
138
+ # order_by:; without one the modes stay in first-appearance order.
139
+ return modes if is_a?(CARecord) && data_class.order_by.nil?
140
+ modes.sort
124
141
  end
125
142
 
126
143
  end
@@ -1,3 +1,5 @@
1
+ require "carray/methods/discovery_along"
2
+
1
3
  class CArray
2
4
 
3
5
  # @overload nunique(axis: nil, keep_axis: false)
@@ -29,7 +31,26 @@ class CArray
29
31
  # axis as a length-1 axis instead of dropping it.
30
32
  # @return [Integer, CArray] Integer for `axis: nil`, otherwise a
31
33
  # reduced `CA_INT64` CArray.
32
- def nunique (axis: nil, keep_axis: false)
34
+ # @overload nunique(along: k)
35
+ # Returns how many distinct **sub-arrays** `self` holds, comparing
36
+ # whole sub-arrays rather than cells: `along: k` names the axis
37
+ # whose index enumerates them, so `z.nunique(along: 0)` counts the
38
+ # distinct rows of a 2-D array. This is `len(np.unique(z, axis=k))`.
39
+ #
40
+ # Note the contrast with `axis:`, which names the axis a *fiber runs
41
+ # along* and counts the distinct values inside each fiber -- one
42
+ # count per fiber, not one count for the array. The two cannot be
43
+ # given together.
44
+ #
45
+ # Sub-arrays holding a masked cell do not participate and are not
46
+ # counted. `object` arrays are refused, because their cells hold
47
+ # Ruby references.
48
+ #
49
+ # @param along [Integer] axis whose index enumerates the sub-arrays.
50
+ # @return [Integer] the number of distinct sub-arrays.
51
+ def nunique (axis: nil, keep_axis: false, along: nil)
52
+ reject_axis_with_along(axis, along, "nunique")
53
+ return fibers_as_cells(along, "nunique").nunique if along
33
54
  # Per-fiber single-pass seen-set hash (C __nunique__), one lane per data type
34
55
  # family (numeric widen / NaN collapse, object rb_hash + rb_eql, fixlen
35
56
  # byte-hash + memcmp). Masked cells are skipped; the accumulator is a no-op
@@ -0,0 +1,91 @@
1
+ class CArray
2
+
3
+ # @overload repeat(count, axis: nil)
4
+ # @overload repeat(counts, axis: nil)
5
+ # Returns each element (or each sub-array) of `self` laid down
6
+ # `count` times in place, one after another.
7
+ #
8
+ # With an Integer, every element is repeated the same number of
9
+ # times. With an array of counts -- one per element, or one per
10
+ # sub-array when `axis:` is given -- each is repeated its own number
11
+ # of times, and a count of 0 drops that one:
12
+ #
13
+ # v = CA_INT([10, 20, 30])
14
+ # v.repeat(2) # => [ 10, 10, 20, 20, 30, 30 ]
15
+ # v.repeat([3, 1, 2]) # => [ 10, 10, 10, 20, 30, 30 ]
16
+ # v.repeat([2, 0, 1]) # => [ 10, 10, 30 ]
17
+ #
18
+ # This is `np.repeat`, and it is not {#tile}: `repeat` puts the
19
+ # copies of one element next to each other, `tile` lays the whole
20
+ # array down again -- `v.tile(2)` is `[10, 20, 30, 10, 20, 30]`.
21
+ #
22
+ # `axis: k` repeats the sub-arrays enumerated by axis `k` rather
23
+ # than the cells, so a 2-D array repeats whole rows and the result
24
+ # keeps its shape apart from that axis:
25
+ #
26
+ # t = CA_INT([[0, 1], [2, 3], [4, 5]])
27
+ # t.repeat([2, 1, 1], axis: 0)
28
+ # # => [ [ 0, 1 ],
29
+ # # [ 0, 1 ],
30
+ # # [ 2, 3 ],
31
+ # # [ 4, 5 ] ]
32
+ #
33
+ # Without `axis:` a multi-dimensional receiver is taken in flatten
34
+ # (row-major) order and the result is 1-D, as `np.repeat` does.
35
+ #
36
+ # The result is a **view** of `self` -- the same element named as
37
+ # many times as it was repeated -- so nothing is copied and writing
38
+ # through it reaches `self`, at every position that names the cell
39
+ # written. Take a `copy` when that is not wanted.
40
+ #
41
+ # Counts must be non-negative, and there must be exactly as many as
42
+ # there are elements (or sub-arrays along `axis:`). A masked count
43
+ # is refused: it names no number of repetitions.
44
+ #
45
+ # @param count [Integer, CArray, Array] how many times to repeat --
46
+ # one number for all, or one per element / sub-array.
47
+ # @param axis [Integer, nil] axis whose index enumerates the
48
+ # sub-arrays to repeat; `nil` repeats cells in flatten order.
49
+ # @return [CArray] the repeated elements, as a view of `self`.
50
+ def repeat (count, axis: nil)
51
+ along = axis.nil? ? nil : normalize_axis(axis, "repeat")
52
+ width = along ? shape[along] : elements
53
+
54
+ counts =
55
+ case count
56
+ when Integer
57
+ if count < 0
58
+ raise ArgumentError, "repeat: count must not be negative (got #{count})"
59
+ end
60
+ CArray.int64(width).fill(count)
61
+ else
62
+ given = count.to_ca.int64
63
+ unless given.elements == width
64
+ raise ArgumentError,
65
+ "repeat: #{given.elements} counts for #{width} " \
66
+ "#{along ? "sub-arrays along axis #{along}" : 'elements'}"
67
+ end
68
+ if given.has_mask? and given.count_masked > 0
69
+ raise ArgumentError,
70
+ "repeat: a masked count names no number of repetitions"
71
+ end
72
+ if given.lt(0).any
73
+ raise ArgumentError, "repeat: counts must not be negative"
74
+ end
75
+ given.flatten
76
+ end
77
+
78
+ source = along ? self : (ndim > 1 ? flatten : self)
79
+ source[*gather_args(CArray.segment_index(lengths: counts), along)]
80
+ end
81
+
82
+ # One index argument per axis: the gathered index on the axis being
83
+ # repeated, whole axes everywhere else.
84
+ private def gather_args (index, along)
85
+ return [index] unless along
86
+ args = [nil] * ndim
87
+ args[along] = index
88
+ args
89
+ end
90
+
91
+ end
@@ -4,8 +4,8 @@ class CArray
4
4
  # Returns `self` resized to `newdim`. The original data is
5
5
  # placed at offset 0 (positive size) or right-aligned (negative
6
6
  # size); the new area outside the original region is filled
7
- # with `fill_value`, or -- for fixlen storage -- left as zero
8
- # bytes. Pass `UNDEF` to mask the new area.
7
+ # with `fill_value`, or left as zero storage when it is not given
8
+ # (0 for a numeric array). Pass `UNDEF` to mask the new area.
9
9
  #
10
10
  # Each entry of `newdim` is: `nil` to keep the current
11
11
  # `shape[i]`, a positive Integer for a new size with the
@@ -13,15 +13,19 @@ class CArray
13
13
  # `d` for a new size `|d|` with the original right-aligned.
14
14
  #
15
15
  # Works on Face arrays (CARecord / CATime / ...) and
16
- # plain fixlen: the resize is done on the storage layout
17
- # (preserving bytes) and re-wrapped as the same Face.
16
+ # plain fixlen: the result is the same Face, and `fill_value` is a
17
+ # value of it (a Time for a time array).
18
18
  #
19
19
  # @param newdim [Array<Integer, nil>] new shape spec, one entry
20
20
  # per axis.
21
- # @param fill_value [Object] value for the extended area.
21
+ # @param fill_value [Object] value for the extended area (zero
22
+ # storage when not given).
22
23
  # @return [CArray] resized copy.
23
- def resize (*newdim, fill_value: 0)
24
- raise "ndim mismatch" if newdim.size != ndim
24
+ NO_FILL_VALUE = ::Object.new.freeze
25
+ private_constant :NO_FILL_VALUE
26
+
27
+ def resize (*newdim, fill_value: NO_FILL_VALUE)
28
+ raise ArgumentError, "ndim mismatch" if newdim.size != ndim
25
29
  offset = Array.new(ndim, 0)
26
30
  newdim = newdim.each_with_index.map do |d, i|
27
31
  case d
@@ -32,24 +36,18 @@ class CArray
32
36
  offset[i] = size - shape[i] if d < 0
33
37
  size
34
38
  else
35
- raise "invalid dimension size"
39
+ raise ArgumentError, "invalid dimension size"
36
40
  end
37
41
  end
38
- face_parent = self.face? ? self : nil
39
- src = self
40
- src = src.parent while src.face?
41
- dt = src.data_type
42
- bytes = (dt == :fixlen) ? src.bytes : nil
43
- out = CArray.new(dt, newdim, bytes: bytes)
44
- # Fill the new area: numeric storage takes fill_value as-is; fixlen
45
- # storage cannot hold a numeric 0, so leave zero bytes and honor only
46
- # UNDEF (mask) or an explicit String fill.
47
- if dt != :fixlen || fill_value.equal?(UNDEF) || fill_value.is_a?(String)
48
- out[] = fill_value
49
- end
50
- out.mask.paste(offset, src.false) if out.has_mask?
51
- out.paste(offset, src)
52
- out = out.face_lift(face_parent) if face_parent
42
+ # The original cells are copied as storage, so a read-only Face (a
43
+ # const string, a categorical) resizes too. The new area starts as zero
44
+ # storage; a given fill_value is written as a value of the array, which a
45
+ # read-only Face refuses.
46
+ out = __blank_like__(newdim)
47
+ out_storage = out.__send__(:__face_storage__)
48
+ out[] = fill_value unless fill_value.equal?(NO_FILL_VALUE)
49
+ out_storage.mask.paste(offset, self.false) if out_storage.has_mask?
50
+ out_storage.paste(offset, __face_storage__)
53
51
  out
54
52
  end
55
53