carray 3.0.5 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +233 -0
  3. data/README.md +1 -1
  4. data/carray.gemspec +3 -2
  5. data/ext/ca_axis_dispatch.c +6 -0
  6. data/ext/ca_axis_group.c +1 -1
  7. data/ext/ca_categorical_iterator.c +7 -7
  8. data/ext/ca_kernel_iterator.c +116 -231
  9. data/ext/ca_kernel_iterator.h +410 -254
  10. data/ext/ca_obj_bincmp.c +9 -1
  11. data/ext/ca_obj_binop.c +10 -1
  12. data/ext/ca_obj_bitfield.c +2 -0
  13. data/ext/ca_obj_byte_swap.c +2 -0
  14. data/ext/ca_obj_face.c +2 -0
  15. data/ext/ca_obj_fake.c +3 -1
  16. data/ext/ca_obj_grid.c +167 -12
  17. data/ext/ca_obj_meld.c +69 -113
  18. data/ext/ca_obj_moncmp.c +9 -1
  19. data/ext/ca_obj_monop.c +40 -2
  20. data/ext/ca_obj_object.c +54 -20
  21. data/ext/ca_obj_reduce.c +15 -1
  22. data/ext/ca_obj_refer.c +65 -20
  23. data/ext/ca_obj_remap.c +69 -10
  24. data/ext/ca_obj_roll.c +1 -0
  25. data/ext/ca_obj_select.c +4 -1
  26. data/ext/ca_obj_select_axis.c +1 -0
  27. data/ext/ca_obj_shift.c +1 -0
  28. data/ext/ca_obj_stack.c +310 -111
  29. data/ext/ca_obj_stride.c +131 -45
  30. data/ext/ca_obj_string.c +18 -2
  31. data/ext/ca_obj_tile.c +1 -0
  32. data/ext/ca_obj_triop.c +9 -1
  33. data/ext/ca_obj_window.c +1 -0
  34. data/ext/ca_op_ipower.c +8 -8
  35. data/ext/caframe_csv_reader.c +1 -1
  36. data/ext/carray.h +2 -1
  37. data/ext/carray_access.c +27 -64
  38. data/ext/carray_address_basis.c +134 -0
  39. data/ext/carray_attribute.c +8 -0
  40. data/ext/carray_broadcast.c +3 -4
  41. data/ext/carray_build_flags.h +1 -1
  42. data/ext/carray_cast.c +8 -11
  43. data/ext/carray_conversion.c +9 -5
  44. data/ext/carray_core.c +811 -72
  45. data/ext/carray_element.c +13 -9
  46. data/ext/carray_histogram.c +25 -3
  47. data/ext/carray_internal.h +147 -12
  48. data/ext/carray_kernels_bincmp.c +2007 -3
  49. data/ext/carray_kernels_binop.c +25365 -5199
  50. data/ext/carray_kernels_init.c +8 -8
  51. data/ext/carray_kernels_monop.c +10696 -3678
  52. data/ext/carray_kernels_reduce_aggregate.c +4429 -702
  53. data/ext/carray_kernels_reduce_boolean.c +1533 -15
  54. data/ext/carray_kernels_reduce_cumulative.c +1808 -248
  55. data/ext/carray_kernels_reduce_extreme.c +4508 -511
  56. data/ext/carray_kernels_reduce_variance.c +2284 -152
  57. data/ext/carray_kernels_scan.c +240 -240
  58. data/ext/carray_kernels_search.c +4211 -264
  59. data/ext/carray_kernels_sort.c +367 -450
  60. data/ext/carray_lazy.c +47 -0
  61. data/ext/carray_loop.c +1 -1
  62. data/ext/carray_mask.c +130 -139
  63. data/ext/carray_median_percentile.c +4 -4
  64. data/ext/carray_memory_view.c +13 -7
  65. data/ext/carray_operator.c +147 -44
  66. data/ext/carray_order.c +3 -2
  67. data/ext/carray_random.c +4 -1
  68. data/ext/carray_slab.c +97 -21
  69. data/ext/carray_sort.c +1 -1
  70. data/ext/carray_test.c +74 -19
  71. data/ext/carray_time_text.c +2 -2
  72. data/ext/extconf.rb +11 -0
  73. data/ext/mkkernel.rb +346 -94
  74. data/ext/ruby_carray.c +4 -1
  75. data/ext/version.h +4 -4
  76. data/lib/carray/attribute.rb +25 -1
  77. data/lib/carray/autoload_carray.rb +3 -3
  78. data/lib/carray/autoload_method_extension.rb +5 -3
  79. data/lib/carray/axis_group.rb +13 -7
  80. data/lib/carray/basics.rb +1 -1
  81. data/lib/carray/block_iterator.rb +17 -3
  82. data/lib/carray/categorical.rb +6 -8
  83. data/lib/carray/categorical_iterator.rb +2 -2
  84. data/lib/carray/complex.rb +2 -2
  85. data/lib/carray/conditional.rb +3 -2
  86. data/lib/carray/frame/column_verbs.rb +45 -14
  87. data/lib/carray/frame/convert.rb +19 -0
  88. data/lib/carray/frame/csv_parser.rb +32 -32
  89. data/lib/carray/frame/describe.rb +4 -2
  90. data/lib/carray/frame/frame.rb +399 -78
  91. data/lib/carray/frame/group.rb +679 -53
  92. data/lib/carray/frame/io.rb +209 -71
  93. data/lib/carray/frame/join.rb +146 -37
  94. data/lib/carray/frame/records.rb +3 -3
  95. data/lib/carray/frame/reshape.rb +89 -11
  96. data/lib/carray/frame/sort.rb +57 -9
  97. data/lib/carray/frame/verbs.rb +384 -49
  98. data/lib/carray/frame/windows.rb +44 -6
  99. data/lib/carray/inspect.rb +7 -2
  100. data/lib/carray/iterator.rb +32 -0
  101. data/lib/carray/lazy.rb +17 -8
  102. data/lib/carray/mask_gap_fill.rb +37 -17
  103. data/lib/carray/meld_reduce.rb +1 -1
  104. data/lib/carray/methods/choose.rb +3 -2
  105. data/lib/carray/methods/meshgrid.rb +3 -1
  106. data/lib/carray/runtime.rb +47 -8
  107. data/lib/carray/segment_iterator.rb +23 -8
  108. data/lib/carray/serialize.rb +34 -17
  109. data/lib/carray/stack.rb +18 -6
  110. data/lib/carray/time.rb +5 -0
  111. data/lib/carray/window_iterator.rb +16 -4
  112. data/yard-stubs/ca_obj_array.rb +26 -1
  113. data/yard-stubs/carray_cast.rb +8 -5
  114. data/yard-stubs/carray_conversion.rb +8 -2
  115. data/yard-stubs/carray_mask.rb +20 -1
  116. data/yard-stubs/carray_math.rb +31 -16
  117. data/yard-stubs/carray_memory_view.rb +7 -0
  118. data/yard-stubs/carray_test.rb +7 -4
  119. metadata +18 -4
@@ -79,8 +79,12 @@ class WindowedFrame
79
79
 
80
80
  # The windowed reductions. Without names, a frame of every column whose
81
81
  # type has the reduction (the on: and by: columns left out), on the
82
- # frame's rows and index; given names, the CArray (one) or the Array of
83
- # them (several), as +df[...]+ returns columns.
82
+ # frame's rows and index; given names, the CArray (one), as +df[name]+
83
+ # returns a column, or a frame of those columns (several) on the frame's
84
+ # rows and index. With +at:+ the answers
85
+ # are one per point, not per row, so names or none, the result is a frame
86
+ # with the points as its index, as a group reduction's result is labelled
87
+ # by the groups.
84
88
  CArray.send(:define_named_methods, self,
85
89
  [:sum, :mean, :min, :max, :prod, :accumulate, :median,
86
90
  :variance, :stddev, :variancep, :stddevp], :window_reduction)
@@ -88,6 +92,38 @@ class WindowedFrame
88
92
  CArray.send(:define_named_methods, self,
89
93
  [:first, :last, :nunique, :count, :all, :any], :window_value)
90
94
 
95
+ # Each window's +p+-th percentile (0 to 100), as +median+ gives the 50th.
96
+ # The percentage comes first, then the names.
97
+ #
98
+ # w.percentile(90, "rain")
99
+ #
100
+ # @param p [Numeric] the percentage.
101
+ # @param names [Array<String>] the columns; default every column that has
102
+ # a percentile.
103
+ # @return [CAFrame, CArray] as the other windowed reductions return.
104
+ def percentile (p, *names, min_count: nil, fill_value: nil)
105
+ CAFrame.__send__(:check_percentage, p)
106
+ window_reduce(:percentile, names, [p], min_count: min_count, fill_value: fill_value)
107
+ end
108
+
109
+ # Each window's five quartile percentiles, as +CAFrame#quantile+ gives
110
+ # them: a column "x" gives "x_p0", "x_p25", "x_p50", "x_p75" and "x_p100",
111
+ # a frame on the frame's rows and index, or with +at:+ on the points.
112
+ #
113
+ # @param names [Array<String>] the columns; default every column that has
114
+ # a quantile.
115
+ # @return [CAFrame]
116
+ def quantile (*names)
117
+ named = {}
118
+ over_named_columns(:quantile, names, @key_names) do |col, name|
119
+ named[name] = @plan.for_value(col).quantile
120
+ end
121
+ cols = CAFrame.__send__(:quantile_columns, named)
122
+ return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index) unless @at
123
+ key = @frame.__send__(:key_column, @key_names.first, "windows")
124
+ CAFrame.new(cols, axis_name: @key_names.first, index: CAFrame.__send__(:key_labels, key, @at.copy))
125
+ end
126
+
91
127
  private
92
128
 
93
129
  def window_reduction (reduction, *names, min_count: nil, fill_value: nil)
@@ -100,12 +136,14 @@ class WindowedFrame
100
136
 
101
137
  # An N-D column (each row a profile) is reduced along the rows, each
102
138
  # window keeping the trailing shape, as a group reduces it.
103
- def window_reduce (reduction, names, **keywords)
139
+ def window_reduce (reduction, names, args = [], **keywords)
104
140
  keywords = keywords.compact
105
- out = over_named_columns(reduction, names, @key_names) do |col|
106
- @plan.for_value(col).public_send(reduction, **keywords)
141
+ named = {}
142
+ out = over_named_columns(reduction, names, @key_names, args) do |col, name|
143
+ named[name] = @plan.for_value(col).public_send(reduction, *args, **keywords)
107
144
  end
108
- return out unless names.empty?
145
+ out = named if @at && !names.empty?
146
+ return out if out.is_a?(CArray) && !@at
109
147
  if @at
110
148
  # Each result has an index of its own, as a group's result has.
111
149
  key = @frame.__send__(:key_column, @key_names.first, "windows")
@@ -262,7 +262,11 @@ class CArray
262
262
  # count, and a truncated data preview.
263
263
  # @return [String]
264
264
  def inspect
265
- return CArray::Inspector.new(self).inspect_string
265
+ # A cell of an object array can hold an expression over the array, whose
266
+ # inspect shows the array again: that is refused as cyclic, as reading a
267
+ # cell that holds the array is.
268
+ return CArray::Inspector.new(self).inspect_string unless object?
269
+ refuse_reentry("inspect") { CArray::Inspector.new(self).inspect_string }
266
270
  end
267
271
 
268
272
  # @overload inspect_full
@@ -290,7 +294,8 @@ class CArray
290
294
  #
291
295
  # @return [String]
292
296
  def inspect_full
293
- return CArray::Inspector.new(self).inspect_string(abbrev: false)
297
+ return CArray::Inspector.new(self).inspect_string(abbrev: false) unless object?
298
+ refuse_reentry("inspect_full") { CArray::Inspector.new(self).inspect_string(abbrev: false) }
294
299
  end
295
300
 
296
301
  private
@@ -71,6 +71,38 @@ class CAIterator
71
71
  end
72
72
  end
73
73
 
74
+ # A member that keeps what it built from its source's values or mask (a
75
+ # padded copy, the views over it, counts of the cells present) names those
76
+ # instance variables here. They are dropped when a public method of the
77
+ # member is entered from outside, so every call reads the source as it is
78
+ # then; within one call the helpers it runs share them. What follows from
79
+ # the source's shape alone is kept for good.
80
+ def self.read_source_per_call (*state)
81
+ names = public_instance_methods(false)
82
+ prepend(Module.new do
83
+ module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
84
+ private def forget_source_state
85
+ #{state.inspect}.each { |v| remove_instance_variable(v) if instance_variable_defined?(v) }
86
+ end
87
+ RUBY
88
+ names.each do |name|
89
+ module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
90
+ def #{name} (...)
91
+ return super if @reading_source
92
+ forget_source_state
93
+ @reading_source = true
94
+ begin
95
+ super
96
+ ensure
97
+ @reading_source = false
98
+ end
99
+ end
100
+ RUBY
101
+ end
102
+ end)
103
+ end
104
+ private_class_method :read_source_per_call
105
+
74
106
  def self.reduce_keywords (op, kw) # :nodoc:
75
107
  if kw.key?(:axis)
76
108
  raise ArgumentError, "#{op}: unknown keyword: :axis (the iterator sets the axes)"
data/lib/carray/lazy.rb CHANGED
@@ -384,12 +384,6 @@ class CArray
384
384
  alias_method word, op
385
385
  end
386
386
 
387
- # `pow` is the canonical C method (defined by rb_define_method in
388
- # ext/ca_op_ipower.c) that `**` is rb_define_alias'd to. The `**`
389
- # dispatch in this file does not reach `pow` through the C alias, so
390
- # `a.lazy.pow(2)` would fall to eager; `pow` is aliased again here.
391
- alias_method :pow, :**
392
-
393
387
  # `fmax` / `fmin` are rb_define_alias'd to `pmax` / `pmin` at Init
394
388
  # time (the same C alias); they are aliased again beside them, in
395
389
  # LazyDispatch.
@@ -474,6 +468,14 @@ class CArray
474
468
  end
475
469
  end
476
470
 
471
+ # `pow` is the canonical C method (defined by rb_define_method in
472
+ # ext/ca_op_ipower.c) that `**` is rb_define_alias'd to. The `**`
473
+ # dispatch in this file does not reach `pow` through the C alias, so
474
+ # `a.lazy.pow(2)` would fall to eager; `pow` is aliased again here.
475
+ # An alias copies the method as it is at that moment, so this follows
476
+ # the integer-exponent `**` above, not the template one.
477
+ alias_method :pow, :**
478
+
477
479
  # coerce: when self is a lazy view and the scalar appears on the LEFT
478
480
  # (e.g. `2 * a.lazy`), Ruby's Numeric#* calls a.lazy.coerce(2). The
479
481
  # default coerce would unwrap the lazy-ness via eager scalar promotion;
@@ -495,6 +497,9 @@ class CArray
495
497
  # equality returning bool, not element-wise. Element-wise equality is
496
498
  # `eq` / `feq`.
497
499
  #
500
+ # A comparison with UNDEF goes to the eager method, which answers `eq` /
501
+ # `ne` from the mask and refuses the others.
502
+ #
498
503
  # `feq` is arity 1 in eager (compile-time FLT_EPSILON / DBL_EPSILON);
499
504
  # we mirror that here. Runtime eps is a future extension (struct field
500
505
  # already reserved).
@@ -518,7 +523,9 @@ class CArray
518
523
 
519
524
  CArray.send(:define_named_methods, LazyDispatch, LAZY_BINCMP_OP_IDS, template: <<~'RUBY')
520
525
  def __NAME__(other)
521
- if lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
526
+ if UNDEF.equal?(other)
527
+ super(other)
528
+ elsif lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
522
529
  if face_among_operands?(self, other)
523
530
  return computed_operand(self).__send__(__method__, computed_operand(other))
524
531
  end
@@ -540,7 +547,9 @@ class CArray
540
547
 
541
548
  CArray.send(:define_named_methods, LazyDispatch, LAZY_BINCMP_TOL_OP_IDS, template: <<~'RUBY')
542
549
  def __NAME__(other, tol)
543
- if lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
550
+ if UNDEF.equal?(other)
551
+ super(other, tol)
552
+ elsif lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
544
553
  if face_among_operands?(self, other)
545
554
  return computed_operand(self).__send__(__method__, computed_operand(other), tol)
546
555
  end
@@ -45,14 +45,14 @@ class CArray
45
45
  "unmask: pass either a constant fill value or method:, not both"
46
46
  end
47
47
  held = gap_fill(method, axis, "unmask")
48
- # Copy the filled values in place (a Face takes a Face of its own kind
49
- # as it is).
50
- value[] = held.value
51
- if held.has_mask?
52
- self.mask = held.mask # residual leading/trailing mask
53
- else
54
- super() # fully filled: drop the mask
55
- end
48
+ # Write only the cells the fill gave a value (a Face takes a Face of
49
+ # its own kind as it is). Writing every cell would let a view that
50
+ # reaches one cell twice send that cell's unfilled copy back last,
51
+ # masking it again.
52
+ return self unless has_mask?
53
+ filled = is_masked
54
+ filled = filled.and(held.is_not_masked) if held.has_mask?
55
+ self[filled] = held[filled] if filled.any
56
56
  return self
57
57
  end
58
58
  fill.equal?(MASK_FILL_UNSET) ? super() : super(fill)
@@ -161,7 +161,12 @@ class CArray
161
161
  end
162
162
 
163
163
  # Fill the masked cells of this 1-D array by linear interpolation against
164
- # the coordinate `x` (same length, ascending over the present cells).
164
+ # the coordinate `x` (same length). `x` need not be in order: the present
165
+ # cells are read as one series in the order of their `x`, so a frame whose
166
+ # index restarts for each station interpolates across the stations by
167
+ # time (interpolate one station at a time to keep them apart). Present
168
+ # cells sharing an `x` keep their row order. A cell whose `x` is masked,
169
+ # NaN or infinite has no place on the line: it is not used and not filled.
165
170
  # The one rule both the core (x = cell position) and CAFrame (x = index)
166
171
  # use: present cells come back exactly as they are, in this array's own
167
172
  # type; a masked cell inside the span of the present ones takes the
@@ -170,17 +175,34 @@ class CArray
170
175
  # NaN or Inf stored in a present cell is left alone. A Face goes through
171
176
  # its own linear_fetch, which keeps its unit and rounds to its grid.
172
177
  def linear_fill_along (x)
173
- present = is_not_masked
178
+ # A cell whose coordinate is masked, NaN or infinite has no place on the
179
+ # line: it is no point to interpolate between, and is not filled.
180
+ placed = x.is_not_masked
181
+ placed &= x.is_finite if !x.face? && x.float?
182
+ placed = placed.strip_mask(false)
183
+ present = is_not_masked & placed
174
184
  # Fewer than two valid points -> nothing to interpolate between; leave
175
185
  # every masked cell masked.
176
186
  return copy if present.count(true) < 2
177
- addr = x[present].linear_section(x) # valid positions -> monotonic grid
187
+ grid = x[present]
188
+ order = nil
189
+ if ( grid[1..-1] < grid[0..-2] ).any
190
+ order = grid.sort_addr # stable: ties keep their row order
191
+ grid = grid[order]
192
+ end
193
+ addr = grid.linear_section(x) # valid positions -> monotonic grid
194
+ fill = (is_masked & placed & addr.is_finite).strip_mask(false)
178
195
  if face?
179
- # The Face's linear_fetch already masks the exterior (out of range) and
180
- # lands on its own grid.
181
- return self[present].linear_fetch(addr)
196
+ # The Face's linear_fetch lands on its own grid; only the cells to fill
197
+ # take it, so a present cell comes back as it is.
198
+ known = self[present]
199
+ known = known[order] if order
200
+ out = copy
201
+ out[fill] = known.linear_fetch(addr)[fill] if fill.any
202
+ return out
182
203
  end
183
204
  v = value[present]
205
+ v = v[order] if order
184
206
  # A complex value interpolates its real and imaginary parts.
185
207
  interp = v.complex? ? v.linear_fetch(addr) : v.float64.linear_fetch(addr)
186
208
  # An integer array takes the nearest integer, half away from zero (as a
@@ -189,9 +211,7 @@ class CArray
189
211
  interp = interp.round if integer?
190
212
  # Built from the values alone, so a fully filled result carries no mask.
191
213
  out = value.copy
192
- masked = is_masked
193
- fill = masked & addr.is_finite
194
- exterior = masked & fill.not
214
+ exterior = is_masked & fill.not
195
215
  out[fill] = interp[fill] if fill.any
196
216
  out[exterior] = UNDEF if exterior.any
197
217
  out
@@ -171,7 +171,7 @@ class CAMeld
171
171
  # more elaborate (min_count / fill_value / keep_axis / multi-axis /
172
172
  # mask propagation) punts to super.
173
173
  def meld_reduce_fast_path_ok?(kw)
174
- return false unless CArray.__fast_paths_enabled__
174
+ return false unless CArray.__fast_paths_enabled__(:meld_reduce)
175
175
  # The mask of a meld is a meld built in C, with no parents list.
176
176
  return false unless parents
177
177
  return false unless (kw.keys - [:axis]).empty?
@@ -41,11 +41,12 @@ class CArray
41
41
  data_type = ca.empty? ? CA_OBJECT : CArray.result_type(*ca)
42
42
  end
43
43
  out = template(data_type)
44
+ dst = out.scalar? ? out.flatten : out # a CScalar takes no index
44
45
  choices.each_with_index do |v, i|
45
46
  s = self.eq(i)
46
- out[s] = v.is_a?(CArray) ? v[s] : v
47
+ dst[s] = !v.is_a?(CArray) ? v : v.scalar? ? v.flatten[s] : v[s]
47
48
  end
48
- out[self.is_masked] = UNDEF if has_mask?
49
+ dst[self.is_masked] = UNDEF if has_mask?
49
50
  text ? CAString.wrap(out) : out
50
51
  end
51
52
 
@@ -58,7 +58,9 @@ class CArray
58
58
  raise ArgumentError,
59
59
  "coordinate vector #{k} should be 1-D (got #{a.ndim}-D)"
60
60
  end
61
- a
61
+ # A CScalar takes no index, so a scalar axis is read as the
62
+ # one-cell array it stands for.
63
+ a.is_a?(CScalar) ? a.reshape(1) : a
62
64
  end
63
65
 
64
66
  ndim = axes.size
@@ -7,10 +7,11 @@ class CArray
7
7
 
8
8
  # A fast path answers by a different algorithm what a general path also
9
9
  # answers. A development build defines this in C, false when run with
10
- # CARRAY_GENERAL_PATH=1 so the test suite takes the general paths (see
10
+ # CARRAY_GENERAL_PATH=1 so the test suite takes the general paths, and
11
+ # false for a name listed in CARRAY_SHORTCUTS_OFF (see
11
12
  # ext/carray_internal.h); a release build has no switch.
12
13
  unless respond_to?(:__fast_paths_enabled__)
13
- def self.__fast_paths_enabled__ # :nodoc:
14
+ def self.__fast_paths_enabled__ (name = nil) # :nodoc:
14
15
  true
15
16
  end
16
17
  end
@@ -57,13 +58,14 @@ class CArray
57
58
  # is more naturally (and exactly, for Rational / BigDecimal weights) a
58
59
  # +(self * weights).sum+ composition than a dedicated :object kernel branch.
59
60
  #
60
- # +weights+ is a Ruby Numeric scalar (W-A1) or a same-shape CArray; the 1-D
61
- # axis-broadcast form (W-A2) is not supported for object weights (pass a
62
- # scalar or a full-shape weight), matching CArray's explicit-broadcast rule.
63
- # +opts+ is the trailing options Hash (axis:/keep_axis:/min_count:/
64
- # fill_value:) forwarded verbatim by the C escape; it goes straight to +sum+.
61
+ # +weights+ takes the forms the numeric kernel takes: a scalar (or a
62
+ # one-element CArray), an array of the shape of the reduced axes (taken in
63
+ # ascending axis order), or an array of self.shape. +opts+ is the trailing
64
+ # options Hash (axis:/keep_axis:/min_count:/fill_value:) forwarded verbatim
65
+ # by the C escape; it goes straight to +sum+.
65
66
  def wsum_object (weights, opts = {}) # :nodoc:
66
- (self * weights).sum(**opts)
67
+ weights = weights_over_self_shape(weights, opts[:axis], "wsum")
68
+ return (self * weights).sum(**opts)
67
69
  end
68
70
  private :wsum_object
69
71
 
@@ -76,6 +78,7 @@ class CArray
76
78
  def wmean_object (weights, opts = {}) # :nodoc:
77
79
  axis = opts[:axis]
78
80
  keep_axis = opts[:keep_axis]
81
+ weights = weights_over_self_shape(weights, axis, "wmean")
79
82
  prod = self * weights
80
83
  wv = (weights.is_a?(CArray) ? weights : self * 0 + weights).copy
81
84
  wv[prod.is_masked] = UNDEF if prod.has_mask?
@@ -97,6 +100,42 @@ class CArray
97
100
  end
98
101
  private :wmean_object
99
102
 
103
+ # The weights of wsum / wmean as a scalar or an array of self.shape. A
104
+ # CScalar is a scalar; any other array, one with a single element included,
105
+ # is held to the shapes. An array of the shape of the reduced axes (in ascending axis order) is
106
+ # repeated along the kept axes; any other shape is refused, with the same
107
+ # message as the numeric kernel.
108
+ def weights_over_self_shape (weights, axis, name) # :nodoc:
109
+ unless weights.is_a?(CArray)
110
+ return weights
111
+ end
112
+ if weights.scalar?
113
+ w = weights[0]
114
+ # A boolean cell reads as true / false; as a weight it is 1 / 0.
115
+ w = (w ? 1 : 0) if weights.data_type == CA_BOOLEAN && !UNDEF.equal?(w)
116
+ return w
117
+ end
118
+ if weights.shape == self.shape
119
+ return weights
120
+ end
121
+ axes = case axis
122
+ when nil then (0...self.ndim).to_a
123
+ when Integer then [axis]
124
+ else axis
125
+ end
126
+ axes = axes.map { |a| ( a < 0 ) ? a + self.ndim : a }.sort
127
+ reduced_shape = axes.map { |a| self.shape[a] }
128
+ unless weights.shape == reduced_shape
129
+ raise ArgumentError,
130
+ "#{name}: weights of shape #{weights.shape.inspect} not accepted " \
131
+ "(expected a scalar, the shape of the reduced axes " \
132
+ "#{reduced_shape.inspect}, or self.shape #{self.shape.inspect})"
133
+ end
134
+ spread = self.shape.each_index.map { |i| axes.include?(i) ? self.shape[i] : 1 }
135
+ return weights.reshape(*spread).broadcast_to(*self.shape)
136
+ end
137
+ private :weights_over_self_shape
138
+
100
139
  # Define on +mod+ one method per name in +names+, each a `def` that passes
101
140
  # its own name and its arguments to the private method +to+:
102
141
  #
@@ -337,9 +337,17 @@ class CASegmentIterator < CAIterator
337
337
  # with the distinctness of `CArray#nunique` (every NaN is one value,
338
338
  # -0.0 is 0.0). A segment with no present value counts 0.
339
339
  # @return [CArray]
340
+ # Coding every cell costs the same for every cell, and the segment-by-
341
+ # segment way the same for every segment, so long segments are faster one
342
+ # by one: from an average of about 100 cells (1M cells on an M2 Max).
343
+ NUNIQUE_KERNEL_MAX_LENGTH = 100
344
+
340
345
  def nunique (axis: nil)
341
346
  return axis_by_core(axis, :nunique) if axis
342
- return per_segment(CA_INT64) { |s| s.nunique } unless CArray.__fast_paths_enabled__
347
+ if ! CArray.__fast_paths_enabled__(:segment_nunique) ||
348
+ grouped.elements >= NUNIQUE_KERNEL_MAX_LENGTH * @k
349
+ return per_segment(CA_INT64) { |s| s.nunique }
350
+ end
343
351
  out = CArray.int64(@k)
344
352
  return out if grouped.elements.zero?
345
353
  # Each cell's value becomes a code, so a distinct (segment, value) pair is
@@ -476,9 +484,8 @@ class CASegmentIterator < CAIterator
476
484
  end
477
485
 
478
486
  # @overload wmean(weights)
479
- # Per-segment weighted mean as float64, matching `CArray#wmean`. Empty
480
- # segment -> MASKED; a present segment whose weights sum to zero -> NaN
481
- # (core's 0/0 contract).
487
+ # Per-segment weighted mean as float64, matching `CArray#wmean`. An empty
488
+ # segment, and one whose weights sum to zero, is masked.
482
489
  # @param weights [CArray]
483
490
  # @return [CArray]
484
491
  def wmean (weights, axis: nil)
@@ -749,7 +756,7 @@ class CASegmentIterator < CAIterator
749
756
  # cumcount in int64 -- which is not the core's answer for every data type
750
757
  # (the core counts a boolean's running sum in uint64).
751
758
  def kernel_scans? (op, answer)
752
- return false unless CArray.__fast_paths_enabled__
759
+ return false unless CArray.__fast_paths_enabled__(:segment_scan_kernel)
753
760
  return false if @value.face? || answer.face? || @value.complex?
754
761
  kernel_type =
755
762
  case op
@@ -906,9 +913,17 @@ class CASegmentIterator < CAIterator
906
913
  # segment's first or last slot and kept when it falls inside the segment.
907
914
  # A read-only Face cannot take UNDEF cell by cell, so its answers are
908
915
  # collected segment by segment, as its other reductions are.
916
+ #
917
+ # The running minimum or maximum costs the same for every cell, and the
918
+ # segment-by-segment way the same for every segment, so long segments
919
+ # are faster one by one: from an average of about 300 cells for :first
920
+ # and 2000 for :last (1M cells on an M2 Max).
921
+ EDGE_KERNEL_MAX_LENGTH = { first: 300, last: 2000 }.freeze
922
+
909
923
  def edge_values (op)
910
924
  n = grouped.elements
911
- if n.zero? || (@value.face? && ! face_output?) || ! CArray.__fast_paths_enabled__
925
+ if n.zero? || (@value.face? && ! face_output?) || ! CArray.__fast_paths_enabled__(:segment_edge_kernel) ||
926
+ n >= EDGE_KERNEL_MAX_LENGTH.fetch(op) * @k
912
927
  return per_segment(core_reduce_kind(op)) { |s| s.public_send(op) }
913
928
  end
914
929
  slot = CArray.int64(n).seq!
@@ -955,7 +970,7 @@ class CASegmentIterator < CAIterator
955
970
  # CARRAY_GENERAL_PATH set reduces every group by the core instead, which
956
971
  # is how the two are compared.
957
972
  def kernel_reads? (data_type = grouped.data_type)
958
- MONOID_TYPES.include?(data_type) && CArray.__fast_paths_enabled__
973
+ MONOID_TYPES.include?(data_type) && CArray.__fast_paths_enabled__(:segment_reduce_kernel)
959
974
  end
960
975
 
961
976
  def moments
@@ -991,7 +1006,7 @@ class CASegmentIterator < CAIterator
991
1006
  def all_any
992
1007
  return @all_any if defined?(@all_any)
993
1008
  @all_any =
994
- if grouped.data_type == CA_BOOLEAN && CArray.__fast_paths_enabled__
1009
+ if grouped.data_type == CA_BOOLEAN && CArray.__fast_paths_enabled__(:segment_all_any_kernel)
995
1010
  a = CArray.boolean(@k)
996
1011
  o = CArray.boolean(@k)
997
1012
  grouped.send(:__reduceat_all_any__, @offsets, a, o)
@@ -685,11 +685,11 @@ class CArray
685
685
 
686
686
  end
687
687
 
688
- # Marshal of a view and a scalar: the {CArray#marshal_dump} payload, loaded
689
- # back by the class method rather than into an empty instance. A view comes
690
- # back as a plain array (a copy owns its memory, as #copy gives); a scalar
691
- # comes back a scalar. A wrap is refused below.
692
- [CAView, CAWrap, CScalar].each do |klass|
688
+ # Marshal of a view, a mask and a scalar: the {CArray#marshal_dump} payload,
689
+ # loaded back by the class method rather than into an empty instance. A
690
+ # view, a wrap and a mask come back as a plain array (a copy owns its
691
+ # memory, as #copy gives); a scalar comes back a scalar.
692
+ [CAView, CAWrap, CScalar, CArrayMask, CAObjectMask].each do |klass|
693
693
  klass.class_eval do
694
694
  # A Face other than a record is refused: the payload would carry its
695
695
  # storage without what makes it the Face (a unit, labels), and load
@@ -703,28 +703,45 @@ end
703
703
  "dump its values with to_type(:object)" +
704
704
  (storage ? ", or its storage, which ##{storage} gives" : "")
705
705
  end
706
+ return dump_payload unless data_type == CA_OBJECT
707
+ # An object view dumps its cells by a nested Marshal.dump, which does
708
+ # not know the objects the outer dump has written. A cell holding the
709
+ # view again (directly or through other views) would dump it without
710
+ # end, so dumping the same view from inside its own dump is refused,
711
+ # as reading such a cell is.
712
+ refuse_reentry("Marshal.dump") { dump_payload }
713
+ end
714
+
715
+ private
716
+
717
+ # Marshal writes the instance variables of an object that has _dump
718
+ # after the String it returns, unless the String has instance
719
+ # variables of its own. A view keeps its parent in one, so a view of
720
+ # a large array would carry the whole parent, and a parent holding the
721
+ # view in a cell would be an object cycle. The String is given one of
722
+ # its own, which _load ignores.
723
+ def dump_payload
706
724
  Marshal.dump(CArray.instance_method(:marshal_dump).bind_call(self))
725
+ .tap { |s| s.instance_variable_set(:@carray_payload, true) }
707
726
  end
708
727
  end
709
728
  end
710
729
 
711
- class CAView
730
+ [CAView, CArrayMask, CAObjectMask].each do |klass|
712
731
  # @return [CArray] the plain array a {#_dump} string describes.
713
- def self._load (str)
714
- CArray.send(:array_from_marshal_payload, Marshal.load(str))
732
+ # Defined with def, not define_singleton_method: a block's method keeps
733
+ # the block's self and cannot be called from another Ractor.
734
+ klass.singleton_class.class_eval do
735
+ def _load (str)
736
+ CArray.send(:array_from_marshal_payload, Marshal.load(str))
737
+ end
715
738
  end
716
739
  end
717
740
 
718
741
  class CAWrap
719
- # A wrap borrows memory, which Marshal cannot carry to another process;
720
- # Ruby would also try to write the objects that keep the borrowed memory
721
- # alive. Its copy owns its memory and marshals.
722
- def _dump (_level)
723
- raise TypeError,
724
- "Marshal cannot carry a #{self.class}: it borrows its memory; " \
725
- "dump .copy instead"
726
- end
727
-
742
+ # A wrap borrows its memory, which Marshal cannot carry to another
743
+ # process: {#_dump} above writes its values, and the load gives a plain
744
+ # array that owns a copy of them, as it does for a view of the wrap.
728
745
  def self._load (str)
729
746
  CArray.send(:array_from_marshal_payload, Marshal.load(str))
730
747
  end
data/lib/carray/stack.rb CHANGED
@@ -106,11 +106,14 @@ class CArray
106
106
  # describe a segment sequence, so absorbing their parents keeps chain
107
107
  # depth at 1 (avoids 2-level xfer_all / reduce chains through the
108
108
  # intermediate CAMeld). A CAMeld with a different meld_axis is left
109
- # intact — its segment structure is orthogonal.
110
- if arrays.any? { |a| a.is_a?(CAMeld) && a.meld_axis == axis_norm }
111
- arrays = arrays.flat_map { |a|
112
- a.is_a?(CAMeld) && a.meld_axis == axis_norm ? a.parents : [a]
113
- }
109
+ # intact — its segment structure is orthogonal. So is a read-only one
110
+ # (frozen or flagged): writes through it are refused, and absorbing it
111
+ # would drop that. The mask of a meld is a CAMeld with no parents of
112
+ # its own; it is passed on, to be refused as a part.
113
+ absorbed = ->(a) { a.is_a?(CAMeld) && a.meld_axis == axis_norm &&
114
+ !a.read_only? && !a.mask_array? }
115
+ if arrays.any?(&absorbed)
116
+ arrays = arrays.flat_map { |a| absorbed.(a) ? a.parents : [a] }
114
117
  end
115
118
  CAMeld.new(arrays, axis: axis_norm)
116
119
  end
@@ -291,10 +294,19 @@ class CAStack
291
294
  # preserving `self.k_axis`. Distinct from `CArray#stack`, which
292
295
  # always treats `self` as a single parent of a new K-stack.
293
296
  # @param others [Array<CArray>] extra parents to append.
297
+ # A read-only stack hands on read-only views of its parents, so
298
+ # writes into its cells stay refused.
294
299
  # @return [CAStack]
295
300
  # @raise [ArgumentError] when no `others` are given.
296
301
  def append (*others)
297
302
  raise ArgumentError, "append: at least one parent required" if others.empty?
298
- CArray.stack(self.parents + others, axis: self.k_axis)
303
+ if mask_array?
304
+ raise ArgumentError, "append: a mask array can have no mask and can not be a part"
305
+ end
306
+ mine = self.parents
307
+ if read_only?
308
+ mine = mine.map { |p| p[].tap(&:set_read_only_flag) }
309
+ end
310
+ CArray.stack(mine + others, axis: self.k_axis)
299
311
  end
300
312
  end
data/lib/carray/time.rb CHANGED
@@ -1122,6 +1122,9 @@ class CATime
1122
1122
  # @raise [RangeError] when the widened ticks overflow int64.
1123
1123
  def to_unit(unit)
1124
1124
  to = CATime::Resolution.parse(unit)
1125
+ # The conversion hands back the storage itself when the unit does not
1126
+ # change; the result is a new array whichever unit is asked for.
1127
+ return face_storage.copy.time(unit: to) if to == self.unit
1125
1128
  CATimeUnitAlgebra.convert_instant_floor(face_storage, self.unit, to).time(unit: to)
1126
1129
  end
1127
1130
 
@@ -1830,6 +1833,8 @@ class CATimedelta
1830
1833
  # @raise [RangeError] when the widened ticks overflow int64.
1831
1834
  def to_unit(unit)
1832
1835
  to = CATime::Resolution.parse(unit)
1836
+ # As CATime#to_unit: a new array even when the unit does not change.
1837
+ return face_storage.copy.timedelta(unit: to) if to == self.unit
1833
1838
  CATimeUnitAlgebra.convert_scale_trunc(face_storage, self.unit, to).timedelta(unit: to)
1834
1839
  end
1835
1840