carray 3.0.4 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +438 -0
  3. data/README.md +1 -1
  4. data/carray.gemspec +3 -2
  5. data/ext/ca_array_pool.c +25 -1
  6. data/ext/ca_axis_dispatch.c +19 -20
  7. data/ext/ca_axis_group.c +1 -1
  8. data/ext/ca_categorical_iterator.c +34 -27
  9. data/ext/ca_compare.c +4 -4
  10. data/ext/ca_composite_dispatch.c +9 -9
  11. data/ext/ca_kernel_iterator.c +122 -237
  12. data/ext/ca_kernel_iterator.h +522 -337
  13. data/ext/ca_obj_array.c +69 -19
  14. data/ext/ca_obj_bincmp.c +10 -1
  15. data/ext/ca_obj_binop.c +11 -1
  16. data/ext/ca_obj_bitarray.c +5 -1
  17. data/ext/ca_obj_bitfield.c +60 -43
  18. data/ext/ca_obj_byte_swap.c +4 -1
  19. data/ext/ca_obj_const_string.c +15 -3
  20. data/ext/ca_obj_face.c +44 -1
  21. data/ext/ca_obj_fake.c +3 -1
  22. data/ext/ca_obj_field.c +13 -3
  23. data/ext/ca_obj_grid.c +219 -17
  24. data/ext/ca_obj_meld.c +69 -113
  25. data/ext/ca_obj_moncmp.c +9 -1
  26. data/ext/ca_obj_monop.c +86 -12
  27. data/ext/ca_obj_object.c +118 -32
  28. data/ext/ca_obj_record.c +3 -0
  29. data/ext/ca_obj_reduce.c +19 -1
  30. data/ext/ca_obj_refer.c +80 -26
  31. data/ext/ca_obj_remap.c +69 -10
  32. data/ext/ca_obj_roll.c +27 -6
  33. data/ext/ca_obj_select.c +38 -6
  34. data/ext/ca_obj_select_axis.c +1 -0
  35. data/ext/ca_obj_shift.c +32 -7
  36. data/ext/ca_obj_stack.c +311 -111
  37. data/ext/ca_obj_stride.c +159 -64
  38. data/ext/ca_obj_string.c +27 -2
  39. data/ext/ca_obj_tile.c +28 -7
  40. data/ext/ca_obj_transpose.c +8 -4
  41. data/ext/ca_obj_triop.c +12 -1
  42. data/ext/ca_obj_window.c +80 -50
  43. data/ext/ca_op_ipower.c +12 -12
  44. data/ext/ca_sort_kernels.h +6 -3
  45. data/ext/caframe_csv_reader.c +1 -1
  46. data/ext/carray.h +35 -1
  47. data/ext/carray_access.c +43 -67
  48. data/ext/carray_address_basis.c +134 -0
  49. data/ext/carray_attribute.c +32 -0
  50. data/ext/carray_broadcast.c +3 -4
  51. data/ext/carray_build_flags.h +1 -1
  52. data/ext/carray_cast.c +251 -93
  53. data/ext/carray_cast_func.rb +56 -26
  54. data/ext/carray_conversion.c +11 -7
  55. data/ext/carray_core.c +814 -72
  56. data/ext/carray_count.c +6 -0
  57. data/ext/carray_element.c +22 -13
  58. data/ext/carray_factorize.c +20 -20
  59. data/ext/carray_generate.c +8 -2
  60. data/ext/carray_histogram.c +26 -4
  61. data/ext/carray_index_classifier.c +4 -4
  62. data/ext/carray_internal.h +159 -12
  63. data/ext/carray_kernels_bincmp.c +2174 -170
  64. data/ext/carray_kernels_binop.c +25653 -5487
  65. data/ext/carray_kernels_init.c +8 -8
  66. data/ext/carray_kernels_moncmp.c +96 -96
  67. data/ext/carray_kernels_monop.c +12253 -5235
  68. data/ext/carray_kernels_reduce_aggregate.c +6150 -2423
  69. data/ext/carray_kernels_reduce_boolean.c +1533 -15
  70. data/ext/carray_kernels_reduce_cumulative.c +2442 -882
  71. data/ext/carray_kernels_reduce_extreme.c +5658 -1661
  72. data/ext/carray_kernels_reduce_variance.c +2534 -402
  73. data/ext/carray_kernels_scan.c +368 -368
  74. data/ext/carray_kernels_search.c +4551 -604
  75. data/ext/carray_kernels_sort.c +659 -751
  76. data/ext/carray_kernels_triop.c +114 -114
  77. data/ext/carray_key_windows.c +4 -0
  78. data/ext/carray_lazy.c +47 -0
  79. data/ext/carray_loop.c +1 -1
  80. data/ext/carray_mask.c +175 -161
  81. data/ext/carray_median_percentile.c +4 -4
  82. data/ext/carray_memory_view.c +15 -9
  83. data/ext/carray_operator.c +152 -49
  84. data/ext/carray_order.c +230 -10
  85. data/ext/carray_random.c +22 -19
  86. data/ext/carray_scatter.c +24 -7
  87. data/ext/carray_slab.c +128 -29
  88. data/ext/carray_sort.c +224 -16
  89. data/ext/carray_sort_kernel.c +5 -7
  90. data/ext/carray_struct.c +35 -43
  91. data/ext/carray_test.c +74 -19
  92. data/ext/carray_time_text.c +6 -4
  93. data/ext/carray_utils.c +63 -21
  94. data/ext/extconf.rb +11 -0
  95. data/ext/mkkernel.rb +546 -199
  96. data/ext/ruby_carray.c +4 -1
  97. data/ext/version.h +4 -4
  98. data/lib/carray/attribute.rb +42 -5
  99. data/lib/carray/autoload_carray.rb +3 -3
  100. data/lib/carray/autoload_method_extension.rb +5 -3
  101. data/lib/carray/axis_group.rb +13 -7
  102. data/lib/carray/basics.rb +28 -8
  103. data/lib/carray/block_iterator.rb +17 -3
  104. data/lib/carray/categorical.rb +35 -23
  105. data/lib/carray/categorical_iterator.rb +8 -8
  106. data/lib/carray/complex.rb +2 -2
  107. data/lib/carray/conditional.rb +3 -2
  108. data/lib/carray/const_string.rb +10 -0
  109. data/lib/carray/data_type_extension.rb +8 -1
  110. data/lib/carray/fixlen_string.rb +41 -2
  111. data/lib/carray/frame/column_verbs.rb +48 -16
  112. data/lib/carray/frame/concat.rb +17 -3
  113. data/lib/carray/frame/convert.rb +19 -0
  114. data/lib/carray/frame/csv_parser.rb +32 -32
  115. data/lib/carray/frame/describe.rb +4 -2
  116. data/lib/carray/frame/frame.rb +405 -85
  117. data/lib/carray/frame/group.rb +777 -64
  118. data/lib/carray/frame/io.rb +212 -72
  119. data/lib/carray/frame/join.rb +146 -37
  120. data/lib/carray/frame/records.rb +3 -3
  121. data/lib/carray/frame/reshape.rb +89 -11
  122. data/lib/carray/frame/sort.rb +57 -9
  123. data/lib/carray/frame/verbs.rb +384 -49
  124. data/lib/carray/frame/windows.rb +44 -6
  125. data/lib/carray/fuse_source.rb +31 -2
  126. data/lib/carray/inspect.rb +7 -2
  127. data/lib/carray/iterator.rb +32 -0
  128. data/lib/carray/lazy.rb +52 -8
  129. data/lib/carray/mask_gap_fill.rb +37 -17
  130. data/lib/carray/meld_reduce.rb +1 -1
  131. data/lib/carray/methods/choose.rb +3 -2
  132. data/lib/carray/methods/is_in.rb +8 -1
  133. data/lib/carray/methods/meshgrid.rb +3 -1
  134. data/lib/carray/runtime.rb +47 -8
  135. data/lib/carray/segment_iterator.rb +35 -16
  136. data/lib/carray/serialize.rb +53 -19
  137. data/lib/carray/stack.rb +33 -18
  138. data/lib/carray/string.rb +8 -0
  139. data/lib/carray/string_operation_extension.rb +13 -5
  140. data/lib/carray/struct_builder.rb +5 -3
  141. data/lib/carray/time.rb +112 -68
  142. data/lib/carray/window_iterator.rb +16 -4
  143. data/yard-stubs/ca_obj_array.rb +26 -1
  144. data/yard-stubs/ca_obj_transpose.rb +5 -4
  145. data/yard-stubs/carray_attribute.rb +3 -1
  146. data/yard-stubs/carray_cast.rb +10 -5
  147. data/yard-stubs/carray_conversion.rb +13 -3
  148. data/yard-stubs/carray_element.rb +7 -6
  149. data/yard-stubs/carray_mask.rb +28 -4
  150. data/yard-stubs/carray_math.rb +45 -20
  151. data/yard-stubs/carray_memory_view.rb +7 -0
  152. data/yard-stubs/carray_order.rb +100 -4
  153. data/yard-stubs/carray_scatter.rb +7 -5
  154. data/yard-stubs/carray_test.rb +7 -4
  155. metadata +18 -4
@@ -79,8 +79,12 @@ class WindowedFrame
79
79
 
80
80
  # The windowed reductions. Without names, a frame of every column whose
81
81
  # type has the reduction (the on: and by: columns left out), on the
82
- # frame's rows and index; given names, the CArray (one) or the Array of
83
- # them (several), as +df[...]+ returns columns.
82
+ # frame's rows and index; given names, the CArray (one), as +df[name]+
83
+ # returns a column, or a frame of those columns (several) on the frame's
84
+ # rows and index. With +at:+ the answers
85
+ # are one per point, not per row, so names or none, the result is a frame
86
+ # with the points as its index, as a group reduction's result is labelled
87
+ # by the groups.
84
88
  CArray.send(:define_named_methods, self,
85
89
  [:sum, :mean, :min, :max, :prod, :accumulate, :median,
86
90
  :variance, :stddev, :variancep, :stddevp], :window_reduction)
@@ -88,6 +92,38 @@ class WindowedFrame
88
92
  CArray.send(:define_named_methods, self,
89
93
  [:first, :last, :nunique, :count, :all, :any], :window_value)
90
94
 
95
+ # Each window's +p+-th percentile (0 to 100), as +median+ gives the 50th.
96
+ # The percentage comes first, then the names.
97
+ #
98
+ # w.percentile(90, "rain")
99
+ #
100
+ # @param p [Numeric] the percentage.
101
+ # @param names [Array<String>] the columns; default every column that has
102
+ # a percentile.
103
+ # @return [CAFrame, CArray] as the other windowed reductions return.
104
+ def percentile (p, *names, min_count: nil, fill_value: nil)
105
+ CAFrame.__send__(:check_percentage, p)
106
+ window_reduce(:percentile, names, [p], min_count: min_count, fill_value: fill_value)
107
+ end
108
+
109
+ # Each window's five quartile percentiles, as +CAFrame#quantile+ gives
110
+ # them: a column "x" gives "x_p0", "x_p25", "x_p50", "x_p75" and "x_p100",
111
+ # a frame on the frame's rows and index, or with +at:+ on the points.
112
+ #
113
+ # @param names [Array<String>] the columns; default every column that has
114
+ # a quantile.
115
+ # @return [CAFrame]
116
+ def quantile (*names)
117
+ named = {}
118
+ over_named_columns(:quantile, names, @key_names) do |col, name|
119
+ named[name] = @plan.for_value(col).quantile
120
+ end
121
+ cols = CAFrame.__send__(:quantile_columns, named)
122
+ return CAFrame.new(cols, axis_name: @frame.axis_name, index: @frame.index) unless @at
123
+ key = @frame.__send__(:key_column, @key_names.first, "windows")
124
+ CAFrame.new(cols, axis_name: @key_names.first, index: CAFrame.__send__(:key_labels, key, @at.copy))
125
+ end
126
+
91
127
  private
92
128
 
93
129
  def window_reduction (reduction, *names, min_count: nil, fill_value: nil)
@@ -100,12 +136,14 @@ class WindowedFrame
100
136
 
101
137
  # An N-D column (each row a profile) is reduced along the rows, each
102
138
  # window keeping the trailing shape, as a group reduces it.
103
- def window_reduce (reduction, names, **keywords)
139
+ def window_reduce (reduction, names, args = [], **keywords)
104
140
  keywords = keywords.compact
105
- out = over_named_columns(reduction, names, @key_names) do |col|
106
- @plan.for_value(col).public_send(reduction, **keywords)
141
+ named = {}
142
+ out = over_named_columns(reduction, names, @key_names, args) do |col, name|
143
+ named[name] = @plan.for_value(col).public_send(reduction, *args, **keywords)
107
144
  end
108
- return out unless names.empty?
145
+ out = named if @at && !names.empty?
146
+ return out if out.is_a?(CArray) && !@at
109
147
  if @at
110
148
  # Each result has an index of its own, as a group's result has.
111
149
  key = @frame.__send__(:key_column, @key_names.first, "windows")
@@ -160,9 +160,9 @@ class CArray
160
160
 
161
161
  def self.extract (block)
162
162
  sequence = RubyVM::InstructionSequence.of(block) rescue nil
163
- location = sequence && sequence.to_a[4][:code_location]
164
163
  path = sequence && (sequence.absolute_path || sequence.path)
165
- unless location && path && File.readable?(path)
164
+ location = path && File.readable?(path) && code_location(sequence, path)
165
+ unless location
166
166
  raise ArgumentError,
167
167
  "CArray.fuse cannot read this block's source (defined in irb, " \
168
168
  "eval, or a file that is no longer there). Write `.lazy` on " \
@@ -181,6 +181,35 @@ class CArray
181
181
  end
182
182
  end
183
183
 
184
+ # [first_line, first_column, last_line, last_column] of the block. The
185
+ # block that starts on the sequence's first line is found in the file's
186
+ # syntax tree. Only when two start on that line is the sequence itself
187
+ # dumped: InstructionSequence#to_a reads the instructions while another
188
+ # Ractor may be running them, and that can crash the process.
189
+ def self.code_location (sequence, path)
190
+ line = sequence.first_lineno
191
+ found = []
192
+ pending = [Prism.parse_file(path).value]
193
+ until pending.empty?
194
+ node = pending.pop
195
+ next unless node
196
+ # A lambda's code starts after the arrow, at its parameters or brace.
197
+ start = case node
198
+ when Prism::BlockNode then node.location
199
+ when Prism::LambdaNode then (node.parameters || node).location
200
+ end
201
+ start = node.opening_loc if start && node.is_a?(Prism::LambdaNode) && !node.parameters
202
+ found << [start, node.location] if start && start.start_line == line
203
+ pending.concat(node.compact_child_nodes)
204
+ end
205
+ if found.size == 1
206
+ start, whole = found.first
207
+ [start.start_line, start.start_column, whole.end_line, whole.end_column]
208
+ else
209
+ sequence.to_a[4][:code_location]
210
+ end
211
+ end
212
+
184
213
  # -- the rewrite -------------------------------------------------------
185
214
 
186
215
  # The leaves are the names being read. Everything else keeps its shape:
@@ -262,7 +262,11 @@ class CArray
262
262
  # count, and a truncated data preview.
263
263
  # @return [String]
264
264
  def inspect
265
- return CArray::Inspector.new(self).inspect_string
265
+ # A cell of an object array can hold an expression over the array, whose
266
+ # inspect shows the array again: that is refused as cyclic, as reading a
267
+ # cell that holds the array is.
268
+ return CArray::Inspector.new(self).inspect_string unless object?
269
+ refuse_reentry("inspect") { CArray::Inspector.new(self).inspect_string }
266
270
  end
267
271
 
268
272
  # @overload inspect_full
@@ -290,7 +294,8 @@ class CArray
290
294
  #
291
295
  # @return [String]
292
296
  def inspect_full
293
- return CArray::Inspector.new(self).inspect_string(abbrev: false)
297
+ return CArray::Inspector.new(self).inspect_string(abbrev: false) unless object?
298
+ refuse_reentry("inspect_full") { CArray::Inspector.new(self).inspect_string(abbrev: false) }
294
299
  end
295
300
 
296
301
  private
@@ -71,6 +71,38 @@ class CAIterator
71
71
  end
72
72
  end
73
73
 
74
+ # A member that keeps what it built from its source's values or mask (a
75
+ # padded copy, the views over it, counts of the cells present) names those
76
+ # instance variables here. They are dropped when a public method of the
77
+ # member is entered from outside, so every call reads the source as it is
78
+ # then; within one call the helpers it runs share them. What follows from
79
+ # the source's shape alone is kept for good.
80
+ def self.read_source_per_call (*state)
81
+ names = public_instance_methods(false)
82
+ prepend(Module.new do
83
+ module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
84
+ private def forget_source_state
85
+ #{state.inspect}.each { |v| remove_instance_variable(v) if instance_variable_defined?(v) }
86
+ end
87
+ RUBY
88
+ names.each do |name|
89
+ module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
90
+ def #{name} (...)
91
+ return super if @reading_source
92
+ forget_source_state
93
+ @reading_source = true
94
+ begin
95
+ super
96
+ ensure
97
+ @reading_source = false
98
+ end
99
+ end
100
+ RUBY
101
+ end
102
+ end)
103
+ end
104
+ private_class_method :read_source_per_call
105
+
74
106
  def self.reduce_keywords (op, kw) # :nodoc:
75
107
  if kw.key?(:axis)
76
108
  raise ArgumentError, "#{op}: unknown keyword: :axis (the iterator sets the axes)"
data/lib/carray/lazy.rb CHANGED
@@ -330,9 +330,28 @@ class CArray
330
330
  # branch uses C99 `remainder`, round-half-to-even).
331
331
  }.freeze
332
332
 
333
+ # A Face answers an operator by its own values (a label, an instant, a
334
+ # String) and refuses what it can't answer; the lazy kernels read the
335
+ # storage and see neither. With a Face among the operands the operator is
336
+ # the eager one, on the lazy operands computed.
337
+ module LazyDispatch
338
+ private
339
+
340
+ def face_among_operands? (*operands)
341
+ operands.any? { |o| o.is_a?(CArray) && o.face? }
342
+ end
343
+
344
+ def computed_operand (operand)
345
+ ( operand.is_a?(CArray) && operand.lazy_view? ) ? operand.copy : operand
346
+ end
347
+ end
348
+
333
349
  CArray.send(:define_named_methods, LazyDispatch, LAZY_BINOP_OP_IDS, template: <<~'RUBY')
334
350
  def __NAME__(other)
335
351
  if lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
352
+ if face_among_operands?(self, other)
353
+ return computed_operand(self).__send__(__method__, computed_operand(other))
354
+ end
336
355
  CABinOp.__build__(self, other, __VALUE__)
337
356
  else
338
357
  super(other)
@@ -365,12 +384,6 @@ class CArray
365
384
  alias_method word, op
366
385
  end
367
386
 
368
- # `pow` is the canonical C method (defined by rb_define_method in
369
- # ext/ca_op_ipower.c) that `**` is rb_define_alias'd to. The `**`
370
- # dispatch in this file does not reach `pow` through the C alias, so
371
- # `a.lazy.pow(2)` would fall to eager; `pow` is aliased again here.
372
- alias_method :pow, :**
373
-
374
387
  # `fmax` / `fmin` are rb_define_alias'd to `pmax` / `pmin` at Init
375
388
  # time (the same C alias); they are aliased again beside them, in
376
389
  # LazyDispatch.
@@ -403,6 +416,10 @@ class CArray
403
416
  (op2.is_a?(CArray) && op2.lazy_view?) ||
404
417
  (op3.is_a?(CArray) && op3.lazy_view?)
405
418
  if lazy_receiver
419
+ if face_among_operands?(self, op2, op3)
420
+ return computed_operand(self).__send__(__method__, computed_operand(op2),
421
+ computed_operand(op3))
422
+ end
406
423
  CATriOp.__build__(self, op2, op3, __VALUE__)
407
424
  else
408
425
  super(op2, op3)
@@ -435,6 +452,10 @@ class CArray
435
452
  # is already fast. The lazy-substrate gap was Float/Complex only.
436
453
  module LazyDispatch
437
454
  def **(other)
455
+ if ( lazy_view? || (other.is_a?(CArray) && other.lazy_view?) ) &&
456
+ face_among_operands?(self, other)
457
+ return computed_operand(self) ** computed_operand(other)
458
+ end
438
459
  if other.is_a?(Integer) && (float? || complex?) &&
439
460
  (lazy_view? || (other.is_a?(CArray) && other.lazy_view?))
440
461
  CABinOp.__build__(self, CScalar.int64.tap { |s| s[0] = other },
@@ -447,6 +468,14 @@ class CArray
447
468
  end
448
469
  end
449
470
 
471
+ # `pow` is the canonical C method (defined by rb_define_method in
472
+ # ext/ca_op_ipower.c) that `**` is rb_define_alias'd to. The `**`
473
+ # dispatch in this file does not reach `pow` through the C alias, so
474
+ # `a.lazy.pow(2)` would fall to eager; `pow` is aliased again here.
475
+ # An alias copies the method as it is at that moment, so this follows
476
+ # the integer-exponent `**` above, not the template one.
477
+ alias_method :pow, :**
478
+
450
479
  # coerce: when self is a lazy view and the scalar appears on the LEFT
451
480
  # (e.g. `2 * a.lazy`), Ruby's Numeric#* calls a.lazy.coerce(2). The
452
481
  # default coerce would unwrap the lazy-ness via eager scalar promotion;
@@ -468,6 +497,9 @@ class CArray
468
497
  # equality returning bool, not element-wise. Element-wise equality is
469
498
  # `eq` / `feq`.
470
499
  #
500
+ # A comparison with UNDEF goes to the eager method, which answers `eq` /
501
+ # `ne` from the mask and refuses the others.
502
+ #
471
503
  # `feq` is arity 1 in eager (compile-time FLT_EPSILON / DBL_EPSILON);
472
504
  # we mirror that here. Runtime eps is a future extension (struct field
473
505
  # already reserved).
@@ -491,7 +523,12 @@ class CArray
491
523
 
492
524
  CArray.send(:define_named_methods, LazyDispatch, LAZY_BINCMP_OP_IDS, template: <<~'RUBY')
493
525
  def __NAME__(other)
494
- if lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
526
+ if UNDEF.equal?(other)
527
+ super(other)
528
+ elsif lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
529
+ if face_among_operands?(self, other)
530
+ return computed_operand(self).__send__(__method__, computed_operand(other))
531
+ end
495
532
  CABinCmp.__build__(self, other, __VALUE__)
496
533
  else
497
534
  super(other)
@@ -510,7 +547,12 @@ class CArray
510
547
 
511
548
  CArray.send(:define_named_methods, LazyDispatch, LAZY_BINCMP_TOL_OP_IDS, template: <<~'RUBY')
512
549
  def __NAME__(other, tol)
513
- if lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
550
+ if UNDEF.equal?(other)
551
+ super(other, tol)
552
+ elsif lazy_view? || (other.is_a?(CArray) && other.lazy_view?)
553
+ if face_among_operands?(self, other)
554
+ return computed_operand(self).__send__(__method__, computed_operand(other), tol)
555
+ end
514
556
  CABinCmp.__build__(self, other, __VALUE__, tol.to_f.abs)
515
557
  else
516
558
  super(other, tol)
@@ -598,6 +640,8 @@ class CAMonOp
598
640
  op_id = __op_id__
599
641
  if op_id >= CAMonOp::CAST_BASE
600
642
  "cast_#{CArray.data_type_name(op_id - CAMonOp::CAST_BASE)}"
643
+ elsif op_id == CAMonOp::OP_BYTE_SWAP
644
+ "swap_bytes" # a view of its own, not a lazy operation by name
601
645
  else
602
646
  (OP_NAMES[op_id] || "op_#{op_id}").to_s
603
647
  end
@@ -45,14 +45,14 @@ class CArray
45
45
  "unmask: pass either a constant fill value or method:, not both"
46
46
  end
47
47
  held = gap_fill(method, axis, "unmask")
48
- # Copy the filled values in place (a Face takes a Face of its own kind
49
- # as it is).
50
- value[] = held.value
51
- if held.has_mask?
52
- self.mask = held.mask # residual leading/trailing mask
53
- else
54
- super() # fully filled: drop the mask
55
- end
48
+ # Write only the cells the fill gave a value (a Face takes a Face of
49
+ # its own kind as it is). Writing every cell would let a view that
50
+ # reaches one cell twice send that cell's unfilled copy back last,
51
+ # masking it again.
52
+ return self unless has_mask?
53
+ filled = is_masked
54
+ filled = filled.and(held.is_not_masked) if held.has_mask?
55
+ self[filled] = held[filled] if filled.any
56
56
  return self
57
57
  end
58
58
  fill.equal?(MASK_FILL_UNSET) ? super() : super(fill)
@@ -161,7 +161,12 @@ class CArray
161
161
  end
162
162
 
163
163
  # Fill the masked cells of this 1-D array by linear interpolation against
164
- # the coordinate `x` (same length, ascending over the present cells).
164
+ # the coordinate `x` (same length). `x` need not be in order: the present
165
+ # cells are read as one series in the order of their `x`, so a frame whose
166
+ # index restarts for each station interpolates across the stations by
167
+ # time (interpolate one station at a time to keep them apart). Present
168
+ # cells sharing an `x` keep their row order. A cell whose `x` is masked,
169
+ # NaN or infinite has no place on the line: it is not used and not filled.
165
170
  # The one rule both the core (x = cell position) and CAFrame (x = index)
166
171
  # use: present cells come back exactly as they are, in this array's own
167
172
  # type; a masked cell inside the span of the present ones takes the
@@ -170,17 +175,34 @@ class CArray
170
175
  # NaN or Inf stored in a present cell is left alone. A Face goes through
171
176
  # its own linear_fetch, which keeps its unit and rounds to its grid.
172
177
  def linear_fill_along (x)
173
- present = is_not_masked
178
+ # A cell whose coordinate is masked, NaN or infinite has no place on the
179
+ # line: it is no point to interpolate between, and is not filled.
180
+ placed = x.is_not_masked
181
+ placed &= x.is_finite if !x.face? && x.float?
182
+ placed = placed.strip_mask(false)
183
+ present = is_not_masked & placed
174
184
  # Fewer than two valid points -> nothing to interpolate between; leave
175
185
  # every masked cell masked.
176
186
  return copy if present.count(true) < 2
177
- addr = x[present].linear_section(x) # valid positions -> monotonic grid
187
+ grid = x[present]
188
+ order = nil
189
+ if ( grid[1..-1] < grid[0..-2] ).any
190
+ order = grid.sort_addr # stable: ties keep their row order
191
+ grid = grid[order]
192
+ end
193
+ addr = grid.linear_section(x) # valid positions -> monotonic grid
194
+ fill = (is_masked & placed & addr.is_finite).strip_mask(false)
178
195
  if face?
179
- # The Face's linear_fetch already masks the exterior (out of range) and
180
- # lands on its own grid.
181
- return self[present].linear_fetch(addr)
196
+ # The Face's linear_fetch lands on its own grid; only the cells to fill
197
+ # take it, so a present cell comes back as it is.
198
+ known = self[present]
199
+ known = known[order] if order
200
+ out = copy
201
+ out[fill] = known.linear_fetch(addr)[fill] if fill.any
202
+ return out
182
203
  end
183
204
  v = value[present]
205
+ v = v[order] if order
184
206
  # A complex value interpolates its real and imaginary parts.
185
207
  interp = v.complex? ? v.linear_fetch(addr) : v.float64.linear_fetch(addr)
186
208
  # An integer array takes the nearest integer, half away from zero (as a
@@ -189,9 +211,7 @@ class CArray
189
211
  interp = interp.round if integer?
190
212
  # Built from the values alone, so a fully filled result carries no mask.
191
213
  out = value.copy
192
- masked = is_masked
193
- fill = masked & addr.is_finite
194
- exterior = masked & fill.not
214
+ exterior = is_masked & fill.not
195
215
  out[fill] = interp[fill] if fill.any
196
216
  out[exterior] = UNDEF if exterior.any
197
217
  out
@@ -171,7 +171,7 @@ class CAMeld
171
171
  # more elaborate (min_count / fill_value / keep_axis / multi-axis /
172
172
  # mask propagation) punts to super.
173
173
  def meld_reduce_fast_path_ok?(kw)
174
- return false unless CArray.__fast_paths_enabled__
174
+ return false unless CArray.__fast_paths_enabled__(:meld_reduce)
175
175
  # The mask of a meld is a meld built in C, with no parents list.
176
176
  return false unless parents
177
177
  return false unless (kw.keys - [:axis]).empty?
@@ -41,11 +41,12 @@ class CArray
41
41
  data_type = ca.empty? ? CA_OBJECT : CArray.result_type(*ca)
42
42
  end
43
43
  out = template(data_type)
44
+ dst = out.scalar? ? out.flatten : out # a CScalar takes no index
44
45
  choices.each_with_index do |v, i|
45
46
  s = self.eq(i)
46
- out[s] = v.is_a?(CArray) ? v[s] : v
47
+ dst[s] = !v.is_a?(CArray) ? v : v.scalar? ? v.flatten[s] : v[s]
47
48
  end
48
- out[self.is_masked] = UNDEF if has_mask?
49
+ dst[self.is_masked] = UNDEF if has_mask?
49
50
  text ? CAString.wrap(out) : out
50
51
  end
51
52
 
@@ -226,8 +226,15 @@ class CArray
226
226
  # set for not matching self. Same rule the scalar operand of a
227
227
  # comparison follows: a String standing in for a cell is padded to the
228
228
  # cell's width.
229
+ # A String longer than the cell keeps its length: self is read at its
230
+ # width instead (NUL-padded), so "abX" is in no two-byte cell.
231
+ w = elems.map { |v| v.is_a?(String) ? v.bytesize : 0 }.max.to_i
232
+ me = self
233
+ if w > bytes
234
+ me = is_a?(CAFixlenString) ? to_fixlen_string(bytes: w) : to_type(CA_FIXLEN, bytes: w)
235
+ end
229
236
  # coerce: a fixlen or object set holds values of self's kind (see above)
230
- [self, elems.to_ca.to_type(data_type, bytes: bytes)]
237
+ [me, elems.to_ca.to_type(data_type, bytes: me.bytes)]
231
238
  elsif data_type == CA_OBJECT
232
239
  # coerce: an object set holds values of self's kind
233
240
  [self, elems.to_ca.to_type(data_type)]
@@ -58,7 +58,9 @@ class CArray
58
58
  raise ArgumentError,
59
59
  "coordinate vector #{k} should be 1-D (got #{a.ndim}-D)"
60
60
  end
61
- a
61
+ # A CScalar takes no index, so a scalar axis is read as the
62
+ # one-cell array it stands for.
63
+ a.is_a?(CScalar) ? a.reshape(1) : a
62
64
  end
63
65
 
64
66
  ndim = axes.size
@@ -7,10 +7,11 @@ class CArray
7
7
 
8
8
  # A fast path answers by a different algorithm what a general path also
9
9
  # answers. A development build defines this in C, false when run with
10
- # CARRAY_GENERAL_PATH=1 so the test suite takes the general paths (see
10
+ # CARRAY_GENERAL_PATH=1 so the test suite takes the general paths, and
11
+ # false for a name listed in CARRAY_SHORTCUTS_OFF (see
11
12
  # ext/carray_internal.h); a release build has no switch.
12
13
  unless respond_to?(:__fast_paths_enabled__)
13
- def self.__fast_paths_enabled__ # :nodoc:
14
+ def self.__fast_paths_enabled__ (name = nil) # :nodoc:
14
15
  true
15
16
  end
16
17
  end
@@ -57,13 +58,14 @@ class CArray
57
58
  # is more naturally (and exactly, for Rational / BigDecimal weights) a
58
59
  # +(self * weights).sum+ composition than a dedicated :object kernel branch.
59
60
  #
60
- # +weights+ is a Ruby Numeric scalar (W-A1) or a same-shape CArray; the 1-D
61
- # axis-broadcast form (W-A2) is not supported for object weights (pass a
62
- # scalar or a full-shape weight), matching CArray's explicit-broadcast rule.
63
- # +opts+ is the trailing options Hash (axis:/keep_axis:/min_count:/
64
- # fill_value:) forwarded verbatim by the C escape; it goes straight to +sum+.
61
+ # +weights+ takes the forms the numeric kernel takes: a scalar (or a
62
+ # one-element CArray), an array of the shape of the reduced axes (taken in
63
+ # ascending axis order), or an array of self.shape. +opts+ is the trailing
64
+ # options Hash (axis:/keep_axis:/min_count:/fill_value:) forwarded verbatim
65
+ # by the C escape; it goes straight to +sum+.
65
66
  def wsum_object (weights, opts = {}) # :nodoc:
66
- (self * weights).sum(**opts)
67
+ weights = weights_over_self_shape(weights, opts[:axis], "wsum")
68
+ return (self * weights).sum(**opts)
67
69
  end
68
70
  private :wsum_object
69
71
 
@@ -76,6 +78,7 @@ class CArray
76
78
  def wmean_object (weights, opts = {}) # :nodoc:
77
79
  axis = opts[:axis]
78
80
  keep_axis = opts[:keep_axis]
81
+ weights = weights_over_self_shape(weights, axis, "wmean")
79
82
  prod = self * weights
80
83
  wv = (weights.is_a?(CArray) ? weights : self * 0 + weights).copy
81
84
  wv[prod.is_masked] = UNDEF if prod.has_mask?
@@ -97,6 +100,42 @@ class CArray
97
100
  end
98
101
  private :wmean_object
99
102
 
103
+ # The weights of wsum / wmean as a scalar or an array of self.shape. A
104
+ # CScalar is a scalar; any other array, one with a single element included,
105
+ # is held to the shapes. An array of the shape of the reduced axes (in ascending axis order) is
106
+ # repeated along the kept axes; any other shape is refused, with the same
107
+ # message as the numeric kernel.
108
+ def weights_over_self_shape (weights, axis, name) # :nodoc:
109
+ unless weights.is_a?(CArray)
110
+ return weights
111
+ end
112
+ if weights.scalar?
113
+ w = weights[0]
114
+ # A boolean cell reads as true / false; as a weight it is 1 / 0.
115
+ w = (w ? 1 : 0) if weights.data_type == CA_BOOLEAN && !UNDEF.equal?(w)
116
+ return w
117
+ end
118
+ if weights.shape == self.shape
119
+ return weights
120
+ end
121
+ axes = case axis
122
+ when nil then (0...self.ndim).to_a
123
+ when Integer then [axis]
124
+ else axis
125
+ end
126
+ axes = axes.map { |a| ( a < 0 ) ? a + self.ndim : a }.sort
127
+ reduced_shape = axes.map { |a| self.shape[a] }
128
+ unless weights.shape == reduced_shape
129
+ raise ArgumentError,
130
+ "#{name}: weights of shape #{weights.shape.inspect} not accepted " \
131
+ "(expected a scalar, the shape of the reduced axes " \
132
+ "#{reduced_shape.inspect}, or self.shape #{self.shape.inspect})"
133
+ end
134
+ spread = self.shape.each_index.map { |i| axes.include?(i) ? self.shape[i] : 1 }
135
+ return weights.reshape(*spread).broadcast_to(*self.shape)
136
+ end
137
+ private :weights_over_self_shape
138
+
100
139
  # Define on +mod+ one method per name in +names+, each a `def` that passes
101
140
  # its own name and its arguments to the private method +to+:
102
141
  #
@@ -252,12 +252,20 @@ class CASegmentIterator < CAIterator
252
252
  return axis_by_masked_copy(axis, :variance) if axis
253
253
  m = moments
254
254
  return per_segment(core_reduce_kind(:variance)) { |s| s.variance } unless m
255
+ centred_variance(m, 1)
256
+ end
257
+
258
+ # The kernel divides the centred sum of squares by count - ddof once, as
259
+ # the core reduction does; scaling the sample variance by (n-1)/n instead
260
+ # rounds twice and lands an ulp away.
261
+ def centred_variance (m, ddof)
255
262
  cnt = m[:count]
256
- means = m[:sum] / cnt.float64 # per-segment mean (garbage where count 0/1,
257
- out = CArray.float64(@k) # ignored by the kernel's n<2 guards)
258
- grouped.send(:__reduceat_variance__, @offsets, means, cnt, out)
263
+ means = m[:sum] / cnt.float64 # per-segment mean (garbage where the count
264
+ out = CArray.float64(@k) # is too small; the kernel masks those)
265
+ grouped.send(:__reduceat_variance__, @offsets, means, cnt, out, ddof)
259
266
  out
260
267
  end
268
+ private :centred_variance
261
269
 
262
270
  # @overload stddev
263
271
  # Returns per-segment SAMPLE standard deviation (ddof=1) as float64.
@@ -329,9 +337,17 @@ class CASegmentIterator < CAIterator
329
337
  # with the distinctness of `CArray#nunique` (every NaN is one value,
330
338
  # -0.0 is 0.0). A segment with no present value counts 0.
331
339
  # @return [CArray]
340
+ # Coding every cell costs the same for every cell, and the segment-by-
341
+ # segment way the same for every segment, so long segments are faster one
342
+ # by one: from an average of about 100 cells (1M cells on an M2 Max).
343
+ NUNIQUE_KERNEL_MAX_LENGTH = 100
344
+
332
345
  def nunique (axis: nil)
333
346
  return axis_by_core(axis, :nunique) if axis
334
- return per_segment(CA_INT64) { |s| s.nunique } unless CArray.__fast_paths_enabled__
347
+ if ! CArray.__fast_paths_enabled__(:segment_nunique) ||
348
+ grouped.elements >= NUNIQUE_KERNEL_MAX_LENGTH * @k
349
+ return per_segment(CA_INT64) { |s| s.nunique }
350
+ end
335
351
  out = CArray.int64(@k)
336
352
  return out if grouped.elements.zero?
337
353
  # Each cell's value becomes a code, so a distinct (segment, value) pair is
@@ -375,11 +391,7 @@ class CASegmentIterator < CAIterator
375
391
  return axis_by_masked_copy(axis, :variancep) if axis
376
392
  m = moments
377
393
  return per_segment(core_reduce_kind(:variancep)) { |s| s.variancep } unless m
378
- cnt = m[:count]
379
- vp = variance * (cnt - 1).float64 / cnt.float64
380
- one = cnt.eq(1) # one value: defined, unlike variance --
381
- vp[one] = (m[:sum] - m[:sum])[one] # 0.0, or NaN for a NaN or an infinity
382
- vp
394
+ centred_variance(m, 0) # one value: 0.0, or NaN for a NaN or an infinity
383
395
  end
384
396
 
385
397
  # @overload stddevp
@@ -472,9 +484,8 @@ class CASegmentIterator < CAIterator
472
484
  end
473
485
 
474
486
  # @overload wmean(weights)
475
- # Per-segment weighted mean as float64, matching `CArray#wmean`. Empty
476
- # segment -> MASKED; a present segment whose weights sum to zero -> NaN
477
- # (core's 0/0 contract).
487
+ # Per-segment weighted mean as float64, matching `CArray#wmean`. An empty
488
+ # segment, and one whose weights sum to zero, is masked.
478
489
  # @param weights [CArray]
479
490
  # @return [CArray]
480
491
  def wmean (weights, axis: nil)
@@ -745,7 +756,7 @@ class CASegmentIterator < CAIterator
745
756
  # cumcount in int64 -- which is not the core's answer for every data type
746
757
  # (the core counts a boolean's running sum in uint64).
747
758
  def kernel_scans? (op, answer)
748
- return false unless CArray.__fast_paths_enabled__
759
+ return false unless CArray.__fast_paths_enabled__(:segment_scan_kernel)
749
760
  return false if @value.face? || answer.face? || @value.complex?
750
761
  kernel_type =
751
762
  case op
@@ -902,9 +913,17 @@ class CASegmentIterator < CAIterator
902
913
  # segment's first or last slot and kept when it falls inside the segment.
903
914
  # A read-only Face cannot take UNDEF cell by cell, so its answers are
904
915
  # collected segment by segment, as its other reductions are.
916
+ #
917
+ # The running minimum or maximum costs the same for every cell, and the
918
+ # segment-by-segment way the same for every segment, so long segments
919
+ # are faster one by one: from an average of about 300 cells for :first
920
+ # and 2000 for :last (1M cells on an M2 Max).
921
+ EDGE_KERNEL_MAX_LENGTH = { first: 300, last: 2000 }.freeze
922
+
905
923
  def edge_values (op)
906
924
  n = grouped.elements
907
- if n.zero? || (@value.face? && ! face_output?) || ! CArray.__fast_paths_enabled__
925
+ if n.zero? || (@value.face? && ! face_output?) || ! CArray.__fast_paths_enabled__(:segment_edge_kernel) ||
926
+ n >= EDGE_KERNEL_MAX_LENGTH.fetch(op) * @k
908
927
  return per_segment(core_reduce_kind(op)) { |s| s.public_send(op) }
909
928
  end
910
929
  slot = CArray.int64(n).seq!
@@ -951,7 +970,7 @@ class CASegmentIterator < CAIterator
951
970
  # CARRAY_GENERAL_PATH set reduces every group by the core instead, which
952
971
  # is how the two are compared.
953
972
  def kernel_reads? (data_type = grouped.data_type)
954
- MONOID_TYPES.include?(data_type) && CArray.__fast_paths_enabled__
973
+ MONOID_TYPES.include?(data_type) && CArray.__fast_paths_enabled__(:segment_reduce_kernel)
955
974
  end
956
975
 
957
976
  def moments
@@ -987,7 +1006,7 @@ class CASegmentIterator < CAIterator
987
1006
  def all_any
988
1007
  return @all_any if defined?(@all_any)
989
1008
  @all_any =
990
- if grouped.data_type == CA_BOOLEAN && CArray.__fast_paths_enabled__
1009
+ if grouped.data_type == CA_BOOLEAN && CArray.__fast_paths_enabled__(:segment_all_any_kernel)
991
1010
  a = CArray.boolean(@k)
992
1011
  o = CArray.boolean(@k)
993
1012
  grouped.send(:__reduceat_all_any__, @offsets, a, o)