carray 3.0.3 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +264 -0
  3. data/README.md +3 -2
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_descriptor.h +27 -24
  6. data/ext/ca_axis_dispatch.c +1 -6
  7. data/ext/ca_axis_group.c +27 -27
  8. data/ext/ca_binop_dispatch.c +1 -29
  9. data/ext/ca_binop_dispatch.h +0 -1
  10. data/ext/ca_categorical_iterator.c +205 -134
  11. data/ext/ca_compare.c +7 -18
  12. data/ext/ca_group_iter.c +16 -17
  13. data/ext/ca_kernel_iterator.c +1234 -1994
  14. data/ext/ca_kernel_iterator.h +391 -437
  15. data/ext/ca_monop_dispatch.c +82 -59
  16. data/ext/ca_monop_dispatch.h +1 -0
  17. data/ext/ca_obj_array.c +96 -39
  18. data/ext/ca_obj_bincmp.c +24 -21
  19. data/ext/ca_obj_binop.c +15 -26
  20. data/ext/ca_obj_bitarray.c +3 -2
  21. data/ext/ca_obj_bitfield.c +5 -6
  22. data/ext/ca_obj_block.c +18 -9
  23. data/ext/ca_obj_byte_swap.c +5 -6
  24. data/ext/ca_obj_const_string.c +173 -19
  25. data/ext/ca_obj_face.c +238 -57
  26. data/ext/ca_obj_face.h +19 -0
  27. data/ext/ca_obj_fake.c +15 -29
  28. data/ext/ca_obj_farray.c +6 -3
  29. data/ext/ca_obj_field.c +17 -5
  30. data/ext/ca_obj_fixlen_string.c +2 -3
  31. data/ext/ca_obj_grid.c +406 -132
  32. data/ext/ca_obj_meld.c +172 -34
  33. data/ext/ca_obj_moncmp.c +23 -8
  34. data/ext/ca_obj_monop.c +67 -57
  35. data/ext/ca_obj_object.c +157 -51
  36. data/ext/ca_obj_record.c +12 -7
  37. data/ext/ca_obj_reduce.c +13 -8
  38. data/ext/ca_obj_refer.c +25 -27
  39. data/ext/ca_obj_remap.c +30 -9
  40. data/ext/ca_obj_repeat.c +9 -10
  41. data/ext/ca_obj_roll.c +6 -5
  42. data/ext/ca_obj_select.c +14 -9
  43. data/ext/ca_obj_select_axis.c +15 -47
  44. data/ext/ca_obj_shift.c +11 -1
  45. data/ext/ca_obj_stack.c +179 -34
  46. data/ext/ca_obj_stride.c +162 -115
  47. data/ext/ca_obj_string.c +71 -4
  48. data/ext/ca_obj_tile.c +6 -5
  49. data/ext/ca_obj_time.c +22 -6
  50. data/ext/ca_obj_timedelta.c +22 -6
  51. data/ext/ca_obj_transpose.c +5 -3
  52. data/ext/ca_obj_triop.c +23 -32
  53. data/ext/ca_obj_window.c +190 -26
  54. data/ext/ca_sweep_engine.c +2 -2
  55. data/ext/caframe_csv_reader.c +80 -1
  56. data/ext/carray.h +181 -75
  57. data/ext/carray_access.c +184 -96
  58. data/ext/carray_address_basis.c +343 -87
  59. data/ext/carray_attribute.c +89 -6
  60. data/ext/carray_bincount.c +10 -8
  61. data/ext/carray_broadcast.c +13 -8
  62. data/ext/carray_build_flags.h +3 -0
  63. data/ext/carray_call_cfunc.c +108 -0
  64. data/ext/carray_cast.c +142 -24
  65. data/ext/carray_cast_func.rb +46 -6
  66. data/ext/carray_class.c +4 -3
  67. data/ext/carray_config.h +28 -0
  68. data/ext/carray_conversion.c +16 -7
  69. data/ext/carray_copy.c +51 -14
  70. data/ext/carray_core.c +147 -55
  71. data/ext/carray_count.c +50 -20
  72. data/ext/carray_element.c +76 -33
  73. data/ext/carray_factorize.c +118 -64
  74. data/ext/carray_generate.c +191 -50
  75. data/ext/carray_histogram.c +62 -17
  76. data/ext/carray_hold.c +12 -15
  77. data/ext/carray_index_classifier.c +388 -176
  78. data/ext/carray_internal.h +336 -29
  79. data/ext/carray_kernels_bincmp.c +4453 -0
  80. data/ext/carray_kernels_binop.c +11368 -0
  81. data/ext/carray_kernels_init.c +1734 -0
  82. data/ext/carray_kernels_moncmp.c +2104 -0
  83. data/ext/carray_kernels_monop.c +18369 -0
  84. data/ext/carray_kernels_reduce_aggregate.c +23818 -0
  85. data/ext/carray_kernels_reduce_boolean.c +406 -0
  86. data/ext/carray_kernels_reduce_cumulative.c +10086 -0
  87. data/ext/carray_kernels_reduce_extreme.c +17634 -0
  88. data/ext/carray_kernels_reduce_variance.c +3849 -0
  89. data/ext/carray_kernels_scan.c +4302 -0
  90. data/ext/carray_kernels_search.c +33829 -0
  91. data/ext/carray_kernels_sort.c +10906 -0
  92. data/ext/carray_kernels_triop.c +1864 -0
  93. data/ext/carray_key_windows.c +216 -0
  94. data/ext/carray_lazy.c +170 -111
  95. data/ext/carray_mask.c +292 -72
  96. data/ext/carray_median_percentile.c +82 -28
  97. data/ext/carray_memory_view.c +669 -561
  98. data/ext/carray_operator.c +124 -47
  99. data/ext/carray_order.c +384 -76
  100. data/ext/carray_partition.c +28 -27
  101. data/ext/carray_random.c +67 -13
  102. data/ext/carray_round.c +273 -0
  103. data/ext/carray_scatter.c +63 -38
  104. data/ext/carray_segment.c +3 -3
  105. data/ext/carray_slab.c +91 -28
  106. data/ext/carray_sort.c +35 -35
  107. data/ext/carray_struct.c +19 -18
  108. data/ext/carray_test.c +26 -5
  109. data/ext/carray_undef.c +4 -0
  110. data/ext/carray_utils.c +246 -45
  111. data/ext/mk_call_cfunc.rb +19 -2
  112. data/ext/mkkernel.rb +198 -240
  113. data/ext/ruby_carray.c +10 -4
  114. data/ext/version.h +4 -4
  115. data/lib/carray/attribute.rb +126 -22
  116. data/lib/carray/autoload_carray.rb +8 -31
  117. data/lib/carray/autoload_method_extension.rb +14 -5
  118. data/lib/carray/axis_group.rb +255 -106
  119. data/lib/carray/basics.rb +78 -10
  120. data/lib/carray/bincount_nd.rb +12 -2
  121. data/lib/carray/block_iterator.rb +126 -23
  122. data/lib/carray/boolean_reduce.rb +17 -17
  123. data/lib/carray/categorical.rb +165 -43
  124. data/lib/carray/categorical_iterator.rb +130 -27
  125. data/lib/carray/complex.rb +5 -5
  126. data/lib/carray/conditional.rb +67 -18
  127. data/lib/carray/const_string.rb +14 -5
  128. data/lib/carray/construct.rb +15 -11
  129. data/lib/carray/data_type_extension.rb +1 -0
  130. data/lib/carray/distance_limit.rb +77 -0
  131. data/lib/carray/fixlen_string.rb +7 -1
  132. data/lib/carray/frame/column_verbs.rb +70 -0
  133. data/lib/carray/frame/convert.rb +69 -0
  134. data/lib/carray/frame/csv_parser.rb +111 -17
  135. data/lib/carray/frame/describe.rb +3 -2
  136. data/lib/carray/frame/frame.rb +246 -21
  137. data/lib/carray/frame/group.rb +510 -95
  138. data/lib/carray/frame/io.rb +115 -28
  139. data/lib/carray/frame/join.rb +312 -51
  140. data/lib/carray/frame/records.rb +83 -7
  141. data/lib/carray/frame/reshape.rb +42 -35
  142. data/lib/carray/frame/sort.rb +7 -19
  143. data/lib/carray/frame/verbs.rb +188 -48
  144. data/lib/carray/frame/windows.rb +117 -0
  145. data/lib/carray/frame.rb +2 -0
  146. data/lib/carray/fuse_source.rb +11 -4
  147. data/lib/carray/fusion.rb +40 -16
  148. data/lib/carray/histogram.rb +6 -1
  149. data/lib/carray/inspect.rb +31 -8
  150. data/lib/carray/iterator.rb +85 -7
  151. data/lib/carray/key_windows.rb +620 -0
  152. data/lib/carray/lazy.rb +182 -183
  153. data/lib/carray/mask_gap_fill.rb +63 -57
  154. data/lib/carray/meld_reduce.rb +12 -14
  155. data/lib/carray/methods/bin.rb +26 -18
  156. data/lib/carray/methods/bincount.rb +16 -0
  157. data/lib/carray/methods/bit_string.rb +1 -1
  158. data/lib/carray/methods/choose.rb +16 -3
  159. data/lib/carray/methods/composition.rb +3 -1
  160. data/lib/carray/methods/gather_nd.rb +14 -5
  161. data/lib/carray/methods/insert_block.rb +2 -2
  162. data/lib/carray/methods/is_between.rb +59 -0
  163. data/lib/carray/methods/is_in.rb +53 -36
  164. data/lib/carray/methods/join.rb +1 -6
  165. data/lib/carray/methods/lag.rb +49 -0
  166. data/lib/carray/methods/locate_addr.rb +47 -51
  167. data/lib/carray/methods/mode.rb +1 -1
  168. data/lib/carray/methods/nunique.rb +11 -6
  169. data/lib/carray/methods/resize.rb +3 -3
  170. data/lib/carray/methods/value_counts.rb +4 -11
  171. data/lib/carray/record_order.rb +3 -3
  172. data/lib/carray/rng.rb +3 -0
  173. data/lib/carray/runtime.rb +53 -8
  174. data/lib/carray/segment_iterator.rb +313 -82
  175. data/lib/carray/serialize.rb +24 -10
  176. data/lib/carray/slab_iterator.rb +50 -0
  177. data/lib/carray/string.rb +105 -5
  178. data/lib/carray/string_operation_extension.rb +8 -1
  179. data/lib/carray/struct.rb +9 -21
  180. data/lib/carray/struct_builder.rb +19 -8
  181. data/lib/carray/time.rb +726 -104
  182. data/lib/carray/window_iterator.rb +122 -16
  183. data/lib/carray.rb +7 -0
  184. data/yard-stubs/ca_obj_bitfield.rb +1 -1
  185. data/yard-stubs/ca_obj_field.rb +4 -0
  186. data/yard-stubs/ca_obj_grid.rb +21 -10
  187. data/yard-stubs/ca_obj_refer.rb +2 -1
  188. data/yard-stubs/ca_obj_roll.rb +2 -2
  189. data/yard-stubs/ca_obj_shift.rb +5 -1
  190. data/yard-stubs/ca_obj_tile.rb +1 -1
  191. data/yard-stubs/ca_obj_window.rb +8 -3
  192. data/yard-stubs/carray_access.rb +6 -1
  193. data/yard-stubs/carray_attribute.rb +0 -5
  194. data/yard-stubs/carray_cast.rb +10 -0
  195. data/yard-stubs/carray_conversion.rb +2 -1
  196. data/yard-stubs/carray_copy.rb +4 -2
  197. data/yard-stubs/carray_count.rb +4 -4
  198. data/yard-stubs/carray_element.rb +10 -4
  199. data/yard-stubs/carray_generate.rb +21 -1
  200. data/yard-stubs/carray_mask.rb +26 -2
  201. data/yard-stubs/carray_math.rb +27 -1
  202. data/yard-stubs/carray_median_percentile.rb +5 -2
  203. data/yard-stubs/carray_memory_view.rb +40 -23
  204. data/yard-stubs/carray_order.rb +29 -18
  205. data/yard-stubs/carray_random.rb +13 -4
  206. data/yard-stubs/carray_scatter.rb +14 -9
  207. data/yard-stubs/carray_slab.rb +9 -2
  208. data/yard-stubs/carray_sort.rb +6 -5
  209. data/yard-stubs/carray_test.rb +5 -3
  210. metadata +25 -3
  211. data/lib/carray/arrow_tensor.rb +0 -401
  212. data/lib/carray/table.rb +0 -108
@@ -220,21 +220,9 @@ class AxisGroup
220
220
  # all / any take axis: only, as the core ones do.
221
221
  LOGICAL_REDUCE_KEYWORDS = [:axis].freeze
222
222
 
223
- def self.__check_reduce_keywords__ (op, kw)
224
- allowed = [:all, :any].include?(op.to_sym) ? LOGICAL_REDUCE_KEYWORDS : REDUCE_KEYWORDS
225
- unknown = kw.keys - allowed
226
- unless unknown.empty?
227
- raise ArgumentError,
228
- "#{op}: unknown keyword#{unknown.size > 1 ? "s" : ""}: " +
229
- unknown.map(&:inspect).join(", ")
230
- end
231
- CArray.__min_count_need__(op, kw[:min_count], 0)
232
- nil
233
- end
234
-
235
223
  # A boolean array carrying value's mask: what a count reads, for a value
236
- # whose payload the group kernel cannot read.
237
- def self.__count_stand_in__ (value)
224
+ # whose payload the group kernel cannot read. Called from C.
225
+ private_class_method def self.count_stand_in (value)
238
226
  return value if value.boolean? || value.integer? || value.float?
239
227
  stand_in = CArray.boolean(*value.shape)
240
228
  stand_in.mask = value.mask if value.has_mask?
@@ -244,26 +232,6 @@ class AxisGroup
244
232
  # A count of the mask reads every cell of the group, as the core's does.
245
233
  MASK_COUNTS = [:count, :count_not_masked, :count_masked].freeze
246
234
 
247
- # min_count: masks a group with fewer contributing cells (present cells,
248
- # or every cell for a count of the mask); fill_value: then fills the masked
249
- # cells -- as a core reduction does with each.
250
- def self.__finish_reduction__ (iter, op, kw, result)
251
- need = CArray.__min_count_need__(op, kw[:min_count], 0)
252
- if need > 0
253
- counts = if MASK_COUNTS.include?(op.to_sym)
254
- iter.elements(axis: kw[:axis])
255
- else
256
- iter.count_not_masked(axis: kw[:axis])
257
- end
258
- if result.is_a?(CArray)
259
- result[counts.lt(need)] = UNDEF
260
- elsif counts < need
261
- result = UNDEF
262
- end
263
- end
264
- CArray.__reduce_fill__(result, kw[:fill_value])
265
- end
266
-
267
235
  # Parse an axis: spec into [has_group, fused_band_slot_positions].
268
236
  def self.parse_axis (axis)
269
237
  has_group = false
@@ -383,60 +351,85 @@ class CAGroupIterator
383
351
  # -- arrived as "wrong number of arguments", which reads as a method that
384
352
  # does not take one rather than one whose engine has no such fold. Both are
385
353
  # composed here instead, out of folds the engine does have.
386
- alias __count_present__ count
387
-
388
- # @overload count
389
- # Per-group count of present (non-masked) cells.
390
- # @overload count(UNDEF)
391
- # Per-group count of masked cells, as {#count_masked}.
392
- # @overload count(v)
393
- # Per-group count of cells equal to `v`.
394
- # @return [CArray]
395
- def count (*args, **kw)
396
- return __count_present__(**kw) if args.empty?
397
- if args.size > 1
398
- raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0..1)"
354
+ # They are in a module in front of the engine's own count, all and any,
355
+ # which they reach with super; so are the reductions below that choose
356
+ # between the engine and the core.
357
+ module ComposedFolds
358
+
359
+ # @overload count
360
+ # Per-group count of present (non-masked) cells.
361
+ # @overload count(UNDEF)
362
+ # Per-group count of masked cells, as {#count_masked}.
363
+ # @overload count(v)
364
+ # Per-group count of cells equal to `v`.
365
+ # @return [CArray]
366
+ def count (*args, **kw)
367
+ if args.empty?
368
+ return core_counts?(kw) ? reduce_by_core(:count, kw) : super(**kw)
369
+ end
370
+ if args.size > 1
371
+ raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0..1)"
372
+ end
373
+ v = args.first
374
+ return count_masked(**kw) if v.equal?(UNDEF)
375
+ # eq leaves a masked cell UNDEF, so the sum leaves it out and its
376
+ # min_count: counts the present cells, as the core count(v) does
377
+ regroup(value.eq(v).int64, spec).sum(**kw).int64
399
378
  end
400
- v = args.first
401
- return count_masked(**kw) if v.equal?(UNDEF)
402
- # eq leaves a masked cell UNDEF, so the sum leaves it out and its
403
- # min_count: counts the present cells, as the core count(v) does
404
- self.class.__build__(value.eq(v).int64, spec).sum(**kw).int64
405
- end
406
379
 
407
- alias __all_folded__ all
408
- alias __any_folded__ any
380
+ # @overload all
381
+ # Per-group `all` over a boolean payload.
382
+ # @return [CArray]
383
+ def all (**kw)
384
+ boolean_payload!(:all)
385
+ core_counts?(kw) ? reduce_by_core(:all, kw) : super(**kw)
386
+ end
409
387
 
410
- # @overload all
411
- # Per-group `all` over a boolean payload.
412
- # @return [CArray]
413
- def all (**kw)
414
- boolean_payload!(:all)
415
- __all_folded__(**kw)
416
- end
388
+ # @overload any
389
+ # Per-group `any` over a boolean payload.
390
+ # @return [CArray]
391
+ def any (**kw)
392
+ boolean_payload!(:any)
393
+ core_counts?(kw) ? reduce_by_core(:any, kw) : super(**kw)
394
+ end
417
395
 
418
- # @overload any
419
- # Per-group `any` over a boolean payload.
420
- # @return [CArray]
421
- def any (**kw)
422
- boolean_payload!(:any)
423
- __any_folded__(**kw)
396
+ # @overload count_not_masked
397
+ # Per-group count of present (non-masked) cells.
398
+ # @return [CArray]
399
+ def count_not_masked (**kw)
400
+ core_counts?(kw) ? reduce_by_core(:count_not_masked, kw) : super(**kw)
401
+ end
402
+
403
+ private
404
+
405
+ # Whether a count or all / any of each group is taken from the core, as
406
+ # the other reductions of a payload the engine does not read are. The
407
+ # engine answers these for every payload; with the fast paths off (a
408
+ # development build run with CARRAY_GENERAL_PATH set) the core does --
409
+ # except for a band folded into the count, which has no route but the
410
+ # engine.
411
+ def core_counts? (kw)
412
+ return false if CArray.__fast_paths_enabled__
413
+ has_group, fused = AxisGroup.parse_axis(kw[:axis])
414
+ has_group && fused.empty?
415
+ end
424
416
  end
417
+ prepend ComposedFolds
425
418
 
426
419
  # Per-group classified cell count (mask-independent) = count on the
427
420
  # mask-stripped value, so every classified cell is counted regardless of the
428
421
  # value mask (unlike count / count_not_masked, which count present cells).
429
422
  def elements (**kw)
430
- self.class.__build__(value.value, spec).count(**kw)
423
+ regroup(value.value, spec).count(**kw)
431
424
  end
432
425
 
433
426
  # Per-group count of value-masked cells = elements - count_not_masked.
434
427
  def count_masked (**kw)
435
428
  has_group, = AxisGroup.parse_axis(kw[:axis])
436
429
  return value.count_masked(**kw) unless has_group
437
- AxisGroup.__check_reduce_keywords__(:count_masked, kw)
430
+ check_reduce_keywords(:count_masked, kw)
438
431
  plain = { axis: kw[:axis] }
439
- AxisGroup.__finish_reduction__(self, :count_masked, kw,
432
+ finish_reduction(:count_masked, kw,
440
433
  elements(**plain) - count_not_masked(**plain))
441
434
  end
442
435
 
@@ -445,16 +438,21 @@ class CAGroupIterator
445
438
  # is reduced group by group by the core, through the composite categorical,
446
439
  # so the data type, the Face and a refusal are the core's: a boolean's sum
447
440
  # is uint64, a time column's minimum a time, its sum refused.
448
- [:sum, :accumulate, :prod, :mean, :min, :max,
449
- :variance, :stddev, :variancep, :stddevp].each do |op|
450
- alias_method :"__#{op}_by_kernel__", op
451
- define_method(op) do |**kw|
452
- has_group, = AxisGroup.parse_axis(kw[:axis])
453
- if ! has_group || kernel_reads_value?
454
- __send__(:"__#{op}_by_kernel__", **kw)
455
- else
456
- reduce_by_core(op, kw)
457
- end
441
+ CArray.send(:define_named_methods, ComposedFolds,
442
+ [:sum, :accumulate, :prod, :mean, :min, :max,
443
+ :variance, :stddev, :variancep, :stddevp],
444
+ :scatter_or_core, super_block: true)
445
+
446
+ # The scatter kernel is the block (super).
447
+ private def scatter_or_core (op, args, kw)
448
+ unless args.empty?
449
+ raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0)"
450
+ end
451
+ has_group, fused = AxisGroup.parse_axis(kw[:axis])
452
+ if ! has_group || kernel_reads_value?(fused)
453
+ yield(**kw)
454
+ else
455
+ reduce_by_core(op, kw)
458
456
  end
459
457
  end
460
458
 
@@ -490,7 +488,7 @@ class CAGroupIterator
490
488
  # value|weight mask, so masked cells drop out. Empty group -> 0.0 (identity).
491
489
  # No weighted kernel needed -- it is a plain group-sum of a derived array.
492
490
  def wsum (weights, **kw)
493
- self.class.__build__(value * weights, spec).sum(**kw)
491
+ regroup(value * weights, spec).sum(**kw)
494
492
  end
495
493
 
496
494
  # Weighted mean = Sum(v*w) / Sum(w), both over the combined present-set. The
@@ -501,19 +499,19 @@ class CAGroupIterator
501
499
  def wmean (weights, **kw)
502
500
  has_group, = AxisGroup.parse_axis(kw[:axis])
503
501
  return value.wmean(weights, **kw) unless has_group
504
- AxisGroup.__check_reduce_keywords__(:wmean, kw)
502
+ check_reduce_keywords(:wmean, kw)
505
503
  plain = { axis: kw[:axis] }
506
504
  prod = value * weights
507
- num = self.class.__build__(prod, spec).sum(**plain)
508
- den = self.class.__build__(value * 0 + weights, spec).sum(**plain)
509
- cnt = self.class.__build__(prod, spec).count_not_masked(**plain)
505
+ num = regroup(prod, spec).sum(**plain)
506
+ den = regroup(value * 0 + weights, spec).sum(**plain)
507
+ cnt = regroup(prod, spec).count_not_masked(**plain)
510
508
  # No present pair is UNDEF (an empty group), set on the denominator so
511
509
  # the division never meets an empty group's 0 / 0.
512
510
  den = den.copy
513
511
  den[cnt.eq(0)] = UNDEF
514
512
  # min_count: counts the present pairs, as the core wmean does.
515
- pairs = self.class.__build__(prod, spec)
516
- AxisGroup.__finish_reduction__(pairs, :wmean, kw, num / den)
513
+ pairs = regroup(prod, spec)
514
+ pairs.finish_reduction(:wmean, kw, num / den)
517
515
  end
518
516
 
519
517
  # ---- segment scan: within-group running statistics ---------------------
@@ -524,7 +522,7 @@ class CAGroupIterator
524
522
  # in row-major position order along the grouped axes (per band). Excluded /
525
523
  # source-masked cells are UNDEF (they join no running total). All five route
526
524
  # through the shared scan_op dispatcher, driving the fused C kernel
527
- # __axis_group_scan__ (ext/ca_axis_group.c) with peak O(k) extra. A scan
525
+ # __scan_by_axis_groups__ (ext/ca_axis_group.c) with peak O(k) extra. A scan
528
526
  # cannot fold a band into the statistic (that would collapse an axis), so it
529
527
  # accepts only axis: :group; without :group each delegates to the value's
530
528
  # same-named scan.
@@ -573,6 +571,59 @@ class CAGroupIterator
573
571
  scan_op(:cumcount, kw)
574
572
  end
575
573
 
574
+ # @overload lag(n = 1, axis: :group)
575
+ # Per-group value `n` steps earlier, counting a group's members in
576
+ # row-major position order along the grouped axes (per band), as the
577
+ # scans do; UNDEF where the group has none, so a lag never reaches
578
+ # another group. Without :group it is the value's own `lag` along an
579
+ # axis. A negative `n` looks ahead.
580
+ # @return [CArray] source-shaped
581
+ def lag (n = 1, axis: nil)
582
+ has_group, fused = AxisGroup.parse_axis(axis)
583
+ return value.lag(n, axis: axis) unless has_group
584
+ unless fused.empty?
585
+ raise ArgumentError,
586
+ "axis_group lag: folding a band into a lag (axis: [:group, k]) " \
587
+ "is not supported; a lag preserves the source shape"
588
+ end
589
+ scan_by_core(:lag, n)
590
+ end
591
+
592
+ # @overload strip_mask(method:, axis: :group)
593
+ # Per-group fill of the masked cells, counting a group's members in
594
+ # row-major position order along the grouped axes (per band), as the
595
+ # scans do: `:forward` / `:ffill` or `:backward` / `:bfill`, never across
596
+ # into another group. Without :group it is the value's own
597
+ # `strip_mask(method:, axis:)`.
598
+ # @return [CArray] source-shaped
599
+ def strip_mask (method:, axis: nil)
600
+ has_group, fused = AxisGroup.parse_axis(axis)
601
+ return value.strip_mask(method: method, axis: axis) unless has_group
602
+ CAIterator.hold_backward?(method)
603
+ unless fused.empty?
604
+ raise ArgumentError,
605
+ "axis_group strip_mask: folding a band into a fill (axis: [:group, k]) " \
606
+ "is not supported; a fill preserves the source shape"
607
+ end
608
+ scan_by_core(:strip_mask, method: method)
609
+ end
610
+
611
+ # @overload order(axis: :group, descending: false, method: :ordinal, kind: :quick)
612
+ # Per-group rank of each cell among its group's members (int64, `0` the
613
+ # smallest; per band), as `CArray#order` ranks them; a cell in no group is
614
+ # UNDEF. Without :group it is the value's own `order`.
615
+ # @return [CArray] source-shaped
616
+ def order (axis: nil, **kw)
617
+ has_group, fused = AxisGroup.parse_axis(axis)
618
+ return value.order(axis: axis, **kw) unless has_group
619
+ unless fused.empty?
620
+ raise ArgumentError,
621
+ "axis_group order: folding a band into an order (axis: [:group, k]) " \
622
+ "is not supported; an order preserves the source shape"
623
+ end
624
+ scan_by_core(:order, **kw)
625
+ end
626
+
576
627
  # ---- materialize path: order statistics + generic iteration ------------
577
628
  #
578
629
  # These need every member of a group held together, so (unlike the scatter
@@ -614,6 +665,41 @@ class CAGroupIterator
614
665
  order_stat(:quantile, [], kw)
615
666
  end
616
667
 
668
+ # @overload first(axis: :group)
669
+ # Per-group first present value, in row-major position order along the
670
+ # grouped axes (per band); UNDEF for a group with none. Without :group it
671
+ # is the value's own `first`.
672
+ # @return [CArray]
673
+ # @overload last(axis: :group)
674
+ # Per-group last present value; UNDEF for a group with none.
675
+ # @return [CArray]
676
+ # @overload nunique(axis: :group)
677
+ # Per-group number of distinct present values (int64); 0 for a group with
678
+ # none.
679
+ # @return [CArray]
680
+ #
681
+ # These take no `min_count:` or `fill_value:`, as the core ones do not. A
682
+ # group is gathered whole, so folding a band into one (axis: [:group, k]) is
683
+ # not supported, as for the order statistics.
684
+ CArray.send(:define_named_methods, self, [:first, :last, :nunique], :group_position)
685
+
686
+ private def group_position (op, **kw)
687
+ has_group, fused = AxisGroup.parse_axis(kw[:axis])
688
+ return value.public_send(op, **kw) unless has_group
689
+ unless fused.empty?
690
+ raise NotImplementedError,
691
+ "axis_group: folding a band into #{op} (axis: [:group, k]) is not " \
692
+ "supported; #{op} gathers every member of a group"
693
+ end
694
+ unknown = kw.keys - [:axis]
695
+ unless unknown.empty?
696
+ raise ArgumentError,
697
+ "#{op}: unknown keyword#{unknown.size > 1 ? "s" : ""}: " +
698
+ unknown.map(&:inspect).join(", ")
699
+ end
700
+ composite_order(op, [])
701
+ end
702
+
617
703
  # @overload sort_addr
618
704
  # Per-group sorted flat source addresses (group-major). Any grouping (single
619
705
  # or composite group slots, rank-1 or rank-N, flat or band-preserving) is
@@ -680,8 +766,50 @@ class CAGroupIterator
680
766
  out
681
767
  end
682
768
 
769
+ protected
770
+
771
+ # min_count: masks a group with fewer contributing cells (present cells,
772
+ # or every cell for a count of the mask); fill_value: then fills the masked
773
+ # cells -- as a core reduction does with each. Protected: wmean finishes
774
+ # through the iterator over its products.
775
+ def finish_reduction (op, kw, result)
776
+ need = CArray.__present_cells_needed_per_result_cell__(op, kw[:min_count], 0)
777
+ if need > 0
778
+ counts = if AxisGroup::MASK_COUNTS.include?(op.to_sym)
779
+ elements(axis: kw[:axis])
780
+ else
781
+ count_not_masked(axis: kw[:axis])
782
+ end
783
+ if result.is_a?(CArray)
784
+ result[counts.lt(need)] = UNDEF
785
+ elsif counts < need
786
+ result = UNDEF
787
+ end
788
+ end
789
+ CArray.__fill_undefined_result_cells__(result, kw[:fill_value])
790
+ end
791
+
683
792
  private
684
793
 
794
+ # An unknown keyword is refused before the work is done.
795
+ def check_reduce_keywords (op, kw)
796
+ allowed = [:all, :any].include?(op.to_sym) ? AxisGroup::LOGICAL_REDUCE_KEYWORDS : AxisGroup::REDUCE_KEYWORDS
797
+ unknown = kw.keys - allowed
798
+ unless unknown.empty?
799
+ raise ArgumentError,
800
+ "#{op}: unknown keyword#{unknown.size > 1 ? "s" : ""}: " +
801
+ unknown.map(&:inspect).join(", ")
802
+ end
803
+ CArray.__present_cells_needed_per_result_cell__(op, kw[:min_count], 0)
804
+ nil
805
+ end
806
+
807
+ # The same grouping over another value.
808
+ def regroup (value, spec)
809
+ self.class.send(:build_for, value, spec)
810
+ end
811
+
812
+
685
813
  # Build the composite categorical over the group block (the sub-array spanning
686
814
  # every group axis). Each group slot -- rank-1 or rank-N -- contributes its
687
815
  # per-axis codes; the slot codes are broadcast into the group-block rank (size
@@ -703,7 +831,8 @@ class CAGroupIterator
703
831
  # askable. Asking a one-cell array of the same data type lets the core's own
704
832
  # refusal through, unworded by us.
705
833
  def boolean_payload! (op)
706
- CArray.new(value.data_type, [1], bytes: value.bytes).public_send(op)
834
+ probe = CAIterator.core_probe(value)
835
+ probe.public_send(op) if probe
707
836
  nil
708
837
  end
709
838
 
@@ -730,6 +859,13 @@ class CAGroupIterator
730
859
  [ccat, kdims, gslots, bslots, gaxes]
731
860
  end
732
861
 
862
+ # With no band axis the group block is the whole value, grouped in one
863
+ # pass. With the fast paths off it goes through the band loop instead,
864
+ # which then has one position.
865
+ def whole_block? (bslots)
866
+ bslots.empty? && CArray.__fast_paths_enabled__
867
+ end
868
+
733
869
  # Iterate the band positions of a band-preserving grouping. For each band
734
870
  # position it yields the group block grouped by the composite categorical,
735
871
  # together with two indices: val_idx (source order -- nil across the group
@@ -779,13 +915,17 @@ class CAGroupIterator
779
915
  end
780
916
  return scan_by_core(op) unless kernel_scans?(op)
781
917
  group_axes, bundles, = spec.reduce_plan([])
782
- value.__axis_group_scan__(group_axes, bundles, op)
918
+ value.__scan_by_axis_groups__(group_axes, bundles, op)
783
919
  end
784
920
 
785
921
  # Whether the scatter kernels read this payload: an integer or a float,
786
- # not a Face. They answer there in the core's data types.
787
- def kernel_reads_value?
788
- ! value.face? && (value.integer? || value.float?)
922
+ # not a Face. They answer there in the core's data types. With the fast
923
+ # paths off (a development build run with CARRAY_GENERAL_PATH set) each
924
+ # group is reduced by the core instead -- except a band folded into the
925
+ # statistic (+fused+), which has no route but the kernel.
926
+ def kernel_reads_value? (fused = [])
927
+ return false if value.face? || ! (value.integer? || value.float?)
928
+ CArray.__fast_paths_enabled__ || ! fused.empty?
789
929
  end
790
930
 
791
931
  # Whether the scan kernel gives the core's answer: it reads an integer,
@@ -793,21 +933,22 @@ class CAGroupIterator
793
933
  # float64 (an object in object), cummax / cummin in the payload's data type
794
934
  # and cumcount in int64 -- not the core's answer for a boolean's running sum.
795
935
  def kernel_scans? (op)
936
+ return false unless CArray.__fast_paths_enabled__
796
937
  return false if value.face? || value.complex?
797
938
  return true if value.object? || op == :cumcount
798
- core = CArray.new(value.data_type, [1], bytes: value.bytes).public_send(op).data_type
939
+ core = CAIterator.core_probe(value).public_send(op).data_type
799
940
  kernel = [:cumsum, :cumprod].include?(op) ? CA_FLOAT64 : value.data_type
800
941
  core == kernel
801
942
  end
802
943
 
803
944
  # A scan the kernel does not answer as the core does: each group block is
804
945
  # grouped through the composite categorical, whose scans are the core's.
805
- def scan_by_core (op)
946
+ def scan_by_core (op, *args, **kw)
806
947
  ccat, _kd, gslots, bslots, gaxes = composite_layout
807
- return value.group_by_category(ccat).public_send(op) if bslots.empty?
948
+ return value.group_by_category(ccat).public_send(op, *args, **kw) if whole_block?(bslots)
808
949
  out = nil
809
950
  each_band_block(ccat, gslots, bslots, gaxes) do |val_idx, _oi, _co, gi|
810
- result = gi.public_send(op)
951
+ result = gi.public_send(op, *args, **kw)
811
952
  out ||= blank_of(result, value.shape)
812
953
  out[*val_idx] = result
813
954
  end
@@ -825,8 +966,8 @@ class CAGroupIterator
825
966
  "(axis: [:group, k]) is not supported for a #{value.class} of " \
826
967
  "data type #{value.data_type_name}"
827
968
  end
828
- AxisGroup.__check_reduce_keywords__(op, kw)
829
- AxisGroup.__finish_reduction__(self, op, kw, composite_order(op, []))
969
+ check_reduce_keywords(op, kw)
970
+ finish_reduction(op, kw, composite_order(op, []))
830
971
  end
831
972
 
832
973
  # An array of +shape+ of the same kind as +model+ -- its data type, or its
@@ -836,7 +977,7 @@ class CAGroupIterator
836
977
  out = if model.face? && model.read_only?
837
978
  CArray.new(CA_OBJECT, shape)
838
979
  else
839
- model.__send__(:__blank_like__, shape)
980
+ model.__send__(:blank_like, shape)
840
981
  end
841
982
  out[] = UNDEF
842
983
  out
@@ -855,12 +996,12 @@ class CAGroupIterator
855
996
  "axis_group: folding a band into an order statistic (axis: [:group, k]) " \
856
997
  "is not supported; order statistics gather every member of a group"
857
998
  end
858
- AxisGroup.__check_reduce_keywords__(op, kw)
999
+ check_reduce_keywords(op, kw)
859
1000
  res = composite_order(op, args)
860
1001
  if op == :quantile
861
- res.map { |r| AxisGroup.__finish_reduction__(self, op, kw, r) }
1002
+ res.map { |r| finish_reduction(op, kw, r) }
862
1003
  else
863
- AxisGroup.__finish_reduction__(self, op, kw, res)
1004
+ finish_reduction(op, kw, res)
864
1005
  end
865
1006
  end
866
1007
 
@@ -875,7 +1016,7 @@ class CAGroupIterator
875
1016
  def composite_order (op, args)
876
1017
  ccat, kdims, gslots, bslots, gaxes = composite_layout
877
1018
  nout = (op == :quantile) ? 5 : 1
878
- if bslots.empty?
1019
+ if whole_block?(bslots)
879
1020
  res = value.group_by_category(ccat).public_send(op, *args)
880
1021
  return nout == 1 ? res.reshape(*kdims) : res.map { |r| r.reshape(*kdims) }
881
1022
  end
@@ -930,7 +1071,15 @@ class CAGroupIterator
930
1071
  end
931
1072
  out
932
1073
  end
1074
+
1075
+ # lag and strip_mask(method:) give the source's values moved or held in
1076
+ # place, so the result shows the source's attributes, as CArray#lag and
1077
+ # CArray#strip_mask(method:) do.
1078
+ prepend(Module.new do
1079
+ def lag (*args, **keywords) = value.__send__(:carry_attrs, super)
1080
+ def strip_mask (**keywords) = value.__send__(:carry_attrs, super)
1081
+ end)
933
1082
  end
934
1083
 
935
1084
  require "carray/categorical"
936
- CArray.__register_axis_group_classes__(CACategorical, AxisGroup)
1085
+ CArray.send(:register_axis_group_classes, CACategorical, AxisGroup)
data/lib/carray/basics.rb CHANGED
@@ -13,9 +13,19 @@
13
13
 
14
14
  class CArray
15
15
 
16
+ # The order sort puts two object values in (ca_obj_order_cmp in C): a
17
+ # Float NaN after every other value, NaN ties NaN, other pairs by `<=>`.
18
+ # nil when `<=>` cannot order the pair; the caller says what refused.
19
+ private_class_method def self.object_order_cmp (a, b)
20
+ a_nan = a.is_a?(Float) && a.nan?
21
+ b_nan = b.is_a?(Float) && b.nan?
22
+ return (a_nan ? 1 : 0) - (b_nan ? 1 : 0) if a_nan || b_nan
23
+ a <=> b
24
+ end
25
+
16
26
  # reshape / flatten / transpose! are defined in C
17
27
  # (ext/ca_obj_refer.c and ext/ca_obj_transpose.c). The bare size-1
18
- # insertion primitive is `__insert_axis_size1__` (C); `insert_axis` below
28
+ # insertion primitive is `insert_size_one_axes` (C); `insert_axis` below
19
29
  # is the user-facing method that owns the name and adds `repeat:`.
20
30
 
21
31
  # @overload insert_axis(*positions, repeat: nil)
@@ -58,7 +68,7 @@ class CArray
58
68
 
59
69
  # No repeat: -> plain size-1 insertion. The source-frame C primitive
60
70
  # handles normalization, range check and multiplicity directly.
61
- return __insert_axis_size1__(*flat) if repeat.nil?
71
+ return insert_size_one_axes(*flat) if repeat.nil?
62
72
 
63
73
  # Source frame: each position names the source axis the new axis goes
64
74
  # before. Gaps live in [0, ndim] (ndim = append at end); negatives count
@@ -98,7 +108,7 @@ class CArray
98
108
  final = {}
99
109
  order.each_with_index { |i, k| final[i] = gaps[i] + k }
100
110
 
101
- inter = __insert_axis_size1__(*order.map { |i| gaps[i] })
111
+ inter = insert_size_one_axes(*order.map { |i| gaps[i] })
102
112
  return inter unless order.any? { |i| reps[i] > 1 }
103
113
 
104
114
  shp = inter.shape
@@ -146,7 +156,7 @@ class CArray
146
156
  # out of range (a time array reads its bounds as times, a record array
147
157
  # orders by its declared members, a Face that cannot be ordered refuses),
148
158
  # and its store writes the bound or the fill value into them.
149
- private def __face_clip__ (min, max, fill_value, lfill, ufill)
159
+ private def face_clip (min, max, fill_value, lfill, ufill)
150
160
  lfill = fill_value if lfill.nil?
151
161
  ufill = fill_value if ufill.nil?
152
162
  out = copy
@@ -158,8 +168,8 @@ class CArray
158
168
  # An array of +shape+ of the same kind as self: its data type and cell
159
169
  # width, or its Face over a zero storage. Nothing is written into it, so
160
170
  # it can be built for a read-only Face too.
161
- private def __blank_like__ (shape)
162
- storage = __face_storage__
171
+ private def blank_like (shape)
172
+ storage = face_storage
163
173
  out = CArray.new(storage.data_type, shape, bytes: storage.bytes)
164
174
  face? ? out.face_lift(self) : out
165
175
  end
@@ -310,7 +320,9 @@ class CArray
310
320
  #
311
321
  # `min` may not exceed `max` (in any cell, when a bound is an
312
322
  # array); equal bounds give every unmasked cell that value.
313
- # A NaN bound bounds nothing.
323
+ # A NaN in a bound array leaves that cell unbounded on that side,
324
+ # as `nil` does for the whole side. A bound is never UNDEF or
325
+ # masked: a missing bound is not read as either, so fill it first.
314
326
  #
315
327
  # Boundary is strict `[min, max]`: values equal to a bound
316
328
  # remain unchanged in both the clamped and filled variants.
@@ -325,7 +337,8 @@ class CArray
325
337
  # @param ufill [Object, nil] override above-range fill.
326
338
  # @return [CArray] new CArray with clamped or filled values.
327
339
  # @raise [ArgumentError] when both `min` and `max` are `nil`,
328
- # or when `min` exceeds `max`.
340
+ # when either is `UNDEF` or has a masked cell, or when `min`
341
+ # exceeds `max`.
329
342
  # @example
330
343
  # a.clip(0, 10) # strict clamp
331
344
  # a.clip(0, 10, -1) # both ends -> -1
@@ -334,7 +347,16 @@ class CArray
334
347
  if min.nil? && max.nil?
335
348
  raise ArgumentError, "clip: at least one of (min, max) must be given"
336
349
  end
337
- return __face_clip__(min, max, fill_value, lfill, ufill) if face?
350
+ [min, max].each do |bound|
351
+ undefined = UNDEF.equal?(bound) ||
352
+ (bound.is_a?(CArray) && bound.any_masked?) ||
353
+ (bound.is_a?(Array) && bound.flatten.any? { |v| UNDEF.equal?(v) })
354
+ next unless undefined
355
+ raise ArgumentError, "clip: a bound can not be UNDEF or masked " \
356
+ "(fill the bound first; a NaN bound leaves its cell " \
357
+ "unbounded, and UNDEF as the fill masks the cells out of range)"
358
+ end
359
+ return face_clip(min, max, fill_value, lfill, ufill) if face?
338
360
  if !min.nil? && !max.nil?
339
361
  reversed = if min.is_a?(CArray) then min.gt(max).any
340
362
  elsif max.is_a?(CArray) then max.lt(min).any
@@ -350,7 +372,7 @@ class CArray
350
372
  ufill = fill_value if ufill.nil?
351
373
 
352
374
  if lfill.nil? && ufill.nil?
353
- return __clip_ki__(min, max) if !min.nil? && !max.nil?
375
+ return clamp_between(min, max) if !min.nil? && !max.nil?
354
376
  return pmax(min) if max.nil?
355
377
  return pmin(max)
356
378
  end
@@ -549,3 +571,49 @@ class Enumerator::ArithmeticSequence
549
571
  return CA_OBJECT(to_a)
550
572
  end
551
573
  end
574
+
575
+ class CArray
576
+ # map_slab / reduce_slab over a Face answer in that Face, as its other
577
+ # reductions do: each block value is written into a blank of the Face
578
+ # (which reads a CATime::Element, a string, a Face array ...) and handed
579
+ # to the slab engine as the storage it became; the result is lifted back
580
+ # at the end. A read-only Face has no blank to write into, so its values
581
+ # are collected as the surface objects they are. A `data_type:` from the
582
+ # caller is the caller's, and passes through.
583
+ module FaceSlabResult
584
+ def map_slab (**kw, &block)
585
+ return super unless face? && kw[:data_type].nil? && block
586
+ return super(**kw, data_type: :object, &block) if read_only?
587
+ out = super(**kw, data_type: face_storage.data_type) { |slab|
588
+ face_slab_storage(block.call(slab), slab.shape)
589
+ }
590
+ out.face_lift(self)
591
+ end
592
+
593
+ def reduce_slab (**kw, &block)
594
+ return super unless face? && kw[:data_type].nil? && block
595
+ return super(**kw, data_type: :object, &block) if read_only?
596
+ fiber = kw.key?(:init)
597
+ init = kw.delete(:init)
598
+ out = super(**kw, data_type: face_storage.data_type) { |slab|
599
+ value = if fiber
600
+ slab.to_a.flatten.inject(init) { |acc, e| block.call(acc, e) }
601
+ else
602
+ block.call(slab)
603
+ end
604
+ face_slab_storage(value, [1])[0]
605
+ }
606
+ out.face_lift(self)
607
+ end
608
+
609
+ private
610
+
611
+ def face_slab_storage (value, shape)
612
+ cell = blank_like(shape)
613
+ cell[] = value
614
+ cell.__send__(:face_storage)
615
+ end
616
+ end
617
+
618
+ prepend FaceSlabResult
619
+ end