carray 3.0.1 → 3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +824 -0
  3. data/README.md +5 -6
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +258 -4
  6. data/ext/ca_axis_group.c +368 -184
  7. data/ext/ca_binop_dispatch.c +54 -32
  8. data/ext/ca_binop_dispatch.h +1 -0
  9. data/ext/ca_categorical_iterator.c +90 -41
  10. data/ext/ca_compare.c +8 -2
  11. data/ext/ca_for_buffer.h +21 -68
  12. data/ext/ca_group_iter.c +27 -11
  13. data/ext/ca_kernel_iterator.c +960 -245
  14. data/ext/ca_kernel_iterator.h +352 -181
  15. data/ext/ca_monop_dispatch.c +4 -4
  16. data/ext/ca_monop_dispatch.h +4 -0
  17. data/ext/ca_obj_array.c +194 -42
  18. data/ext/ca_obj_bincmp.c +152 -121
  19. data/ext/ca_obj_binop.c +107 -210
  20. data/ext/ca_obj_bitarray.c +37 -24
  21. data/ext/ca_obj_bitfield.c +95 -62
  22. data/ext/ca_obj_block.c +5 -4
  23. data/ext/ca_obj_byte_swap.c +40 -40
  24. data/ext/ca_obj_const_string.c +180 -50
  25. data/ext/ca_obj_face.c +428 -62
  26. data/ext/ca_obj_face.h +107 -47
  27. data/ext/ca_obj_fake.c +21 -50
  28. data/ext/ca_obj_farray.c +1 -0
  29. data/ext/ca_obj_field.c +16 -3
  30. data/ext/ca_obj_fixlen_string.c +39 -26
  31. data/ext/ca_obj_grid.c +181 -124
  32. data/ext/ca_obj_meld.c +235 -55
  33. data/ext/ca_obj_moncmp.c +61 -61
  34. data/ext/ca_obj_monop.c +125 -107
  35. data/ext/ca_obj_object.c +551 -233
  36. data/ext/ca_obj_record.c +51 -16
  37. data/ext/ca_obj_reduce.c +5 -8
  38. data/ext/ca_obj_refer.c +117 -33
  39. data/ext/ca_obj_remap.c +57 -17
  40. data/ext/ca_obj_repeat.c +21 -5
  41. data/ext/ca_obj_roll.c +115 -11
  42. data/ext/ca_obj_select.c +88 -75
  43. data/ext/ca_obj_select_axis.c +58 -60
  44. data/ext/ca_obj_shift.c +22 -102
  45. data/ext/ca_obj_stack.c +223 -123
  46. data/ext/ca_obj_stride.c +267 -35
  47. data/ext/ca_obj_string.c +25 -22
  48. data/ext/ca_obj_tile.c +32 -16
  49. data/ext/ca_obj_time.c +23 -25
  50. data/ext/ca_obj_timedelta.c +21 -23
  51. data/ext/ca_obj_transpose.c +12 -3
  52. data/ext/ca_obj_triop.c +110 -117
  53. data/ext/ca_obj_window.c +157 -29
  54. data/ext/ca_op_ipower.c +25 -13
  55. data/ext/ca_op_powi.h +57 -30
  56. data/ext/ca_pow5_table.h +656 -0
  57. data/ext/ca_rng_normal.h +42 -0
  58. data/ext/ca_rng_xoshiro256pp.h +105 -0
  59. data/ext/ca_sort_kernels.h +70 -0
  60. data/ext/ca_sweep_engine.c +385 -198
  61. data/ext/ca_sweep_engine.h +31 -23
  62. data/ext/ca_triop_dispatch.c +4 -1
  63. data/ext/ca_triop_dispatch.h +5 -2
  64. data/ext/caframe_csv_reader.c +463 -0
  65. data/ext/carray.h +90 -28
  66. data/ext/carray_access.c +487 -231
  67. data/ext/carray_address_basis.c +731 -0
  68. data/ext/carray_attribute.c +45 -3
  69. data/ext/carray_bincount.c +32 -2
  70. data/ext/carray_broadcast.c +49 -17
  71. data/ext/carray_call_cfunc.c +667 -511
  72. data/ext/carray_cast.c +1556 -247
  73. data/ext/carray_cast_func.rb +34 -21
  74. data/ext/carray_conversion.c +197 -73
  75. data/ext/carray_copy.c +84 -38
  76. data/ext/carray_core.c +591 -119
  77. data/ext/carray_count.c +148 -96
  78. data/ext/carray_element.c +18 -27
  79. data/ext/carray_factorize.c +604 -291
  80. data/ext/carray_generate.c +76 -40
  81. data/ext/carray_histogram.c +152 -94
  82. data/ext/carray_hold.c +2 -1
  83. data/ext/carray_index_classifier.c +63 -96
  84. data/ext/carray_internal.h +400 -1
  85. data/ext/carray_lazy.c +834 -79
  86. data/ext/carray_loop.c +88 -66
  87. data/ext/carray_mask.c +705 -324
  88. data/ext/carray_math_kernel.h +1 -0
  89. data/ext/carray_median_percentile.c +263 -63
  90. data/ext/carray_memory_view.c +254 -248
  91. data/ext/carray_operator.c +1093 -363
  92. data/ext/carray_order.c +133 -69
  93. data/ext/carray_partition.c +102 -56
  94. data/ext/carray_random.c +582 -204
  95. data/ext/carray_scatter.c +308 -65
  96. data/ext/carray_segment.c +257 -0
  97. data/ext/carray_slab.c +183 -70
  98. data/ext/carray_sort.c +235 -240
  99. data/ext/carray_sort_kernel.c +8 -1
  100. data/ext/carray_struct.c +99 -21
  101. data/ext/carray_test.c +167 -64
  102. data/ext/carray_time_text.c +1182 -0
  103. data/ext/carray_utils.c +363 -98
  104. data/ext/extconf.rb +27 -2
  105. data/ext/mk_call_cfunc.rb +103 -118
  106. data/ext/mk_pow5_table.rb +35 -0
  107. data/ext/mkkernel.rb +1349 -851
  108. data/ext/ruby_carray.c +17 -2
  109. data/ext/version.h +4 -4
  110. data/ext/version.rb +4 -2
  111. data/lib/carray/attribute.rb +39 -0
  112. data/lib/carray/autoload_carray.rb +24 -3
  113. data/lib/carray/autoload_method_extension.rb +18 -0
  114. data/lib/carray/axis_group.rb +245 -20
  115. data/lib/carray/basics.rb +95 -25
  116. data/lib/carray/bincount_nd.rb +6 -0
  117. data/lib/carray/block_iterator.rb +200 -54
  118. data/lib/carray/boolean_reduce.rb +21 -29
  119. data/lib/carray/categorical.rb +244 -59
  120. data/lib/carray/categorical_iterator.rb +391 -757
  121. data/lib/carray/complex.rb +36 -29
  122. data/lib/carray/conditional.rb +94 -7
  123. data/lib/carray/const_string.rb +177 -27
  124. data/lib/carray/construct.rb +40 -0
  125. data/lib/carray/data_type_extension.rb +3 -0
  126. data/lib/carray/data_type_limits.rb +91 -0
  127. data/lib/carray/fixlen_string.rb +1 -1
  128. data/lib/carray/frame/concat.rb +157 -6
  129. data/lib/carray/frame/convert.rb +13 -7
  130. data/lib/carray/frame/csv_parser.rb +590 -76
  131. data/lib/carray/frame/describe.rb +79 -0
  132. data/lib/carray/frame/frame.rb +205 -35
  133. data/lib/carray/frame/group.rb +199 -40
  134. data/lib/carray/frame/io.rb +452 -60
  135. data/lib/carray/frame/join.rb +134 -46
  136. data/lib/carray/frame/records.rb +54 -19
  137. data/lib/carray/frame/reshape.rb +448 -0
  138. data/lib/carray/frame/sort.rb +26 -2
  139. data/lib/carray/frame/verbs.rb +726 -56
  140. data/lib/carray/frame.rb +2 -0
  141. data/lib/carray/fuse_source.rb +111 -2
  142. data/lib/carray/fusion.rb +229 -38
  143. data/lib/carray/inspect.rb +71 -20
  144. data/lib/carray/iterator.rb +157 -2
  145. data/lib/carray/lazy.rb +54 -67
  146. data/lib/carray/mask_gap_fill.rb +66 -50
  147. data/lib/carray/math.rb +32 -19
  148. data/lib/carray/meld_reduce.rb +40 -7
  149. data/lib/carray/methods/align_addr.rb +8 -4
  150. data/lib/carray/methods/bincount.rb +2 -2
  151. data/lib/carray/methods/bit_string.rb +14 -3
  152. data/lib/carray/methods/composition.rb +16 -10
  153. data/lib/carray/methods/discovery_along.rb +74 -0
  154. data/lib/carray/methods/factorize.rb +50 -0
  155. data/lib/carray/methods/gather_nd.rb +13 -2
  156. data/lib/carray/methods/insert_block.rb +9 -14
  157. data/lib/carray/methods/is_in.rb +130 -6
  158. data/lib/carray/methods/join.rb +1 -1
  159. data/lib/carray/methods/locate_addr.rb +235 -21
  160. data/lib/carray/methods/mask_duplicates.rb +35 -1
  161. data/lib/carray/methods/mode.rb +18 -1
  162. data/lib/carray/methods/nunique.rb +22 -1
  163. data/lib/carray/methods/repeat.rb +91 -0
  164. data/lib/carray/methods/resize.rb +21 -23
  165. data/lib/carray/methods/snap.rb +34 -12
  166. data/lib/carray/methods/unique.rb +52 -3
  167. data/lib/carray/record_order.rb +208 -0
  168. data/lib/carray/rng.rb +86 -0
  169. data/lib/carray/runtime.rb +14 -11
  170. data/lib/carray/segment_iterator.rb +990 -0
  171. data/lib/carray/serialize.rb +197 -27
  172. data/lib/carray/slab_iterator.rb +109 -40
  173. data/lib/carray/stack.rb +10 -4
  174. data/lib/carray/string_operation_extension.rb +9 -4
  175. data/lib/carray/struct.rb +142 -51
  176. data/lib/carray/struct_builder.rb +110 -14
  177. data/lib/carray/table.rb +2 -2
  178. data/lib/carray/time.rb +535 -105
  179. data/lib/carray/window_iterator.rb +259 -82
  180. data/lib/carray.rb +4 -2
  181. data/yard-stubs/ca_obj_array.rb +4 -4
  182. data/yard-stubs/ca_obj_bitarray.rb +7 -0
  183. data/yard-stubs/ca_obj_bitfield.rb +23 -12
  184. data/yard-stubs/ca_obj_block.rb +2 -7
  185. data/yard-stubs/ca_obj_fake.rb +1 -1
  186. data/yard-stubs/ca_obj_field.rb +1 -1
  187. data/yard-stubs/ca_obj_refer.rb +3 -3
  188. data/yard-stubs/ca_obj_roll.rb +0 -1
  189. data/yard-stubs/ca_obj_stride.rb +10 -7
  190. data/yard-stubs/ca_obj_transpose.rb +1 -1
  191. data/yard-stubs/ca_obj_window.rb +11 -4
  192. data/yard-stubs/carray_access.rb +6 -5
  193. data/yard-stubs/carray_broadcast.rb +2 -2
  194. data/yard-stubs/carray_cast.rb +78 -43
  195. data/yard-stubs/carray_conversion.rb +13 -1
  196. data/yard-stubs/carray_copy.rb +8 -3
  197. data/yard-stubs/carray_core.rb +3 -83
  198. data/yard-stubs/carray_count.rb +16 -2
  199. data/yard-stubs/carray_lazy.rb +210 -1
  200. data/yard-stubs/carray_loop.rb +4 -1
  201. data/yard-stubs/carray_mask.rb +34 -10
  202. data/yard-stubs/carray_math.rb +1499 -4
  203. data/yard-stubs/carray_median_percentile.rb +26 -8
  204. data/yard-stubs/carray_order.rb +43 -82
  205. data/yard-stubs/carray_scatter.rb +10 -3
  206. data/yard-stubs/carray_segment.rb +72 -0
  207. data/yard-stubs/carray_slab.rb +11 -8
  208. data/yard-stubs/carray_sort.rb +50 -8
  209. data/yard-stubs/carray_test.rb +10 -0
  210. data/yard-stubs/carray_utils.rb +0 -26
  211. metadata +19 -19
  212. data/ext/ca_for_each_element.h +0 -239
  213. data/ext/carray_build_flags.h +0 -3
  214. data/ext/carray_config.h +0 -28
  215. data/ext/carray_kernels_bincmp.c +0 -4446
  216. data/ext/carray_kernels_binop.c +0 -11001
  217. data/ext/carray_kernels_init.c +0 -1131
  218. data/ext/carray_kernels_map.c +0 -3467
  219. data/ext/carray_kernels_moncmp.c +0 -2097
  220. data/ext/carray_kernels_monop.c +0 -18313
  221. data/ext/carray_kernels_reduce_aggregate.c +0 -25837
  222. data/ext/carray_kernels_reduce_boolean.c +0 -330
  223. data/ext/carray_kernels_reduce_cumulative.c +0 -14593
  224. data/ext/carray_kernels_reduce_extreme.c +0 -16948
  225. data/ext/carray_kernels_reduce_variance.c +0 -3910
  226. data/ext/carray_kernels_scan.c +0 -3693
  227. data/ext/carray_kernels_search.c +0 -32138
  228. data/ext/carray_kernels_sort.c +0 -10626
  229. data/ext/carray_kernels_triop.c +0 -1392
data/lib/carray/basics.rb CHANGED
@@ -80,31 +80,31 @@ class CArray
80
80
  Array.new(flat.length, repeat)
81
81
  end
82
82
 
83
- # Validate each value. A positive Integer only; nil is not a valid
84
- # per-axis repeat.
85
- reps.each do |r|
86
- unless r.is_a?(Integer)
87
- raise ArgumentError,
88
- "insert_axis: repeat must be a positive Integer, got #{r.inspect}"
83
+ # Validate each value. A positive Integer only; nil is not a valid
84
+ # per-axis repeat.
85
+ reps.each do |r|
86
+ unless r.is_a?(Integer)
87
+ raise ArgumentError,
88
+ "insert_axis: repeat must be a positive Integer, got #{r.inspect}"
89
+ end
90
+ raise ArgumentError, "insert_axis: repeat count must be >= 1" if r < 1
89
91
  end
90
- raise ArgumentError, "insert_axis: repeat count must be >= 1" if r < 1
91
- end
92
92
 
93
- # Final output layout: stable order by (gap, argument index) keeps
94
- # same-gap axes in argument order; the k-th inserted axis lands at output
95
- # position gap + k. This output position only drives broadcast_to; the
96
- # insertion itself always goes through the source-frame primitive.
97
- order = (0...flat.length).sort_by { |i| [gaps[i], i] }
98
- final = {}
99
- order.each_with_index { |i, k| final[i] = gaps[i] + k }
93
+ # Final output layout: stable order by (gap, argument index) keeps
94
+ # same-gap axes in argument order; the k-th inserted axis lands at output
95
+ # position gap + k. This output position only drives broadcast_to; the
96
+ # insertion itself always goes through the source-frame primitive.
97
+ order = (0...flat.length).sort_by { |i| [gaps[i], i] }
98
+ final = {}
99
+ order.each_with_index { |i, k| final[i] = gaps[i] + k }
100
100
 
101
- inter = __insert_axis_size1__(*order.map { |i| gaps[i] })
102
- return inter unless order.any? { |i| reps[i] > 1 }
101
+ inter = __insert_axis_size1__(*order.map { |i| gaps[i] })
102
+ return inter unless order.any? { |i| reps[i] > 1 }
103
103
 
104
- shp = inter.shape
105
- order.each { |i| shp[final[i]] = reps[i] if reps[i] > 1 }
106
- inter.broadcast_to(*shp)
107
- end
104
+ shp = inter.shape
105
+ order.each { |i| shp[final[i]] = reps[i] if reps[i] > 1 }
106
+ inter.broadcast_to(*shp)
107
+ end
108
108
 
109
109
  # @overload drop_axis
110
110
  # Returns a view of `self` with every size-1 axis dropped.
@@ -142,6 +142,28 @@ end
142
142
  return template(:boolean) { 1 }
143
143
  end
144
144
 
145
+ # A Face is clipped in its own terms: its comparisons decide which cells are
146
+ # out of range (a time array reads its bounds as times, a record array
147
+ # orders by its declared members, a Face that cannot be ordered refuses),
148
+ # and its store writes the bound or the fill value into them.
149
+ private def __face_clip__ (min, max, fill_value, lfill, ufill)
150
+ lfill = fill_value if lfill.nil?
151
+ ufill = fill_value if ufill.nil?
152
+ out = copy
153
+ out[self < min] = lfill.nil? ? min : lfill unless min.nil?
154
+ out[self > max] = ufill.nil? ? max : ufill unless max.nil?
155
+ out
156
+ end
157
+
158
+ # An array of +shape+ of the same kind as self: its data type and cell
159
+ # width, or its Face over a zero storage. Nothing is written into it, so
160
+ # it can be built for a read-only Face too.
161
+ private def __blank_like__ (shape)
162
+ storage = __face_storage__
163
+ out = CArray.new(storage.data_type, shape, bytes: storage.bytes)
164
+ face? ? out.face_lift(self) : out
165
+ end
166
+
145
167
  # Sub-region copy (paste / crop), the old C `ca_paste` / `ca_clip` pair.
146
168
  # `clip` is reserved for value clamping (clamp to a min/max range), so
147
169
  # the read-out side is `crop`. Kept together as a pair (crop is rare, but paste is reached
@@ -159,6 +181,10 @@ end
159
181
  # @raise [ArgumentError] when `offset.length != self.ndim`.
160
182
  def paste (offset, src)
161
183
  raise ArgumentError, "offset length must equal ndim" if offset.length != ndim
184
+ # A zero-length source covers no cell, so there is nothing to write --
185
+ # and the empty range it would ask for is not a window CAWindow can
186
+ # build. This is what lets concatenate / mosaic take an empty piece.
187
+ return self if src.elements.zero?
162
188
  ranges = offset.each_with_index.map { |o, i| o...(o + src.shape[i]) }
163
189
  self.window(*ranges)[] = src
164
190
  self
@@ -246,6 +272,31 @@ end
246
272
 
247
273
  # @!endgroup
248
274
 
275
+ # An Integer shift count on an integer array means what it means to
276
+ # Integer#<< and #>>: negative shifts the other way, and a count past the
277
+ # width shifts every bit out. The kernels do that for a count of the
278
+ # array's own type; this brings a count the type cannot hold -- -1 for a
279
+ # uint8 array, 300 for an int8 one -- to one it can.
280
+ module IntegerShiftCount
281
+ UNSIGNED = [CA_UINT8, CA_UINT16, CA_UINT32, CA_UINT64].freeze
282
+
283
+ def << (other)
284
+ return super unless other.is_a?(Integer) && integer? && ! face?
285
+ w = bytes * 8
286
+ return self >> -other if other < 0 && UNSIGNED.include?(data_type)
287
+ super(other.clamp(-w, w))
288
+ end
289
+
290
+ def >> (other)
291
+ return super unless other.is_a?(Integer) && integer? && ! face?
292
+ w = bytes * 8
293
+ return self << -other if other < 0 && UNSIGNED.include?(data_type)
294
+ super(other.clamp(-w, w))
295
+ end
296
+ end
297
+ # Prepended by carray/lazy, after it has wrapped << and >>: prepended
298
+ # here, the method lazy aliases as the eager form would be this one.
299
+
249
300
  # @overload clip(min, max = nil, fill_value = nil, lfill: nil, ufill: nil)
250
301
  # Returns `self` with every element clamped to `[min, max]`.
251
302
  #
@@ -254,7 +305,12 @@ end
254
305
  # `fill_value` (or `lfill` / `ufill`) is given, out-of-range
255
306
  # cells are replaced by the fill instead of clamped -- pass
256
307
  # `UNDEF` to mask that end. `fill_value` is sugar for symmetric
257
- # dual-fill; `lfill` / `ufill` override per side.
308
+ # dual-fill; `lfill` / `ufill` override per side, and a side
309
+ # given no fill is still clamped.
310
+ #
311
+ # `min` may not exceed `max` (in any cell, when a bound is an
312
+ # array); equal bounds give every unmasked cell that value.
313
+ # A NaN bound bounds nothing.
258
314
  #
259
315
  # Boundary is strict `[min, max]`: values equal to a bound
260
316
  # remain unchanged in both the clamped and filled variants.
@@ -268,7 +324,8 @@ end
268
324
  # @param lfill [Object, nil] override below-range fill.
269
325
  # @param ufill [Object, nil] override above-range fill.
270
326
  # @return [CArray] new CArray with clamped or filled values.
271
- # @raise [ArgumentError] when both `min` and `max` are `nil`.
327
+ # @raise [ArgumentError] when both `min` and `max` are `nil`,
328
+ # or when `min` exceeds `max`.
272
329
  # @example
273
330
  # a.clip(0, 10) # strict clamp
274
331
  # a.clip(0, 10, -1) # both ends -> -1
@@ -277,6 +334,16 @@ end
277
334
  if min.nil? && max.nil?
278
335
  raise ArgumentError, "clip: at least one of (min, max) must be given"
279
336
  end
337
+ return __face_clip__(min, max, fill_value, lfill, ufill) if face?
338
+ if !min.nil? && !max.nil?
339
+ reversed = if min.is_a?(CArray) then min.gt(max).any
340
+ elsif max.is_a?(CArray) then max.lt(min).any
341
+ else min > max
342
+ end
343
+ if reversed
344
+ raise ArgumentError, "clip: min must be less than or equal to max"
345
+ end
346
+ end
280
347
 
281
348
  # `fill_value` as a single argument is sugar applied to both ends; kwargs override.
282
349
  lfill = fill_value if lfill.nil?
@@ -288,9 +355,12 @@ end
288
355
  return pmin(max)
289
356
  end
290
357
 
358
+ # A side without a fill still clamps.
291
359
  out = self.copy
292
- out[:lt, min] = lfill unless min.nil? || lfill.nil?
293
- out[:gt, max] = ufill unless max.nil? || ufill.nil?
360
+ out = out.pmax(min) if !min.nil? && lfill.nil?
361
+ out = out.pmin(max) if !max.nil? && ufill.nil?
362
+ out[self.lt(min)] = lfill unless min.nil? || lfill.nil?
363
+ out[self.gt(max)] = ufill unless max.nil? || ufill.nil?
294
364
  out
295
365
  end
296
366
 
@@ -269,6 +269,12 @@ class CArray
269
269
  term = strides_ext[k] == 1 ? b : b * strides_ext[k]
270
270
  ravel = ravel.nil? ? term : ravel + term
271
271
  end
272
+ # A NaN weight drops its sample, as the fiber kernel and histogram
273
+ # do; bincount itself would add the NaN in.
274
+ if tweights && tweights.float?
275
+ nan = tweights.is_nan
276
+ tweights = tweights.mask_where(nan) if nan.any
277
+ end
272
278
  chunk_counts = ravel.bincount(weights: tweights, length: total_ext)
273
279
  @full_counts[] = @full_counts + chunk_counts.reshape(*@full_counts.shape)
274
280
  else
@@ -167,19 +167,44 @@ class CABlockIterator < CAIterator
167
167
  # below a full tile, so `min_count:` marks them UNDEF with no masking. The
168
168
  # output data type is seeded from the first region (all regions share it).
169
169
  def assemble
170
- outs = nil
170
+ grids = nil
171
+ faces = nil
172
+ single = true
171
173
  each_region do |strip_ranges, tiles, out_ranges|
172
174
  view = @source[*strip_ranges].block_view(*tiles)
173
175
  res = yield(view, tiles)
174
- if res.is_a?(Array)
175
- outs ||= res.map { |r| CArray.new(r.data_type, @shape) }
176
- res.each_index { |k| outs[k][*out_ranges] = res[k] }
177
- else
178
- outs ||= CArray.new(res.data_type, @shape)
179
- outs[*out_ranges] = res
180
- end
176
+ single = false if res.is_a?(Array)
177
+ res = [res] unless res.is_a?(Array)
178
+ faces ||= res.map { |r| r.face? ? r : nil }
179
+ parts = res.map { |r| r.face? ? face_grid_part(r) : r }
180
+ grids ||= parts.map { |r| CArray.new(r.data_type, @shape, bytes: r.bytes) }
181
+ parts.each_index { |k| grids[k][*out_ranges] = parts[k] }
181
182
  end
182
- outs
183
+ unless grids
184
+ # An empty source has no region. A view with unit tiles has no cells
185
+ # either; reducing it gives each output's type for the empty grid.
186
+ res = yield(@source.block_view(*[1] * @sndim), [1] * @sndim)
187
+ single = !res.is_a?(Array)
188
+ res = [res] unless res.is_a?(Array)
189
+ faces = res.map { |r| r.face? ? r : nil }
190
+ grids = res.map { |r| r = face_grid_part(r) if r.face?; CArray.new(r.data_type, @shape, bytes: r.bytes) }
191
+ end
192
+ outs = grids.each_index.map { |k| faces[k] ? face_grid(grids[k], faces[k]) : grids[k] }
193
+ single ? outs[0] : outs
194
+ end
195
+
196
+ # A region result that is a Face (a CATime mean, a CATimedelta stddev) is
197
+ # assembled on its storage and the Face is put back on the whole grid;
198
+ # assembling the Face itself would copy its surface bytes into an array that
199
+ # no longer knows it is one. A CAConstString's storage points into its own
200
+ # buffer, which the regions do not share, so it is assembled as strings.
201
+ def face_grid_part (r)
202
+ r.is_a?(CAConstString) ? r.to_string.parent : r.parent
203
+ end
204
+
205
+ def face_grid (grid, face)
206
+ return CArray.const_string(grid, encoding: face.encoding) if face.is_a?(CAConstString)
207
+ grid.face_lift(face)
183
208
  end
184
209
 
185
210
  # Fold every region with a single core reduction `op` (the tier-1 shape).
@@ -197,19 +222,92 @@ class CABlockIterator < CAIterator
197
222
  # data type, mask, empty / all-masked (identity vs UNDEF) and epsilon-close
198
223
  # contracts unchanged. `min_count:` / `fill_value:` pass straight to the core.
199
224
 
200
- # @overload sum(min_count: nil, fill_value: nil)
201
- # Per-tile sum. @return [CArray] tile-grid shaped
202
- # @overload accumulate(min_count: nil, fill_value: nil)
203
- # Per-tile sum kept in the source's own data type, wrapping at its width,
204
- # as the core `accumulate` does -- `sum` answers in the type the core
205
- # promotes to (float64 for integers), so a tile count over `uint8` cells
206
- # stays one byte wide instead of eight.
225
+ # @!method sum(min_count: nil, fill_value: nil)
226
+ # Per-tile sum.
227
+ # @param min_count [Integer, nil] fewest cells that must be present
228
+ # for a result; a piece with fewer comes back masked.
229
+ # @param fill_value [Object, nil] value to put in place of a masked
230
+ # result instead of leaving it masked.
231
+ # @return [CArray] tile-grid shaped
232
+ # @!method accumulate(min_count: nil, fill_value: nil)
233
+ # Per-tile sum kept in the source's own data type, wrapping at its
234
+ # width, as the core `accumulate` does -- `sum` answers in the type
235
+ # the core promotes to (float64 for integers).
236
+ # @param min_count [Integer, nil] fewest cells that must be present
237
+ # for a result; a piece with fewer comes back masked.
238
+ # @param fill_value [Object, nil] value to put in place of a masked
239
+ # result instead of leaving it masked.
240
+ # @return [CArray] tile-grid shaped
241
+ # @!method prod(min_count: nil, fill_value: nil)
242
+ # Per-tile product.
243
+ # @param min_count [Integer, nil] fewest cells that must be present
244
+ # for a result; a piece with fewer comes back masked.
245
+ # @param fill_value [Object, nil] value to put in place of a masked
246
+ # result instead of leaving it masked.
247
+ # @return [CArray] tile-grid shaped
248
+ # @!method mean(min_count: nil, fill_value: nil)
249
+ # Per-tile arithmetic mean.
250
+ # @param min_count [Integer, nil] fewest cells that must be present
251
+ # for a result; a piece with fewer comes back masked.
252
+ # @param fill_value [Object, nil] value to put in place of a masked
253
+ # result instead of leaving it masked.
254
+ # @return [CArray] tile-grid shaped
255
+ # @!method min(min_count: nil, fill_value: nil)
256
+ # Per-tile minimum.
257
+ # @param min_count [Integer, nil] fewest cells that must be present
258
+ # for a result; a piece with fewer comes back masked.
259
+ # @param fill_value [Object, nil] value to put in place of a masked
260
+ # result instead of leaving it masked.
261
+ # @return [CArray] tile-grid shaped
262
+ # @!method max(min_count: nil, fill_value: nil)
263
+ # Per-tile maximum.
264
+ # @param min_count [Integer, nil] fewest cells that must be present
265
+ # for a result; a piece with fewer comes back masked.
266
+ # @param fill_value [Object, nil] value to put in place of a masked
267
+ # result instead of leaving it masked.
268
+ # @return [CArray] tile-grid shaped
269
+ # @!method variance(min_count: nil, fill_value: nil)
270
+ # Per-tile sample variance (divisor `n - 1`).
271
+ # @param min_count [Integer, nil] fewest cells that must be present
272
+ # for a result; a piece with fewer comes back masked.
273
+ # @param fill_value [Object, nil] value to put in place of a masked
274
+ # result instead of leaving it masked.
275
+ # @return [CArray] tile-grid shaped
276
+ # @!method stddev(min_count: nil, fill_value: nil)
277
+ # Per-tile sample standard deviation (divisor `n - 1`).
278
+ # @param min_count [Integer, nil] fewest cells that must be present
279
+ # for a result; a piece with fewer comes back masked.
280
+ # @param fill_value [Object, nil] value to put in place of a masked
281
+ # result instead of leaving it masked.
207
282
  # @return [CArray] tile-grid shaped
208
- # The rest are analogous: prod / mean / min / max, sample and population
209
- # variance / stddev, all / any.
210
- [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
283
+ # @!method all
284
+ # Whether every cell of each tile is true.
285
+ # Takes no `min_count:` or `fill_value:`, as the core `all` does not.
286
+ # @return [CArray] tile-grid shaped
287
+ # @!method any
288
+ # Whether any cell of each tile is true.
289
+ # Takes no `min_count:` or `fill_value:`, as the core `any` does not.
290
+ # @return [CArray] tile-grid shaped
291
+ # @!method variancep(min_count: nil, fill_value: nil)
292
+ # Per-tile population variance (divisor `n`).
293
+ # @param min_count [Integer, nil] fewest cells that must be present
294
+ # for a result; a piece with fewer comes back masked.
295
+ # @param fill_value [Object, nil] value to put in place of a masked
296
+ # result instead of leaving it masked.
297
+ # @return [CArray] tile-grid shaped
298
+ # @!method stddevp(min_count: nil, fill_value: nil)
299
+ # Per-tile population standard deviation (divisor `n`).
300
+ # @param min_count [Integer, nil] fewest cells that must be present
301
+ # for a result; a piece with fewer comes back masked.
302
+ # @param fill_value [Object, nil] value to put in place of a masked
303
+ # result instead of leaving it masked.
304
+ # @return [CArray] tile-grid shaped
305
+ [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev,
211
306
  :variancep, :stddevp].each do |op|
212
307
  define_method(op) do |min_count: nil, fill_value: nil|
308
+ if FACE_WINNER_INDEX.key?(op) && @source.face?
309
+ return face_winner(FACE_WINNER_INDEX[op], min_count, fill_value)
310
+ end
213
311
  kw = {}
214
312
  kw[:min_count] = min_count unless min_count.nil?
215
313
  kw[:fill_value] = fill_value unless fill_value.nil?
@@ -217,6 +315,17 @@ class CABlockIterator < CAIterator
217
315
  end
218
316
  end
219
317
 
318
+ [:all, :any].each do |op|
319
+ define_method(op) { |**kw| fold(op, **CAIterator.reduce_keywords(op, kw)) }
320
+ end
321
+
322
+ # A Face's minimum and maximum are cells of the Face, so they are gathered
323
+ # from the source by the winner's address. Assembling the per-region results
324
+ # instead copies the Face's storage into an array that no longer knows it is
325
+ # one: a CATime came back as 8-byte strings.
326
+ FACE_WINNER_INDEX = { min: :min_index, max: :max_index }.freeze
327
+ private_constant :FACE_WINNER_INDEX
328
+
220
329
  # ---- count family and elements ----------------------------------------
221
330
 
222
331
  # @overload count(v = <none>)
@@ -224,15 +333,16 @@ class CABlockIterator < CAIterator
224
333
  # partial edge tile); `count(UNDEF)` counts masked cells; `count(v)` counts
225
334
  # cells equal to `v`.
226
335
  # @return [CArray] tile-grid shaped
227
- def count (*args)
228
- return count_not_masked if args.empty?
336
+ def count (*args, **kw)
337
+ return count_not_masked(**kw) if args.empty?
338
+ kw = CAIterator.reduce_keywords(:count, kw)
229
339
  out = nil
230
340
  each_region do |strip_ranges, tiles, out_ranges|
231
341
  view = @source[*strip_ranges].block_view(*tiles)
232
342
  # block_view is a CAStride, so #count is not shadowed; dispatch
233
343
  # CArray#count explicitly anyway, matching the family regularity.
234
- red = CArray.instance_method(:count).bind_call(view, *args, axis: @tile_axes)
235
- out ||= CArray.new(red.data_type, @shape)
344
+ red = CArray.instance_method(:count).bind_call(view, *args, axis: @tile_axes, **kw)
345
+ out ||= CArray.new(red.data_type, @shape, bytes: red.bytes)
236
346
  out[*out_ranges] = red
237
347
  end
238
348
  out
@@ -242,16 +352,21 @@ class CABlockIterator < CAIterator
242
352
  # Per-tile count of present (non-masked) cells. For a partial edge tile
243
353
  # this is the real cell count (the OOB cells are not present).
244
354
  # @return [CArray]
245
- def count_not_masked
246
- fold(:count_not_masked)
355
+ def count_not_masked (**kw)
356
+ fold(:count_not_masked, **CAIterator.reduce_keywords(:count_not_masked, kw))
247
357
  end
248
358
 
249
359
  # @overload count_masked
250
360
  # Per-tile count of masked cells. Equals `elements - count_not_masked`;
251
361
  # the OOB cells of a partial edge tile count here.
252
362
  # @return [CArray]
253
- def count_masked
254
- elements - count_not_masked
363
+ def count_masked (**kw)
364
+ kw = CAIterator.reduce_keywords(:count_masked, kw)
365
+ need = CArray.__min_count_need__(:count_masked, kw[:min_count], 0)
366
+ out = elements - count_not_masked
367
+ # A count of the mask reads every cell of the tile.
368
+ out[elements.lt(need)] = UNDEF if need > 0
369
+ CArray.__reduce_fill__(out, kw[:fill_value])
255
370
  end
256
371
 
257
372
  # @overload elements
@@ -273,23 +388,28 @@ class CABlockIterator < CAIterator
273
388
  # Per-tile `[min, max]` (two tile-grid-shaped CArrays, a single fused pass).
274
389
  # @return [Array<CArray>]
275
390
  def minmax (min_count: nil, fill_value: nil)
391
+ if @source.face?
392
+ return [min(min_count: min_count, fill_value: fill_value),
393
+ max(min_count: min_count, fill_value: fill_value)]
394
+ end
276
395
  kw = {}
277
396
  kw[:min_count] = min_count unless min_count.nil?
278
397
  kw[:fill_value] = fill_value unless fill_value.nil?
279
398
  assemble { |view, _| view.minmax(axis: @tile_axes, **kw) }
280
399
  end
281
400
 
282
- # @overload min_index
401
+ # @!method min_index
283
402
  # Per-tile position of the minimum, as a flat index within the tile (a
284
403
  # partial edge tile indexes within its own present cells).
285
404
  # @return [CArray] tile-grid shaped
286
- # @overload max_index
405
+ # @!method max_index
287
406
  # Per-tile position of the maximum (tile-local flat index).
288
407
  # @return [CArray]
289
408
  [:min_index, :max_index].each do |op|
290
409
  class_eval <<~RUBY, __FILE__, __LINE__ + 1
291
- def #{op}
292
- assemble { |view, _| view.#{op}(axis: @tile_axes) }
410
+ def #{op} (**kw)
411
+ kw = CAIterator.reduce_keywords(:#{op}, kw)
412
+ assemble { |view, _| view.#{op}(axis: @tile_axes, **kw) }
293
413
  end
294
414
  RUBY
295
415
  end
@@ -300,26 +420,49 @@ class CABlockIterator < CAIterator
300
420
  # minima. Unlike `min_index` (the tile-local position) this indexes back
301
421
  # into the original array. An all-masked tile is a masked cell.
302
422
  # @return [CArray] tile-grid shaped int64
303
- def min_addr; winner_addr(:min_index); end
423
+ def min_addr (**kw); winner_addr(:min_index, kw); end
304
424
 
305
425
  # @overload max_addr
306
426
  # Per-tile flat source address of the maximum. See {#min_addr}.
307
427
  # @return [CArray] tile-grid shaped int64
308
- def max_addr; winner_addr(:max_index); end
428
+ def max_addr (**kw); winner_addr(:max_index, kw); end
309
429
 
310
430
  private
311
431
 
432
+ # Per-tile minimum or maximum of a Face source, gathered from the source by
433
+ # the winner's address so the result is the Face itself. An all-masked tile,
434
+ # and with `min_count` a tile with too few present cells, is a masked cell.
435
+ def face_winner (idx_op, min_count, fill_value)
436
+ addr = winner_addr(idx_op)
437
+ need = CArray.__min_count_need__(idx_op == :min_index ? :min : :max, min_count, 0)
438
+ addr[count_not_masked.lt(need)] = UNDEF if need > 0
439
+ src = @source.reshape(@source.elements)
440
+ empty = addr.is_masked
441
+ return src[addr].copy unless empty.any
442
+ # A masked index drops the cell rather than masking it, so an empty tile
443
+ # needs a placeholder address. Pointing it at a masked cell of the source
444
+ # carries the mask through the gather, which a read-only Face (whose copy
445
+ # cannot be written) needs. Otherwise mask the placeholder afterwards.
446
+ if fill_value.nil? && src.has_mask? && (hole = src.is_masked.where).elements > 0
447
+ return src[addr.strip_mask(hole[0])].copy
448
+ end
449
+ out = src[addr.strip_mask(0)].copy
450
+ out[empty] = fill_value.nil? ? UNDEF : fill_value
451
+ out
452
+ end
453
+
312
454
  # Per-tile source address of the winner. The tile-local flat index (min_index /
313
455
  # max_index) is looked up in a source-address companion tiled the same way, so
314
456
  # the tile geometry is reused rather than recomputed. An all-masked tile yields
315
457
  # a masked index; it is gathered at 0 and re-masked in the result.
316
- def winner_addr (idx_op)
458
+ def winner_addr (idx_op, kw = {})
459
+ kw = CAIterator.reduce_keywords(idx_op, kw)
317
460
  saddr = CArray.int64(*@source.shape).seq!
318
461
  out = nil
319
462
  each_region do |strip_ranges, tiles, out_ranges|
320
463
  vview = @source[*strip_ranges].block_view(*tiles)
321
464
  sview = saddr[*strip_ranges].block_view(*tiles)
322
- mi = vview.public_send(idx_op, axis: @tile_axes) # tile-local flat index
465
+ mi = vview.public_send(idx_op, axis: @tile_axes, **kw.slice(:min_count)) # tile-local flat index
323
466
  grid = (0...@sndim).map { |i| vview.shape[i] }
324
467
  cells = tiles.inject(1) { |p, t| p * t }
325
468
  safe = mi.copy
@@ -331,7 +474,7 @@ class CABlockIterator < CAIterator
331
474
  out ||= CArray.int64(*@shape)
332
475
  out[*out_ranges] = picked
333
476
  end
334
- out
477
+ CArray.__reduce_fill__(out || CArray.int64(*@shape), kw[:fill_value])
335
478
  end
336
479
 
337
480
  public
@@ -365,15 +508,17 @@ class CABlockIterator < CAIterator
365
508
  # Per-tile weighted sum; `weights` is shaped like one full tile (`Π b_i`).
366
509
  # At a partial edge tile the weight kernel is sliced to the present cells.
367
510
  # @return [CArray]
368
- def wsum (weights)
369
- weighted(weights) { |view, w| view.wsum(w, axis: @tile_axes) }
511
+ def wsum (weights, **kw)
512
+ kw = CAIterator.reduce_keywords(:wsum, kw)
513
+ weighted(weights) { |view, w| view.wsum(w, axis: @tile_axes, **kw) }
370
514
  end
371
515
 
372
516
  # @overload wmean(weights)
373
517
  # Per-tile weighted mean; `weights` shaped like one full tile.
374
518
  # @return [CArray]
375
- def wmean (weights)
376
- weighted(weights) { |view, w| view.wmean(w, axis: @tile_axes) }
519
+ def wmean (weights, **kw)
520
+ kw = CAIterator.reduce_keywords(:wmean, kw)
521
+ weighted(weights) { |view, w| view.wmean(w, axis: @tile_axes, **kw) }
377
522
  end
378
523
 
379
524
  private
@@ -403,21 +548,22 @@ class CABlockIterator < CAIterator
403
548
  # Core per-axis order statistics take a single integer axis, so a 1-D tile
404
549
  # delegates straight to the C path; a multi-axis tile is materialized per
405
550
  # region and its tile axes flattened into one before the single-axis core
406
- # order-stat. A masked source raises (the known per-axis masked limitation);
407
- # strip the mask with `ca.value` first if needed.
551
+ # order-stat. Masked cells are left out, as in `CArray#median(axis:)`.
408
552
 
409
553
  # @overload median
410
554
  # Per-tile median. @return [CArray]
411
- def median
412
- order_stat { |v, axis| v.median(axis: axis) }
555
+ def median (**kw)
556
+ kw = CAIterator.reduce_keywords(:median, kw)
557
+ order_stat { |v, axis| v.median(axis: axis, **kw) }
413
558
  end
414
559
 
415
560
  # @overload percentile(*pers)
416
561
  # Per-tile percentile(s). One argument returns one CArray, several an array
417
562
  # of CArrays (as `CArray#percentile`).
418
563
  # @return [CArray, Array<CArray>]
419
- def percentile (*pers)
420
- order_stat { |v, axis| v.percentile(*pers, axis: axis) }
564
+ def percentile (*pers, **kw)
565
+ kw = CAIterator.reduce_keywords(:percentile, kw)
566
+ order_stat { |v, axis| v.percentile(*pers, axis: axis, **kw) }
421
567
  end
422
568
 
423
569
  # @overload quantile
@@ -509,7 +655,7 @@ class CABlockIterator < CAIterator
509
655
  # @return [CArray, Enumerator] source-shaped
510
656
  def map
511
657
  return to_enum(:map) unless block_given?
512
- pout = CArray.new(padded_source.data_type, padded_source.shape)
658
+ pout = CArray.new(padded_source.data_type, padded_source.shape, bytes: padded_source.bytes)
513
659
  pgv = pout.block_view(*@sizes)
514
660
  tgv = tile_grid_view
515
661
  nils = Array.new(@sndim, nil)
@@ -530,19 +676,19 @@ class CABlockIterator < CAIterator
530
676
  # cumcount -> int64 running count of present cells; the output data type is seeded
531
677
  # from the first tile's scan.
532
678
 
533
- # @overload cumsum
679
+ # @!method cumsum
534
680
  # Per-tile inclusive running sum (float64), source-shaped.
535
681
  # @return [CArray]
536
- # @overload cumprod
682
+ # @!method cumprod
537
683
  # Per-tile inclusive running product (float64), source-shaped.
538
684
  # @return [CArray]
539
- # @overload cummax
685
+ # @!method cummax
540
686
  # Per-tile inclusive running maximum (value data type), source-shaped.
541
687
  # @return [CArray]
542
- # @overload cummin
688
+ # @!method cummin
543
689
  # Per-tile inclusive running minimum (value data type), source-shaped.
544
690
  # @return [CArray]
545
- # @overload cumcount
691
+ # @!method cumcount
546
692
  # Per-tile running count of present cells (int64), source-shaped.
547
693
  # @return [CArray]
548
694
  [:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
@@ -564,7 +710,7 @@ class CABlockIterator < CAIterator
564
710
  tile = tgv[*g, *nils]
565
711
  scanned = tile.copy.reshape(tile.elements).public_send(op).reshape(*@sizes)
566
712
  unless pout
567
- pout = CArray.new(scanned.data_type, padded_source.shape)
713
+ pout = CArray.new(scanned.data_type, padded_source.shape, bytes: scanned.bytes)
568
714
  pout[] = UNDEF
569
715
  pgv = pout.block_view(*@sizes)
570
716
  end
@@ -583,7 +729,7 @@ class CABlockIterator < CAIterator
583
729
  if pshape == @source.shape.to_a
584
730
  @source
585
731
  else
586
- pad = CArray.new(@source.data_type, pshape)
732
+ pad = CArray.new(@source.data_type, pshape, bytes: @source.bytes)
587
733
  pad[] = UNDEF
588
734
  pad[*@sndim.times.map { |i| 0...@source.shape[i] }] = @source
589
735
  pad