carray 3.0.0 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +4 -4
  2. data/.yardopts +2 -1
  3. data/CHANGELOG.md +845 -3
  4. data/{NEWS.md → CHANGELOG.v1.md} +1 -1
  5. data/README.md +33 -33
  6. data/carray.gemspec +12 -8
  7. data/ext/ca_axis_dispatch.c +33 -4
  8. data/ext/ca_axis_group.c +256 -114
  9. data/ext/ca_binop_dispatch.c +5 -6
  10. data/ext/ca_binop_dispatch.h +0 -7
  11. data/ext/ca_categorical_iterator.c +121 -67
  12. data/ext/ca_for_each_element.h +6 -8
  13. data/ext/ca_group_iter.c +1 -1
  14. data/ext/ca_kernel_iterator.c +386 -74
  15. data/ext/ca_kernel_iterator.h +145 -45
  16. data/ext/ca_obj_array.c +62 -20
  17. data/ext/ca_obj_bincmp.c +57 -27
  18. data/ext/ca_obj_binop.c +86 -30
  19. data/ext/ca_obj_block.c +4 -4
  20. data/ext/ca_obj_const_string.c +88 -27
  21. data/ext/ca_obj_face.c +104 -0
  22. data/ext/ca_obj_face.h +48 -1
  23. data/ext/ca_obj_fake.c +11 -0
  24. data/ext/ca_obj_fixlen_string.c +21 -6
  25. data/ext/ca_obj_grid.c +11 -3
  26. data/ext/ca_obj_meld.c +125 -22
  27. data/ext/ca_obj_moncmp.c +33 -14
  28. data/ext/ca_obj_monop.c +19 -7
  29. data/ext/ca_obj_object.c +107 -4
  30. data/ext/ca_obj_record.c +3 -1
  31. data/ext/ca_obj_refer.c +30 -6
  32. data/ext/ca_obj_roll.c +13 -5
  33. data/ext/ca_obj_select.c +49 -34
  34. data/ext/ca_obj_select_axis.c +16 -6
  35. data/ext/ca_obj_shift.c +3 -3
  36. data/ext/ca_obj_stack.c +3 -8
  37. data/ext/ca_obj_stride.c +178 -24
  38. data/ext/ca_obj_string.c +11 -5
  39. data/ext/ca_obj_tile.c +13 -5
  40. data/ext/ca_obj_time.c +3 -1
  41. data/ext/ca_obj_timedelta.c +3 -1
  42. data/ext/ca_obj_transpose.c +1 -1
  43. data/ext/ca_obj_triop.c +62 -23
  44. data/ext/ca_obj_window.c +94 -19
  45. data/ext/ca_op_cmplx64.h +123 -0
  46. data/ext/ca_op_ipower.c +1 -5
  47. data/ext/ca_rng_normal.h +42 -0
  48. data/ext/ca_rng_xoshiro256pp.h +105 -0
  49. data/ext/ca_sort_kernels.h +5 -5
  50. data/ext/ca_sweep_engine.c +362 -155
  51. data/ext/ca_sweep_engine.h +39 -9
  52. data/ext/ca_transform_common.c +7 -0
  53. data/ext/carray.h +148 -132
  54. data/ext/carray_access.c +83 -49
  55. data/ext/carray_address_basis.c +590 -0
  56. data/ext/carray_bincount.c +8 -8
  57. data/ext/carray_broadcast.c +102 -9
  58. data/ext/carray_build_flags.h +3 -0
  59. data/ext/carray_call_cfunc.c +2010 -335
  60. data/ext/carray_call_cfunc.h +153 -0
  61. data/ext/carray_cast.c +184 -72
  62. data/ext/carray_conversion.c +22 -22
  63. data/ext/carray_copy.c +67 -32
  64. data/ext/carray_core.c +140 -13
  65. data/ext/carray_count.c +10 -11
  66. data/ext/carray_element.c +7 -7
  67. data/ext/carray_factorize.c +72 -51
  68. data/ext/carray_hold.c +1 -1
  69. data/ext/carray_index_classifier.c +7 -21
  70. data/ext/carray_internal.h +36 -3
  71. data/ext/carray_kernels_bincmp.c +1 -0
  72. data/ext/carray_kernels_binop.c +320 -298
  73. data/ext/carray_kernels_init.c +1095 -0
  74. data/ext/carray_kernels_map.c +1 -0
  75. data/ext/carray_kernels_moncmp.c +1 -0
  76. data/ext/carray_kernels_monop.c +412 -411
  77. data/ext/carray_kernels_reduce_aggregate.c +256 -87
  78. data/ext/carray_kernels_reduce_boolean.c +1 -0
  79. data/ext/carray_kernels_reduce_cumulative.c +324 -54
  80. data/ext/carray_kernels_reduce_extreme.c +605 -58
  81. data/ext/carray_kernels_reduce_variance.c +1 -0
  82. data/ext/carray_kernels_scan.c +5 -4
  83. data/ext/carray_kernels_search.c +187 -106
  84. data/ext/carray_kernels_sort.c +1 -0
  85. data/ext/carray_kernels_triop.c +1 -0
  86. data/ext/carray_lazy.c +175 -5
  87. data/ext/carray_loop.c +7 -1
  88. data/ext/carray_mask.c +29 -13
  89. data/ext/carray_median_percentile.c +56 -1
  90. data/ext/carray_memory_view.c +46 -21
  91. data/ext/carray_operator.c +50 -53
  92. data/ext/carray_order.c +1 -1
  93. data/ext/carray_partition.c +4 -4
  94. data/ext/carray_random.c +391 -47
  95. data/ext/carray_scatter.c +1 -1
  96. data/ext/carray_slab.c +13 -0
  97. data/ext/carray_sort.c +22 -24
  98. data/ext/carray_sort_kernel.c +10 -10
  99. data/ext/carray_test.c +1 -1
  100. data/ext/extconf.rb +18 -0
  101. data/ext/mk_call_cfunc.rb +323 -145
  102. data/ext/mkkernel.rb +651 -129
  103. data/ext/ruby_carray.c +28 -17
  104. data/ext/version.h +4 -4
  105. data/lib/carray/autoload_carray.rb +24 -3
  106. data/lib/carray/autoload_method_extension.rb +14 -1
  107. data/lib/carray/axis_group.rb +84 -7
  108. data/lib/carray/basics.rb +63 -59
  109. data/lib/carray/bincount_nd.rb +12 -12
  110. data/lib/carray/block_iterator.rb +111 -25
  111. data/lib/carray/categorical.rb +154 -37
  112. data/lib/carray/categorical_iterator.rb +300 -151
  113. data/lib/carray/conditional.rb +14 -14
  114. data/lib/carray/const_string.rb +131 -27
  115. data/lib/carray/construct.rb +61 -0
  116. data/lib/carray/core_extensions.rb +62 -5
  117. data/lib/carray/data_type_extension.rb +21 -1
  118. data/lib/carray/data_type_limits.rb +91 -0
  119. data/lib/carray/fixlen_string.rb +1 -1
  120. data/lib/carray/frame/concat.rb +3 -3
  121. data/lib/carray/frame/convert.rb +1 -1
  122. data/lib/carray/frame/csv_parser.rb +11 -4
  123. data/lib/carray/frame/frame.rb +85 -21
  124. data/lib/carray/frame/group.rb +36 -3
  125. data/lib/carray/frame/io.rb +224 -15
  126. data/lib/carray/frame/records.rb +18 -4
  127. data/lib/carray/frame/sort.rb +1 -1
  128. data/lib/carray/frame/verbs.rb +32 -12
  129. data/lib/carray/fuse_source.rb +123 -0
  130. data/lib/carray/fusion.rb +218 -0
  131. data/lib/carray/histogram.rb +16 -16
  132. data/lib/carray/inspect.rb +43 -16
  133. data/lib/carray/iterator.rb +147 -3
  134. data/lib/carray/lazy.rb +88 -73
  135. data/lib/carray/mask_gap_fill.rb +3 -1
  136. data/lib/carray/meld_reduce.rb +2 -2
  137. data/lib/carray/methods/align_addr.rb +1 -1
  138. data/lib/carray/methods/composition.rb +1 -1
  139. data/lib/carray/methods/discovery_along.rb +74 -0
  140. data/lib/carray/methods/factorize.rb +50 -0
  141. data/lib/carray/methods/is_in.rb +25 -14
  142. data/lib/carray/methods/locate_addr.rb +81 -3
  143. data/lib/carray/methods/mask_duplicates.rb +36 -2
  144. data/lib/carray/methods/meshgrid.rb +4 -5
  145. data/lib/carray/methods/mode.rb +2 -2
  146. data/lib/carray/methods/nunique.rb +23 -2
  147. data/lib/carray/methods/repeat.rb +110 -0
  148. data/lib/carray/methods/snap.rb +7 -2
  149. data/lib/carray/methods/unique.rb +44 -4
  150. data/lib/carray/methods/value_counts.rb +2 -2
  151. data/lib/carray/rng.rb +86 -0
  152. data/lib/carray/runtime.rb +0 -19
  153. data/lib/carray/slab_iterator.rb +77 -19
  154. data/lib/carray/string_operation_extension.rb +10 -6
  155. data/lib/carray/time.rb +906 -491
  156. data/lib/carray/window_iterator.rb +421 -27
  157. data/lib/carray.rb +7 -5
  158. data/yard-stubs/ca_obj_array.rb +385 -0
  159. data/yard-stubs/ca_obj_bitarray.rb +38 -0
  160. data/yard-stubs/ca_obj_bitfield.rb +43 -0
  161. data/yard-stubs/ca_obj_block.rb +68 -0
  162. data/yard-stubs/ca_obj_byte_swap.rb +56 -0
  163. data/yard-stubs/ca_obj_fake.rb +31 -0
  164. data/yard-stubs/ca_obj_farray.rb +32 -0
  165. data/yard-stubs/ca_obj_field.rb +45 -0
  166. data/yard-stubs/ca_obj_grid.rb +35 -0
  167. data/yard-stubs/ca_obj_refer.rb +72 -0
  168. data/yard-stubs/ca_obj_roll.rb +45 -0
  169. data/yard-stubs/ca_obj_shift.rb +43 -0
  170. data/yard-stubs/ca_obj_stride.rb +181 -0
  171. data/yard-stubs/ca_obj_tile.rb +29 -0
  172. data/yard-stubs/ca_obj_transpose.rb +40 -0
  173. data/yard-stubs/ca_obj_window.rb +57 -0
  174. data/yard-stubs/carray_access.rb +131 -0
  175. data/yard-stubs/carray_attribute.rb +246 -0
  176. data/yard-stubs/carray_broadcast.rb +37 -0
  177. data/yard-stubs/carray_cast.rb +489 -0
  178. data/yard-stubs/carray_class.rb +65 -0
  179. data/yard-stubs/carray_conversion.rb +76 -0
  180. data/yard-stubs/carray_copy.rb +79 -0
  181. data/yard-stubs/carray_core.rb +34 -0
  182. data/yard-stubs/carray_count.rb +84 -0
  183. data/yard-stubs/carray_element.rb +108 -0
  184. data/yard-stubs/carray_generate.rb +66 -0
  185. data/yard-stubs/carray_lazy.rb +228 -0
  186. data/yard-stubs/carray_loop.rb +140 -0
  187. data/yard-stubs/carray_mask.rb +259 -0
  188. data/yard-stubs/carray_math.rb +1615 -0
  189. data/yard-stubs/carray_mathfunc.rb +45 -0
  190. data/yard-stubs/carray_median_percentile.rb +103 -0
  191. data/yard-stubs/carray_memory_view.rb +163 -0
  192. data/yard-stubs/carray_order.rb +252 -0
  193. data/yard-stubs/carray_random.rb +89 -0
  194. data/yard-stubs/carray_scatter.rb +106 -0
  195. data/yard-stubs/carray_slab.rb +59 -0
  196. data/yard-stubs/carray_sort.rb +165 -0
  197. data/yard-stubs/carray_test.rb +85 -0
  198. data/yard-stubs/carray_undef.rb +64 -0
  199. data/yard-stubs/carray_utils.rb +97 -0
  200. data/yard-stubs/ruby_carray.rb +193 -0
  201. metadata +67 -10
  202. data/ext/ca_obj_unbound_repeat.c +0 -496
@@ -19,7 +19,7 @@
19
19
  # is a `block_view` (a CAStride: zero-copy over the source via compose-fold),
20
20
  # reduced over the trailing tile axes by a core reduction and scattered into the
21
21
  # ceil-shaped tile grid. One core reduction per region; the named reductions
22
- # delegate straight to the core, so their dtype / mask / empty (ERI) / epsilon
22
+ # delegate straight to the core, so their data type / mask / empty (ERI) / epsilon
23
23
  # contracts are the core's, unchanged. Because tiles do not overlap, no
24
24
  # padded entity is built (unlike CAWindowIterator) and the interior stays on
25
25
  # the source buffer.
@@ -165,7 +165,7 @@ class CABlockIterator < CAIterator
165
165
  # quantile) which are assembled into that many grids. Boundary regions use a
166
166
  # smaller (unmasked, ragged) block_view whose present cell count is naturally
167
167
  # below a full tile, so `min_count:` marks them UNDEF with no masking. The
168
- # output dtype is seeded from the first region (all regions share it).
168
+ # output data type is seeded from the first region (all regions share it).
169
169
  def assemble
170
170
  outs = nil
171
171
  each_region do |strip_ranges, tiles, out_ranges|
@@ -184,7 +184,7 @@ class CABlockIterator < CAIterator
184
184
 
185
185
  # Fold every region with a single core reduction `op` (the tier-1 shape).
186
186
  def fold (op, **kw)
187
- assemble { |view, _tiles| view.send(op, axis: @tile_axes, **kw) }
187
+ assemble { |view, _tiles| view.public_send(op, axis: @tile_axes, **kw) }
188
188
  end
189
189
 
190
190
  public
@@ -194,14 +194,96 @@ class CABlockIterator < CAIterator
194
194
  # A per-tile fold to one value over the trailing tile axes is exactly a core
195
195
  # per-axis reduction over those axes, so every reduction delegates to
196
196
  # `block_view.<op>(axis: tile_axes, ...)` per region. This inherits the core
197
- # dtype, mask, empty / all-masked (identity vs UNDEF) and epsilon-close
197
+ # data type, mask, empty / all-masked (identity vs UNDEF) and epsilon-close
198
198
  # contracts unchanged. `min_count:` / `fill_value:` pass straight to the core.
199
199
 
200
- # @overload sum(min_count: nil, fill_value: nil)
201
- # Per-tile sum. @return [CArray] tile-grid shaped
202
- # The rest are analogous: prod / mean / min / max, sample and population
203
- # variance / stddev, all / any.
204
- [:sum, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
200
+ # @!method sum(min_count: nil, fill_value: nil)
201
+ # Per-tile sum.
202
+ # @param min_count [Integer, nil] fewest cells that must be present
203
+ # for a result; a piece with fewer comes back masked.
204
+ # @param fill_value [Object, nil] value to put in place of a masked
205
+ # result instead of leaving it masked.
206
+ # @return [CArray] tile-grid shaped
207
+ # @!method accumulate(min_count: nil, fill_value: nil)
208
+ # Per-tile sum kept in the source's own data type, wrapping at its
209
+ # width, as the core `accumulate` does -- `sum` answers in the type
210
+ # the core promotes to (float64 for integers).
211
+ # @param min_count [Integer, nil] fewest cells that must be present
212
+ # for a result; a piece with fewer comes back masked.
213
+ # @param fill_value [Object, nil] value to put in place of a masked
214
+ # result instead of leaving it masked.
215
+ # @return [CArray] tile-grid shaped
216
+ # @!method prod(min_count: nil, fill_value: nil)
217
+ # Per-tile product.
218
+ # @param min_count [Integer, nil] fewest cells that must be present
219
+ # for a result; a piece with fewer comes back masked.
220
+ # @param fill_value [Object, nil] value to put in place of a masked
221
+ # result instead of leaving it masked.
222
+ # @return [CArray] tile-grid shaped
223
+ # @!method mean(min_count: nil, fill_value: nil)
224
+ # Per-tile arithmetic mean.
225
+ # @param min_count [Integer, nil] fewest cells that must be present
226
+ # for a result; a piece with fewer comes back masked.
227
+ # @param fill_value [Object, nil] value to put in place of a masked
228
+ # result instead of leaving it masked.
229
+ # @return [CArray] tile-grid shaped
230
+ # @!method min(min_count: nil, fill_value: nil)
231
+ # Per-tile minimum.
232
+ # @param min_count [Integer, nil] fewest cells that must be present
233
+ # for a result; a piece with fewer comes back masked.
234
+ # @param fill_value [Object, nil] value to put in place of a masked
235
+ # result instead of leaving it masked.
236
+ # @return [CArray] tile-grid shaped
237
+ # @!method max(min_count: nil, fill_value: nil)
238
+ # Per-tile maximum.
239
+ # @param min_count [Integer, nil] fewest cells that must be present
240
+ # for a result; a piece with fewer comes back masked.
241
+ # @param fill_value [Object, nil] value to put in place of a masked
242
+ # result instead of leaving it masked.
243
+ # @return [CArray] tile-grid shaped
244
+ # @!method variance(min_count: nil, fill_value: nil)
245
+ # Per-tile sample variance (divisor `n - 1`).
246
+ # @param min_count [Integer, nil] fewest cells that must be present
247
+ # for a result; a piece with fewer comes back masked.
248
+ # @param fill_value [Object, nil] value to put in place of a masked
249
+ # result instead of leaving it masked.
250
+ # @return [CArray] tile-grid shaped
251
+ # @!method stddev(min_count: nil, fill_value: nil)
252
+ # Per-tile sample standard deviation (divisor `n - 1`).
253
+ # @param min_count [Integer, nil] fewest cells that must be present
254
+ # for a result; a piece with fewer comes back masked.
255
+ # @param fill_value [Object, nil] value to put in place of a masked
256
+ # result instead of leaving it masked.
257
+ # @return [CArray] tile-grid shaped
258
+ # @!method all(min_count: nil, fill_value: nil)
259
+ # Whether every cell of each tile is true.
260
+ # @param min_count [Integer, nil] fewest cells that must be present
261
+ # for a result; a piece with fewer comes back masked.
262
+ # @param fill_value [Object, nil] value to put in place of a masked
263
+ # result instead of leaving it masked.
264
+ # @return [CArray] tile-grid shaped
265
+ # @!method any(min_count: nil, fill_value: nil)
266
+ # Whether any cell of each tile is true.
267
+ # @param min_count [Integer, nil] fewest cells that must be present
268
+ # for a result; a piece with fewer comes back masked.
269
+ # @param fill_value [Object, nil] value to put in place of a masked
270
+ # result instead of leaving it masked.
271
+ # @return [CArray] tile-grid shaped
272
+ # @!method variancep(min_count: nil, fill_value: nil)
273
+ # Per-tile population variance (divisor `n`).
274
+ # @param min_count [Integer, nil] fewest cells that must be present
275
+ # for a result; a piece with fewer comes back masked.
276
+ # @param fill_value [Object, nil] value to put in place of a masked
277
+ # result instead of leaving it masked.
278
+ # @return [CArray] tile-grid shaped
279
+ # @!method stddevp(min_count: nil, fill_value: nil)
280
+ # Per-tile population standard deviation (divisor `n`).
281
+ # @param min_count [Integer, nil] fewest cells that must be present
282
+ # for a result; a piece with fewer comes back masked.
283
+ # @param fill_value [Object, nil] value to put in place of a masked
284
+ # result instead of leaving it masked.
285
+ # @return [CArray] tile-grid shaped
286
+ [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
205
287
  :variancep, :stddevp].each do |op|
206
288
  define_method(op) do |min_count: nil, fill_value: nil|
207
289
  kw = {}
@@ -273,15 +355,19 @@ class CABlockIterator < CAIterator
273
355
  assemble { |view, _| view.minmax(axis: @tile_axes, **kw) }
274
356
  end
275
357
 
276
- # @overload min_index
358
+ # @!method min_index
277
359
  # Per-tile position of the minimum, as a flat index within the tile (a
278
360
  # partial edge tile indexes within its own present cells).
279
361
  # @return [CArray] tile-grid shaped
280
- # @overload max_index
362
+ # @!method max_index
281
363
  # Per-tile position of the maximum (tile-local flat index).
282
364
  # @return [CArray]
283
365
  [:min_index, :max_index].each do |op|
284
- define_method(op) { assemble { |view, _| view.send(op, axis: @tile_axes) } }
366
+ class_eval <<~RUBY, __FILE__, __LINE__ + 1
367
+ def #{op}
368
+ assemble { |view, _| view.#{op}(axis: @tile_axes) }
369
+ end
370
+ RUBY
285
371
  end
286
372
 
287
373
  # @overload min_addr
@@ -309,7 +395,7 @@ class CABlockIterator < CAIterator
309
395
  each_region do |strip_ranges, tiles, out_ranges|
310
396
  vview = @source[*strip_ranges].block_view(*tiles)
311
397
  sview = saddr[*strip_ranges].block_view(*tiles)
312
- mi = vview.send(idx_op, axis: @tile_axes) # tile-local flat index
398
+ mi = vview.public_send(idx_op, axis: @tile_axes) # tile-local flat index
313
399
  grid = (0...@sndim).map { |i| vview.shape[i] }
314
400
  cells = tiles.inject(1) { |p, t| p * t }
315
401
  safe = mi.copy
@@ -380,7 +466,7 @@ class CABlockIterator < CAIterator
380
466
  wt = tiles == @sizes ? weights : weights[*tiles.map { |t| 0...t }]
381
467
  wshape = ([1] * @sndim) + tiles
382
468
  # Pass the weights through unchanged, like the window iterator: core wsum /
383
- # wmean own weight/dtype coercion, so do not pre-coerce the weights here.
469
+ # wmean own weight / type coercion, so do not pre-coerce the weights here.
384
470
  wfull = wt.reshape(*wshape).broadcast_to(*view.shape)
385
471
  yield view, wfull
386
472
  end
@@ -516,23 +602,23 @@ class CABlockIterator < CAIterator
516
602
  # reusing the same padded (masked-margin) entity as the iterate escape
517
603
  # hatches, so this is the slow path (a per-tile materialize), and the
518
604
  # out-of-bounds cells of a partial edge tile are dropped from the result.
519
- # cumsum / cumprod -> float64, cummax / cummin preserve the value dtype,
520
- # cumcount -> int64 running count of present cells; the output dtype is seeded
605
+ # cumsum / cumprod -> float64, cummax / cummin preserve the value data type,
606
+ # cumcount -> int64 running count of present cells; the output data type is seeded
521
607
  # from the first tile's scan.
522
608
 
523
- # @overload cumsum
609
+ # @!method cumsum
524
610
  # Per-tile inclusive running sum (float64), source-shaped.
525
611
  # @return [CArray]
526
- # @overload cumprod
612
+ # @!method cumprod
527
613
  # Per-tile inclusive running product (float64), source-shaped.
528
614
  # @return [CArray]
529
- # @overload cummax
530
- # Per-tile inclusive running maximum (value dtype), source-shaped.
615
+ # @!method cummax
616
+ # Per-tile inclusive running maximum (value data type), source-shaped.
531
617
  # @return [CArray]
532
- # @overload cummin
533
- # Per-tile inclusive running minimum (value dtype), source-shaped.
618
+ # @!method cummin
619
+ # Per-tile inclusive running minimum (value data type), source-shaped.
534
620
  # @return [CArray]
535
- # @overload cumcount
621
+ # @!method cumcount
536
622
  # Per-tile running count of present cells (int64), source-shaped.
537
623
  # @return [CArray]
538
624
  [:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
@@ -542,7 +628,7 @@ class CABlockIterator < CAIterator
542
628
  private
543
629
 
544
630
  # Drive a within-tile segment scan: scan each tile flattened row-major and
545
- # scatter back to a source-shaped result. The output dtype is taken from the
631
+ # scatter back to a source-shaped result. The output data type is taken from the
546
632
  # first tile's scan (all tiles share it); the padded margin is UNDEF and the
547
633
  # OOB cells of a partial edge tile are cropped from the result.
548
634
  def block_scan (op)
@@ -552,7 +638,7 @@ class CABlockIterator < CAIterator
552
638
  pgv = nil
553
639
  CArray.each_index(*@shape) do |*g|
554
640
  tile = tgv[*g, *nils]
555
- scanned = tile.copy.reshape(tile.elements).send(op).reshape(*@sizes)
641
+ scanned = tile.copy.reshape(tile.elements).public_send(op).reshape(*@sizes)
556
642
  unless pout
557
643
  pout = CArray.new(scanned.data_type, padded_source.shape)
558
644
  pout[] = UNDEF
@@ -2,7 +2,7 @@
2
2
  #
3
3
  # carray/categorical.rb
4
4
  #
5
- # CACategorical — a categorical dtype: dense integer codes (the storage) plus
5
+ # CACategorical — a categorical data type: dense integer codes (the storage) plus
6
6
  # a label vocabulary. Structurally the same as a pandas Categorical or an
7
7
  # Arrow dictionary: each element is an index into a small set of categories.
8
8
  #
@@ -17,8 +17,12 @@
17
17
  # (per-category counts), `cat.codes.count(code)`, etc.
18
18
  # - per-cell access = decode the code into its label (`cat[i]` -> category)
19
19
  #
20
- # Exclusion (missing / out-of-vocabulary) is encoded two ways at once, and
21
- # because the Face is READONLY they can never desync:
20
+ # Exclusion (missing / out-of-vocabulary) is encoded two ways at once. Both
21
+ # are written when the categorical is built — categorize produces them, and
22
+ # from_codes normalises whatever it receives into them — and the Face is
23
+ # READONLY afterwards, so they stay in step. Both are needed because they have
24
+ # different readers: the axis-group kernel classifies on the code byte, the
25
+ # materialising paths classify on the mask.
22
26
  #
23
27
  # - the cell is MASKED -> CArray-native idiom: `cat[i]` is
24
28
  # UNDEF, `is_masked` / mask-aware
@@ -66,7 +70,7 @@ class CACategorical < CAObject
66
70
  }.freeze
67
71
 
68
72
  # The exclusion sentinel per codes data_type: the all-ones bit pattern, read
69
- # as type-max for an unsigned dtype and as -1 for a signed one. Either way it
73
+ # as type-max for an unsigned data type and as -1 for a signed one. Either way it
70
74
  # is out of every valid [0, k) range and byte-identical to a pandas / Arrow
71
75
  # missing code.
72
76
  SENTINEL = {
@@ -76,46 +80,148 @@ class CACategorical < CAObject
76
80
  CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => -1,
77
81
  }.freeze
78
82
 
83
+ # The largest vocabulary each codes data type can carry: every valid code in
84
+ # 0...k has to be representable AND distinct from the exclusion sentinel. An
85
+ # unsigned type spends its top value on the sentinel; a signed one spends -1,
86
+ # which was never a valid index, so a signed type carries one more label than
87
+ # the unsigned type of the same width. categorize picks a width by this rule
88
+ # when it builds codes; from_codes has to check it when it receives them.
89
+ MAX_LABELS = {
90
+ CA_UINT8 => 0xFF, CA_INT8 => 0x80,
91
+ CA_UINT16 => 0xFFFF, CA_INT16 => 0x8000,
92
+ CA_UINT32 => 0xFFFFFFFF, CA_INT32 => 0x80000000,
93
+ CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => 0x8000000000000000,
94
+ }.freeze
95
+
79
96
  class << self
80
97
  # Wrap already-dense codes + labels with no discovery — the import receiver
81
- # for a pandas Categorical or an Arrow dictionary. `codes` becomes the
82
- # Face's storage parent verbatim (zero-copy when it is a wrapped memory
83
- # view), and from_codes takes ownership of it.
98
+ # for a pandas Categorical or an Arrow dictionary. from_codes takes
99
+ # ownership of `codes`, which becomes the Face's storage parent: verbatim
100
+ # when it is an entity that needs no rewriting, so a wrapped memory view
101
+ # stays a view, and materialised when it is a view over an array the caller
102
+ # still holds, since a view owns no bytes to take ownership of.
103
+ #
104
+ # This is the one door through which an already-built encoding enters, so
105
+ # it validates rather than assumes, and it normalises before handing over:
106
+ # a cell is excluded when it arrives masked OR holds the all-ones sentinel
107
+ # (type-max for unsigned codes, -1 for signed — both the pandas / Arrow
108
+ # missing code), and every excluded cell leaves here holding the sentinel
109
+ # AND masked. Writing both matters because the two encodings have different
110
+ # readers: the axis-group kernel classifies on the code byte, the
111
+ # materialising paths classify on the mask. A categorical that carries only
112
+ # one of them answers membership two ways, silently. An Arrow dictionary
113
+ # carries its missingness in a validity bitmap with arbitrary code bytes —
114
+ # frequently 0 — so arriving masked-only is the normal import, not an edge.
84
115
  #
85
- # Excluded cells are identified by the all-ones sentinel value (type-max
86
- # for unsigned codes, -1 for signed — both the pandas / Arrow missing code)
87
- # and masked here, so the categorical is well-formed regardless of whether
88
- # the caller pre-masked. Only the mask buffer is touched; the code bytes
89
- # are left intact (so a pandas byte-reinterpret round-trips).
116
+ # #initialize marks the codes read-only, so this is also the last point at
117
+ # which they can be written; a read-only argument is copied rather than
118
+ # refused.
90
119
  # @overload from_codes(codes, labels)
91
120
  # Returns a {CACategorical} wrapping already-dense integer
92
121
  # `codes` with the given `labels`, without discovery. `codes`
93
- # becomes the Face's storage parent; excluded cells (identified
94
- # by the type-max sentinel value) are masked automatically.
122
+ # becomes the Face's storage parent, materialised first if it is
123
+ # a view. A cell that is masked or holds the type-max sentinel is
124
+ # excluded, and leaves as both.
95
125
  # @param codes [CArray] integer code storage.
96
126
  # @param labels [Array, CArray] category vocabulary indexed by
97
- # code.
127
+ # code. Must be unique, and must fit the codes data type with
128
+ # the sentinel reserved.
98
129
  # @return [CACategorical]
99
- # @raise [ArgumentError] when `codes` is not an integer CArray.
130
+ # @raise [ArgumentError] when `codes` is not an integer CArray,
131
+ # when `labels` holds duplicates or is too large for the codes
132
+ # data type, or when an unmasked code is outside `0...labels.size`
133
+ # and is not the sentinel.
100
134
  def from_codes(codes, labels)
101
135
  unless codes.is_a?(CArray) && SENTINEL.key?(codes.data_type)
102
136
  got = codes.is_a?(CArray) ? codes.data_type : codes.class
103
137
  raise ArgumentError, "from_codes: codes must be an integer CArray (got #{got})"
104
138
  end
105
- excluded = codes.eq(SENTINEL[codes.data_type])
106
- if excluded.count(true) > 0
107
- codes.mask = codes.has_mask? ? (codes.mask | excluded) : excluded
139
+
140
+ labels_arr = labels.respond_to?(:to_a) ? labels.to_a : Array(labels)
141
+ if labels_arr.uniq.size != labels_arr.size
142
+ raise ArgumentError, "from_codes: labels must be unique (got duplicates)"
143
+ end
144
+ k = labels_arr.size
145
+ sentinel = SENTINEL[codes.data_type]
146
+ max = MAX_LABELS[codes.data_type]
147
+ if k > max
148
+ raise ArgumentError,
149
+ "from_codes: #{k} labels do not fit #{CArray.data_type_name(codes.data_type)} " \
150
+ "codes, which carry at most #{max} (the top value is reserved as the " \
151
+ "exclusion sentinel); widen the codes data type"
152
+ end
153
+
154
+ # Classify on the code bytes with the mask set aside (`.value`): a
155
+ # comparison against a masked cell yields UNDEF, which would read as
156
+ # "not excluded" and let the cell through carrying a valid-looking code.
157
+ raw = codes.value
158
+ if sentinel == -1 # signed codes: -1 is the sentinel
159
+ out = raw.lt(0)
160
+ # Only test the upper bound when k is representable in the codes data
161
+ # type. At the very top of the range (k == max) no value can reach k
162
+ # anyway, and comparing against an unrepresentable literal would wrap
163
+ # and flag every cell as out of range.
164
+ out = out | raw.ge(k) if k <= max - 1
165
+ else # unsigned codes: type-max is the sentinel
166
+ out = raw.ge(k) # k <= max here, so always representable
167
+ end
168
+ masked = codes.has_mask? ? codes.is_masked : nil
169
+ excluded = masked ? (out | masked) : out
170
+ # A masked cell may hold any byte at all — that is the CArray contract —
171
+ # so it is never corrupt, only in need of normalising. An *unmasked* cell
172
+ # holding an out-of-range code that is not the sentinel is neither a
173
+ # category nor missingness; refuse it here, where the input is still in
174
+ # the caller's hands, instead of letting it surface later as an IndexError
175
+ # from the grouping plan or as a wrong label from a decode.
176
+ corrupt = out & raw.ne(sentinel)
177
+ corrupt = corrupt & masked.not if masked
178
+ if corrupt.any
179
+ bad = raw[corrupt].to_a.uniq.sort
180
+ shown = bad.first(4).join(", ") + (bad.size > 4 ? ", ..." : "")
181
+ raise ArgumentError,
182
+ "from_codes: code#{bad.size == 1 ? "" : "s"} #{shown} outside " \
183
+ "0...#{k} for #{k} label#{k == 1 ? "" : "s"} " \
184
+ "(use #{sentinel} to exclude a cell, or mask it)"
185
+ end
186
+
187
+ # Take real ownership, then normalise so the byte reader and the mask
188
+ # reader agree from here on.
189
+ #
190
+ # A view owns no bytes — its root does, and the caller still holds that
191
+ # root. #initialize marks what it is given read-only, which would stop
192
+ # `cat.codes[i] = x` but not `root[i] = x`, so the codes could still
193
+ # change underneath a Face that is supposed to be immutable (and
194
+ # underneath the grouping plan memoised against them). Marking the root
195
+ # instead is worse: it would freeze bytes outside the window the caller
196
+ # handed over. So a view is materialised. Read-only codes are copied for
197
+ # the adjacent reason — the normalising writes need somewhere to land.
198
+ #
199
+ # An entity that needs no rewriting is adopted verbatim, which is what
200
+ # keeps a wrapped memory view a view. That buffer's producer can still
201
+ # write it; that is the borrowed-buffer bargain, and not something this
202
+ # constructor can close.
203
+ needs_mask = masked ? (excluded & masked.not).any : excluded.any
204
+ needs_byte = (excluded & raw.ne(sentinel)).any
205
+ if !codes.entity? || ((needs_mask || needs_byte) && codes.read_only?)
206
+ codes = codes.copy
108
207
  end
109
- new(codes, labels)
208
+ if needs_mask || needs_byte
209
+ codes.value[excluded] = sentinel if needs_byte
210
+ codes.mask = excluded
211
+ end
212
+
213
+ new(codes, labels_arr)
110
214
  end
111
215
  end
112
216
 
113
- # codes : integer CArray, the storage parent. Excluded cells are both
114
- # masked AND store the type-max sentinel value (= the all-ones bit
115
- # pattern, which is signed -1 byte-for-byte — the pandas / Arrow
116
- # missing code). Because the Face is READONLY the two never desync,
117
- # so consumers may rely on either: the mask (CArray-native) or the
118
- # sentinel (axis-group's out-of-range skip, zero-copy export).
217
+ # codes : integer CArray, the storage parent, already normalised by the
218
+ # caller (categorize builds it that way; from_codes rewrites what it
219
+ # receives). Excluded cells are both masked AND store the type-max
220
+ # sentinel value (= the all-ones bit pattern, which is signed -1
221
+ # byte-for-byte — the pandas / Arrow missing code). Marking the codes
222
+ # read-only below keeps the two in step from here on, so consumers may
223
+ # rely on either: the mask (CArray-native) or the sentinel
224
+ # (axis-group's out-of-range skip, zero-copy export).
119
225
  # labels: Array | CArray, the vocabulary; labels[code] = category.
120
226
  # @overload initialize(codes, labels)
121
227
  # Allocates a READONLY {CACategorical} Face whose storage is
@@ -143,8 +249,9 @@ class CACategorical < CAObject
143
249
  # through views/Faces (a reshape of frozen codes is frozen) and would block
144
250
  # the grouping cache from memoising. The flag gives the same write protection
145
251
  # (mutations raise) while keeping the object non-frozen. One-way: it takes
146
- # ownership of `codes` (categorize / from_codes build or receive it, mask
147
- # already derived above); a caller keeping a mutable array must pass `.copy`.
252
+ # ownership of `codes`, which from_codes has already validated, materialised
253
+ # if it was a view, and normalised; a caller keeping a mutable entity of its
254
+ # own must pass `.copy`.
148
255
  codes.set_read_only_flag
149
256
  end
150
257
 
@@ -188,12 +295,11 @@ class CACategorical < CAObject
188
295
  other.is_a?(CACategorical) && @labels == other.labels
189
296
  end
190
297
 
191
- # Face hook: decode a per-cell code into its category label. An out-of-range
192
- # code (e.g. an unmasked external sentinel) decodes to nil rather than a
193
- # wrong category via Ruby negative indexing.
298
+ # Face hook: decode a per-cell code into its category label, through the one
299
+ # decode every code-to-label path shares (see #label_at).
194
300
  def storage_to_scalar(raw)
195
301
  code = raw.is_a?(String) ? raw.unpack1(UNPACK_FORMAT.fetch(parent.data_type)) : raw
196
- (code < 0 || code >= @labels.size) ? nil : @labels[code]
302
+ label_at(code)
197
303
  end
198
304
 
199
305
  # ---- category-space operations (by label, not code) -------------------
@@ -350,10 +456,21 @@ class CACategorical < CAObject
350
456
 
351
457
  private
352
458
 
353
- # Labels for a code array (the discovery kernels skip masked cells, so the
354
- # code arrays reaching here hold real codes only).
459
+ # The one decode from a code to its label. An out-of-range code — a negative
460
+ # one, or an unmasked sentinel that reached us from outside — has no label,
461
+ # so it decodes to nil. Writing `@labels[code]` instead would read a negative
462
+ # code from the *end* of the vocabulary and hand back a real label for a cell
463
+ # that has none. Every code-to-label path goes through here rather than
464
+ # indexing @labels itself, because two decodes are two chances to disagree,
465
+ # and disagreeing is how #unique came to contradict #to_a on the same cell.
466
+ def label_at (code)
467
+ (code < 0 || code >= @labels.size) ? nil : @labels[code]
468
+ end
469
+
470
+ # Labels for a code array. The discovery kernels skip masked cells, so what
471
+ # reaches here is real codes — but it is decoded like any other code.
355
472
  def labels_for (code_array)
356
- CA_OBJECT(code_array.to_a.map { |c| @labels[c] })
473
+ CA_OBJECT(code_array.to_a.map { |c| label_at(c) })
357
474
  end
358
475
 
359
476
  # This categorical's cells as their labels; a masked cell stays masked.
@@ -521,7 +638,7 @@ class CArray
521
638
  # Integer / Float keys stay distinct (eql?, so 1 and 1.0 are separate
522
639
  # categories). The discovery path below is reserved for sort_labels (which
523
640
  # reorders the vocabulary, desyncing the appearance-order codes), an explicit
524
- # labels list, and the dtypes the factorize kernel does not take (complex).
641
+ # labels list, and the data types the factorize kernel does not take (complex).
525
642
  if labels.nil? && !sort_labels && (integer? || float? || object? || fixlen? || boolean?)
526
643
  codes, levels = __factorize_appearance__
527
644
  return CACategorical.from_codes(codes, levels.to_a)
@@ -541,7 +658,7 @@ class CArray
541
658
  end
542
659
  end
543
660
 
544
- # Choose a narrow unsigned code dtype, reserving its top value as the
661
+ # Choose a narrow unsigned code type, reserving its top value as the
545
662
  # exclusion sentinel so it never collides with a real code 0..k-1.
546
663
  k = labels_arr.size
547
664
  code_type, sentinel =