carray 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +2 -1
- data/CHANGELOG.md +845 -3
- data/{NEWS.md → CHANGELOG.v1.md} +1 -1
- data/README.md +33 -33
- data/carray.gemspec +12 -8
- data/ext/ca_axis_dispatch.c +33 -4
- data/ext/ca_axis_group.c +256 -114
- data/ext/ca_binop_dispatch.c +5 -6
- data/ext/ca_binop_dispatch.h +0 -7
- data/ext/ca_categorical_iterator.c +121 -67
- data/ext/ca_for_each_element.h +6 -8
- data/ext/ca_group_iter.c +1 -1
- data/ext/ca_kernel_iterator.c +386 -74
- data/ext/ca_kernel_iterator.h +145 -45
- data/ext/ca_obj_array.c +62 -20
- data/ext/ca_obj_bincmp.c +57 -27
- data/ext/ca_obj_binop.c +86 -30
- data/ext/ca_obj_block.c +4 -4
- data/ext/ca_obj_const_string.c +88 -27
- data/ext/ca_obj_face.c +104 -0
- data/ext/ca_obj_face.h +48 -1
- data/ext/ca_obj_fake.c +11 -0
- data/ext/ca_obj_fixlen_string.c +21 -6
- data/ext/ca_obj_grid.c +11 -3
- data/ext/ca_obj_meld.c +125 -22
- data/ext/ca_obj_moncmp.c +33 -14
- data/ext/ca_obj_monop.c +19 -7
- data/ext/ca_obj_object.c +107 -4
- data/ext/ca_obj_record.c +3 -1
- data/ext/ca_obj_refer.c +30 -6
- data/ext/ca_obj_roll.c +13 -5
- data/ext/ca_obj_select.c +49 -34
- data/ext/ca_obj_select_axis.c +16 -6
- data/ext/ca_obj_shift.c +3 -3
- data/ext/ca_obj_stack.c +3 -8
- data/ext/ca_obj_stride.c +178 -24
- data/ext/ca_obj_string.c +11 -5
- data/ext/ca_obj_tile.c +13 -5
- data/ext/ca_obj_time.c +3 -1
- data/ext/ca_obj_timedelta.c +3 -1
- data/ext/ca_obj_transpose.c +1 -1
- data/ext/ca_obj_triop.c +62 -23
- data/ext/ca_obj_window.c +94 -19
- data/ext/ca_op_cmplx64.h +123 -0
- data/ext/ca_op_ipower.c +1 -5
- data/ext/ca_rng_normal.h +42 -0
- data/ext/ca_rng_xoshiro256pp.h +105 -0
- data/ext/ca_sort_kernels.h +5 -5
- data/ext/ca_sweep_engine.c +362 -155
- data/ext/ca_sweep_engine.h +39 -9
- data/ext/ca_transform_common.c +7 -0
- data/ext/carray.h +148 -132
- data/ext/carray_access.c +83 -49
- data/ext/carray_address_basis.c +590 -0
- data/ext/carray_bincount.c +8 -8
- data/ext/carray_broadcast.c +102 -9
- data/ext/carray_build_flags.h +3 -0
- data/ext/carray_call_cfunc.c +2010 -335
- data/ext/carray_call_cfunc.h +153 -0
- data/ext/carray_cast.c +184 -72
- data/ext/carray_conversion.c +22 -22
- data/ext/carray_copy.c +67 -32
- data/ext/carray_core.c +140 -13
- data/ext/carray_count.c +10 -11
- data/ext/carray_element.c +7 -7
- data/ext/carray_factorize.c +72 -51
- data/ext/carray_hold.c +1 -1
- data/ext/carray_index_classifier.c +7 -21
- data/ext/carray_internal.h +36 -3
- data/ext/carray_kernels_bincmp.c +1 -0
- data/ext/carray_kernels_binop.c +320 -298
- data/ext/carray_kernels_init.c +1095 -0
- data/ext/carray_kernels_map.c +1 -0
- data/ext/carray_kernels_moncmp.c +1 -0
- data/ext/carray_kernels_monop.c +412 -411
- data/ext/carray_kernels_reduce_aggregate.c +256 -87
- data/ext/carray_kernels_reduce_boolean.c +1 -0
- data/ext/carray_kernels_reduce_cumulative.c +324 -54
- data/ext/carray_kernels_reduce_extreme.c +605 -58
- data/ext/carray_kernels_reduce_variance.c +1 -0
- data/ext/carray_kernels_scan.c +5 -4
- data/ext/carray_kernels_search.c +187 -106
- data/ext/carray_kernels_sort.c +1 -0
- data/ext/carray_kernels_triop.c +1 -0
- data/ext/carray_lazy.c +175 -5
- data/ext/carray_loop.c +7 -1
- data/ext/carray_mask.c +29 -13
- data/ext/carray_median_percentile.c +56 -1
- data/ext/carray_memory_view.c +46 -21
- data/ext/carray_operator.c +50 -53
- data/ext/carray_order.c +1 -1
- data/ext/carray_partition.c +4 -4
- data/ext/carray_random.c +391 -47
- data/ext/carray_scatter.c +1 -1
- data/ext/carray_slab.c +13 -0
- data/ext/carray_sort.c +22 -24
- data/ext/carray_sort_kernel.c +10 -10
- data/ext/carray_test.c +1 -1
- data/ext/extconf.rb +18 -0
- data/ext/mk_call_cfunc.rb +323 -145
- data/ext/mkkernel.rb +651 -129
- data/ext/ruby_carray.c +28 -17
- data/ext/version.h +4 -4
- data/lib/carray/autoload_carray.rb +24 -3
- data/lib/carray/autoload_method_extension.rb +14 -1
- data/lib/carray/axis_group.rb +84 -7
- data/lib/carray/basics.rb +63 -59
- data/lib/carray/bincount_nd.rb +12 -12
- data/lib/carray/block_iterator.rb +111 -25
- data/lib/carray/categorical.rb +154 -37
- data/lib/carray/categorical_iterator.rb +300 -151
- data/lib/carray/conditional.rb +14 -14
- data/lib/carray/const_string.rb +131 -27
- data/lib/carray/construct.rb +61 -0
- data/lib/carray/core_extensions.rb +62 -5
- data/lib/carray/data_type_extension.rb +21 -1
- data/lib/carray/data_type_limits.rb +91 -0
- data/lib/carray/fixlen_string.rb +1 -1
- data/lib/carray/frame/concat.rb +3 -3
- data/lib/carray/frame/convert.rb +1 -1
- data/lib/carray/frame/csv_parser.rb +11 -4
- data/lib/carray/frame/frame.rb +85 -21
- data/lib/carray/frame/group.rb +36 -3
- data/lib/carray/frame/io.rb +224 -15
- data/lib/carray/frame/records.rb +18 -4
- data/lib/carray/frame/sort.rb +1 -1
- data/lib/carray/frame/verbs.rb +32 -12
- data/lib/carray/fuse_source.rb +123 -0
- data/lib/carray/fusion.rb +218 -0
- data/lib/carray/histogram.rb +16 -16
- data/lib/carray/inspect.rb +43 -16
- data/lib/carray/iterator.rb +147 -3
- data/lib/carray/lazy.rb +88 -73
- data/lib/carray/mask_gap_fill.rb +3 -1
- data/lib/carray/meld_reduce.rb +2 -2
- data/lib/carray/methods/align_addr.rb +1 -1
- data/lib/carray/methods/composition.rb +1 -1
- data/lib/carray/methods/discovery_along.rb +74 -0
- data/lib/carray/methods/factorize.rb +50 -0
- data/lib/carray/methods/is_in.rb +25 -14
- data/lib/carray/methods/locate_addr.rb +81 -3
- data/lib/carray/methods/mask_duplicates.rb +36 -2
- data/lib/carray/methods/meshgrid.rb +4 -5
- data/lib/carray/methods/mode.rb +2 -2
- data/lib/carray/methods/nunique.rb +23 -2
- data/lib/carray/methods/repeat.rb +110 -0
- data/lib/carray/methods/snap.rb +7 -2
- data/lib/carray/methods/unique.rb +44 -4
- data/lib/carray/methods/value_counts.rb +2 -2
- data/lib/carray/rng.rb +86 -0
- data/lib/carray/runtime.rb +0 -19
- data/lib/carray/slab_iterator.rb +77 -19
- data/lib/carray/string_operation_extension.rb +10 -6
- data/lib/carray/time.rb +906 -491
- data/lib/carray/window_iterator.rb +421 -27
- data/lib/carray.rb +7 -5
- data/yard-stubs/ca_obj_array.rb +385 -0
- data/yard-stubs/ca_obj_bitarray.rb +38 -0
- data/yard-stubs/ca_obj_bitfield.rb +43 -0
- data/yard-stubs/ca_obj_block.rb +68 -0
- data/yard-stubs/ca_obj_byte_swap.rb +56 -0
- data/yard-stubs/ca_obj_fake.rb +31 -0
- data/yard-stubs/ca_obj_farray.rb +32 -0
- data/yard-stubs/ca_obj_field.rb +45 -0
- data/yard-stubs/ca_obj_grid.rb +35 -0
- data/yard-stubs/ca_obj_refer.rb +72 -0
- data/yard-stubs/ca_obj_roll.rb +45 -0
- data/yard-stubs/ca_obj_shift.rb +43 -0
- data/yard-stubs/ca_obj_stride.rb +181 -0
- data/yard-stubs/ca_obj_tile.rb +29 -0
- data/yard-stubs/ca_obj_transpose.rb +40 -0
- data/yard-stubs/ca_obj_window.rb +57 -0
- data/yard-stubs/carray_access.rb +131 -0
- data/yard-stubs/carray_attribute.rb +246 -0
- data/yard-stubs/carray_broadcast.rb +37 -0
- data/yard-stubs/carray_cast.rb +489 -0
- data/yard-stubs/carray_class.rb +65 -0
- data/yard-stubs/carray_conversion.rb +76 -0
- data/yard-stubs/carray_copy.rb +79 -0
- data/yard-stubs/carray_core.rb +34 -0
- data/yard-stubs/carray_count.rb +84 -0
- data/yard-stubs/carray_element.rb +108 -0
- data/yard-stubs/carray_generate.rb +66 -0
- data/yard-stubs/carray_lazy.rb +228 -0
- data/yard-stubs/carray_loop.rb +140 -0
- data/yard-stubs/carray_mask.rb +259 -0
- data/yard-stubs/carray_math.rb +1615 -0
- data/yard-stubs/carray_mathfunc.rb +45 -0
- data/yard-stubs/carray_median_percentile.rb +103 -0
- data/yard-stubs/carray_memory_view.rb +163 -0
- data/yard-stubs/carray_order.rb +252 -0
- data/yard-stubs/carray_random.rb +89 -0
- data/yard-stubs/carray_scatter.rb +106 -0
- data/yard-stubs/carray_slab.rb +59 -0
- data/yard-stubs/carray_sort.rb +165 -0
- data/yard-stubs/carray_test.rb +85 -0
- data/yard-stubs/carray_undef.rb +64 -0
- data/yard-stubs/carray_utils.rb +97 -0
- data/yard-stubs/ruby_carray.rb +193 -0
- metadata +67 -10
- data/ext/ca_obj_unbound_repeat.c +0 -496
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
# is a `block_view` (a CAStride: zero-copy over the source via compose-fold),
|
|
20
20
|
# reduced over the trailing tile axes by a core reduction and scattered into the
|
|
21
21
|
# ceil-shaped tile grid. One core reduction per region; the named reductions
|
|
22
|
-
# delegate straight to the core, so their
|
|
22
|
+
# delegate straight to the core, so their data type / mask / empty (ERI) / epsilon
|
|
23
23
|
# contracts are the core's, unchanged. Because tiles do not overlap, no
|
|
24
24
|
# padded entity is built (unlike CAWindowIterator) and the interior stays on
|
|
25
25
|
# the source buffer.
|
|
@@ -165,7 +165,7 @@ class CABlockIterator < CAIterator
|
|
|
165
165
|
# quantile) which are assembled into that many grids. Boundary regions use a
|
|
166
166
|
# smaller (unmasked, ragged) block_view whose present cell count is naturally
|
|
167
167
|
# below a full tile, so `min_count:` marks them UNDEF with no masking. The
|
|
168
|
-
# output
|
|
168
|
+
# output data type is seeded from the first region (all regions share it).
|
|
169
169
|
def assemble
|
|
170
170
|
outs = nil
|
|
171
171
|
each_region do |strip_ranges, tiles, out_ranges|
|
|
@@ -184,7 +184,7 @@ class CABlockIterator < CAIterator
|
|
|
184
184
|
|
|
185
185
|
# Fold every region with a single core reduction `op` (the tier-1 shape).
|
|
186
186
|
def fold (op, **kw)
|
|
187
|
-
assemble { |view, _tiles| view.
|
|
187
|
+
assemble { |view, _tiles| view.public_send(op, axis: @tile_axes, **kw) }
|
|
188
188
|
end
|
|
189
189
|
|
|
190
190
|
public
|
|
@@ -194,14 +194,96 @@ class CABlockIterator < CAIterator
|
|
|
194
194
|
# A per-tile fold to one value over the trailing tile axes is exactly a core
|
|
195
195
|
# per-axis reduction over those axes, so every reduction delegates to
|
|
196
196
|
# `block_view.<op>(axis: tile_axes, ...)` per region. This inherits the core
|
|
197
|
-
#
|
|
197
|
+
# data type, mask, empty / all-masked (identity vs UNDEF) and epsilon-close
|
|
198
198
|
# contracts unchanged. `min_count:` / `fill_value:` pass straight to the core.
|
|
199
199
|
|
|
200
|
-
#
|
|
201
|
-
# Per-tile sum.
|
|
202
|
-
#
|
|
203
|
-
#
|
|
204
|
-
[
|
|
200
|
+
# @!method sum(min_count: nil, fill_value: nil)
|
|
201
|
+
# Per-tile sum.
|
|
202
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
203
|
+
# for a result; a piece with fewer comes back masked.
|
|
204
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
205
|
+
# result instead of leaving it masked.
|
|
206
|
+
# @return [CArray] tile-grid shaped
|
|
207
|
+
# @!method accumulate(min_count: nil, fill_value: nil)
|
|
208
|
+
# Per-tile sum kept in the source's own data type, wrapping at its
|
|
209
|
+
# width, as the core `accumulate` does -- `sum` answers in the type
|
|
210
|
+
# the core promotes to (float64 for integers).
|
|
211
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
212
|
+
# for a result; a piece with fewer comes back masked.
|
|
213
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
214
|
+
# result instead of leaving it masked.
|
|
215
|
+
# @return [CArray] tile-grid shaped
|
|
216
|
+
# @!method prod(min_count: nil, fill_value: nil)
|
|
217
|
+
# Per-tile product.
|
|
218
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
219
|
+
# for a result; a piece with fewer comes back masked.
|
|
220
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
221
|
+
# result instead of leaving it masked.
|
|
222
|
+
# @return [CArray] tile-grid shaped
|
|
223
|
+
# @!method mean(min_count: nil, fill_value: nil)
|
|
224
|
+
# Per-tile arithmetic mean.
|
|
225
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
226
|
+
# for a result; a piece with fewer comes back masked.
|
|
227
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
228
|
+
# result instead of leaving it masked.
|
|
229
|
+
# @return [CArray] tile-grid shaped
|
|
230
|
+
# @!method min(min_count: nil, fill_value: nil)
|
|
231
|
+
# Per-tile minimum.
|
|
232
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
233
|
+
# for a result; a piece with fewer comes back masked.
|
|
234
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
235
|
+
# result instead of leaving it masked.
|
|
236
|
+
# @return [CArray] tile-grid shaped
|
|
237
|
+
# @!method max(min_count: nil, fill_value: nil)
|
|
238
|
+
# Per-tile maximum.
|
|
239
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
240
|
+
# for a result; a piece with fewer comes back masked.
|
|
241
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
242
|
+
# result instead of leaving it masked.
|
|
243
|
+
# @return [CArray] tile-grid shaped
|
|
244
|
+
# @!method variance(min_count: nil, fill_value: nil)
|
|
245
|
+
# Per-tile sample variance (divisor `n - 1`).
|
|
246
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
247
|
+
# for a result; a piece with fewer comes back masked.
|
|
248
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
249
|
+
# result instead of leaving it masked.
|
|
250
|
+
# @return [CArray] tile-grid shaped
|
|
251
|
+
# @!method stddev(min_count: nil, fill_value: nil)
|
|
252
|
+
# Per-tile sample standard deviation (divisor `n - 1`).
|
|
253
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
254
|
+
# for a result; a piece with fewer comes back masked.
|
|
255
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
256
|
+
# result instead of leaving it masked.
|
|
257
|
+
# @return [CArray] tile-grid shaped
|
|
258
|
+
# @!method all(min_count: nil, fill_value: nil)
|
|
259
|
+
# Whether every cell of each tile is true.
|
|
260
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
261
|
+
# for a result; a piece with fewer comes back masked.
|
|
262
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
263
|
+
# result instead of leaving it masked.
|
|
264
|
+
# @return [CArray] tile-grid shaped
|
|
265
|
+
# @!method any(min_count: nil, fill_value: nil)
|
|
266
|
+
# Whether any cell of each tile is true.
|
|
267
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
268
|
+
# for a result; a piece with fewer comes back masked.
|
|
269
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
270
|
+
# result instead of leaving it masked.
|
|
271
|
+
# @return [CArray] tile-grid shaped
|
|
272
|
+
# @!method variancep(min_count: nil, fill_value: nil)
|
|
273
|
+
# Per-tile population variance (divisor `n`).
|
|
274
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
275
|
+
# for a result; a piece with fewer comes back masked.
|
|
276
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
277
|
+
# result instead of leaving it masked.
|
|
278
|
+
# @return [CArray] tile-grid shaped
|
|
279
|
+
# @!method stddevp(min_count: nil, fill_value: nil)
|
|
280
|
+
# Per-tile population standard deviation (divisor `n`).
|
|
281
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
282
|
+
# for a result; a piece with fewer comes back masked.
|
|
283
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
284
|
+
# result instead of leaving it masked.
|
|
285
|
+
# @return [CArray] tile-grid shaped
|
|
286
|
+
[:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
|
|
205
287
|
:variancep, :stddevp].each do |op|
|
|
206
288
|
define_method(op) do |min_count: nil, fill_value: nil|
|
|
207
289
|
kw = {}
|
|
@@ -273,15 +355,19 @@ class CABlockIterator < CAIterator
|
|
|
273
355
|
assemble { |view, _| view.minmax(axis: @tile_axes, **kw) }
|
|
274
356
|
end
|
|
275
357
|
|
|
276
|
-
#
|
|
358
|
+
# @!method min_index
|
|
277
359
|
# Per-tile position of the minimum, as a flat index within the tile (a
|
|
278
360
|
# partial edge tile indexes within its own present cells).
|
|
279
361
|
# @return [CArray] tile-grid shaped
|
|
280
|
-
#
|
|
362
|
+
# @!method max_index
|
|
281
363
|
# Per-tile position of the maximum (tile-local flat index).
|
|
282
364
|
# @return [CArray]
|
|
283
365
|
[:min_index, :max_index].each do |op|
|
|
284
|
-
|
|
366
|
+
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
367
|
+
def #{op}
|
|
368
|
+
assemble { |view, _| view.#{op}(axis: @tile_axes) }
|
|
369
|
+
end
|
|
370
|
+
RUBY
|
|
285
371
|
end
|
|
286
372
|
|
|
287
373
|
# @overload min_addr
|
|
@@ -309,7 +395,7 @@ class CABlockIterator < CAIterator
|
|
|
309
395
|
each_region do |strip_ranges, tiles, out_ranges|
|
|
310
396
|
vview = @source[*strip_ranges].block_view(*tiles)
|
|
311
397
|
sview = saddr[*strip_ranges].block_view(*tiles)
|
|
312
|
-
mi = vview.
|
|
398
|
+
mi = vview.public_send(idx_op, axis: @tile_axes) # tile-local flat index
|
|
313
399
|
grid = (0...@sndim).map { |i| vview.shape[i] }
|
|
314
400
|
cells = tiles.inject(1) { |p, t| p * t }
|
|
315
401
|
safe = mi.copy
|
|
@@ -380,7 +466,7 @@ class CABlockIterator < CAIterator
|
|
|
380
466
|
wt = tiles == @sizes ? weights : weights[*tiles.map { |t| 0...t }]
|
|
381
467
|
wshape = ([1] * @sndim) + tiles
|
|
382
468
|
# Pass the weights through unchanged, like the window iterator: core wsum /
|
|
383
|
-
# wmean own weight/
|
|
469
|
+
# wmean own weight / type coercion, so do not pre-coerce the weights here.
|
|
384
470
|
wfull = wt.reshape(*wshape).broadcast_to(*view.shape)
|
|
385
471
|
yield view, wfull
|
|
386
472
|
end
|
|
@@ -516,23 +602,23 @@ class CABlockIterator < CAIterator
|
|
|
516
602
|
# reusing the same padded (masked-margin) entity as the iterate escape
|
|
517
603
|
# hatches, so this is the slow path (a per-tile materialize), and the
|
|
518
604
|
# out-of-bounds cells of a partial edge tile are dropped from the result.
|
|
519
|
-
# cumsum / cumprod -> float64, cummax / cummin preserve the value
|
|
520
|
-
# cumcount -> int64 running count of present cells; the output
|
|
605
|
+
# cumsum / cumprod -> float64, cummax / cummin preserve the value data type,
|
|
606
|
+
# cumcount -> int64 running count of present cells; the output data type is seeded
|
|
521
607
|
# from the first tile's scan.
|
|
522
608
|
|
|
523
|
-
#
|
|
609
|
+
# @!method cumsum
|
|
524
610
|
# Per-tile inclusive running sum (float64), source-shaped.
|
|
525
611
|
# @return [CArray]
|
|
526
|
-
#
|
|
612
|
+
# @!method cumprod
|
|
527
613
|
# Per-tile inclusive running product (float64), source-shaped.
|
|
528
614
|
# @return [CArray]
|
|
529
|
-
#
|
|
530
|
-
# Per-tile inclusive running maximum (value
|
|
615
|
+
# @!method cummax
|
|
616
|
+
# Per-tile inclusive running maximum (value data type), source-shaped.
|
|
531
617
|
# @return [CArray]
|
|
532
|
-
#
|
|
533
|
-
# Per-tile inclusive running minimum (value
|
|
618
|
+
# @!method cummin
|
|
619
|
+
# Per-tile inclusive running minimum (value data type), source-shaped.
|
|
534
620
|
# @return [CArray]
|
|
535
|
-
#
|
|
621
|
+
# @!method cumcount
|
|
536
622
|
# Per-tile running count of present cells (int64), source-shaped.
|
|
537
623
|
# @return [CArray]
|
|
538
624
|
[:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
|
|
@@ -542,7 +628,7 @@ class CABlockIterator < CAIterator
|
|
|
542
628
|
private
|
|
543
629
|
|
|
544
630
|
# Drive a within-tile segment scan: scan each tile flattened row-major and
|
|
545
|
-
# scatter back to a source-shaped result. The output
|
|
631
|
+
# scatter back to a source-shaped result. The output data type is taken from the
|
|
546
632
|
# first tile's scan (all tiles share it); the padded margin is UNDEF and the
|
|
547
633
|
# OOB cells of a partial edge tile are cropped from the result.
|
|
548
634
|
def block_scan (op)
|
|
@@ -552,7 +638,7 @@ class CABlockIterator < CAIterator
|
|
|
552
638
|
pgv = nil
|
|
553
639
|
CArray.each_index(*@shape) do |*g|
|
|
554
640
|
tile = tgv[*g, *nils]
|
|
555
|
-
scanned = tile.copy.reshape(tile.elements).
|
|
641
|
+
scanned = tile.copy.reshape(tile.elements).public_send(op).reshape(*@sizes)
|
|
556
642
|
unless pout
|
|
557
643
|
pout = CArray.new(scanned.data_type, padded_source.shape)
|
|
558
644
|
pout[] = UNDEF
|
data/lib/carray/categorical.rb
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
#
|
|
3
3
|
# carray/categorical.rb
|
|
4
4
|
#
|
|
5
|
-
# CACategorical — a categorical
|
|
5
|
+
# CACategorical — a categorical data type: dense integer codes (the storage) plus
|
|
6
6
|
# a label vocabulary. Structurally the same as a pandas Categorical or an
|
|
7
7
|
# Arrow dictionary: each element is an index into a small set of categories.
|
|
8
8
|
#
|
|
@@ -17,8 +17,12 @@
|
|
|
17
17
|
# (per-category counts), `cat.codes.count(code)`, etc.
|
|
18
18
|
# - per-cell access = decode the code into its label (`cat[i]` -> category)
|
|
19
19
|
#
|
|
20
|
-
# Exclusion (missing / out-of-vocabulary) is encoded two ways at once
|
|
21
|
-
#
|
|
20
|
+
# Exclusion (missing / out-of-vocabulary) is encoded two ways at once. Both
|
|
21
|
+
# are written when the categorical is built — categorize produces them, and
|
|
22
|
+
# from_codes normalises whatever it receives into them — and the Face is
|
|
23
|
+
# READONLY afterwards, so they stay in step. Both are needed because they have
|
|
24
|
+
# different readers: the axis-group kernel classifies on the code byte, the
|
|
25
|
+
# materialising paths classify on the mask.
|
|
22
26
|
#
|
|
23
27
|
# - the cell is MASKED -> CArray-native idiom: `cat[i]` is
|
|
24
28
|
# UNDEF, `is_masked` / mask-aware
|
|
@@ -66,7 +70,7 @@ class CACategorical < CAObject
|
|
|
66
70
|
}.freeze
|
|
67
71
|
|
|
68
72
|
# The exclusion sentinel per codes data_type: the all-ones bit pattern, read
|
|
69
|
-
# as type-max for an unsigned
|
|
73
|
+
# as type-max for an unsigned data type and as -1 for a signed one. Either way it
|
|
70
74
|
# is out of every valid [0, k) range and byte-identical to a pandas / Arrow
|
|
71
75
|
# missing code.
|
|
72
76
|
SENTINEL = {
|
|
@@ -76,46 +80,148 @@ class CACategorical < CAObject
|
|
|
76
80
|
CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => -1,
|
|
77
81
|
}.freeze
|
|
78
82
|
|
|
83
|
+
# The largest vocabulary each codes data type can carry: every valid code in
|
|
84
|
+
# 0...k has to be representable AND distinct from the exclusion sentinel. An
|
|
85
|
+
# unsigned type spends its top value on the sentinel; a signed one spends -1,
|
|
86
|
+
# which was never a valid index, so a signed type carries one more label than
|
|
87
|
+
# the unsigned type of the same width. categorize picks a width by this rule
|
|
88
|
+
# when it builds codes; from_codes has to check it when it receives them.
|
|
89
|
+
MAX_LABELS = {
|
|
90
|
+
CA_UINT8 => 0xFF, CA_INT8 => 0x80,
|
|
91
|
+
CA_UINT16 => 0xFFFF, CA_INT16 => 0x8000,
|
|
92
|
+
CA_UINT32 => 0xFFFFFFFF, CA_INT32 => 0x80000000,
|
|
93
|
+
CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => 0x8000000000000000,
|
|
94
|
+
}.freeze
|
|
95
|
+
|
|
79
96
|
class << self
|
|
80
97
|
# Wrap already-dense codes + labels with no discovery — the import receiver
|
|
81
|
-
# for a pandas Categorical or an Arrow dictionary.
|
|
82
|
-
# Face's storage parent verbatim
|
|
83
|
-
#
|
|
98
|
+
# for a pandas Categorical or an Arrow dictionary. from_codes takes
|
|
99
|
+
# ownership of `codes`, which becomes the Face's storage parent: verbatim
|
|
100
|
+
# when it is an entity that needs no rewriting, so a wrapped memory view
|
|
101
|
+
# stays a view, and materialised when it is a view over an array the caller
|
|
102
|
+
# still holds, since a view owns no bytes to take ownership of.
|
|
103
|
+
#
|
|
104
|
+
# This is the one door through which an already-built encoding enters, so
|
|
105
|
+
# it validates rather than assumes, and it normalises before handing over:
|
|
106
|
+
# a cell is excluded when it arrives masked OR holds the all-ones sentinel
|
|
107
|
+
# (type-max for unsigned codes, -1 for signed — both the pandas / Arrow
|
|
108
|
+
# missing code), and every excluded cell leaves here holding the sentinel
|
|
109
|
+
# AND masked. Writing both matters because the two encodings have different
|
|
110
|
+
# readers: the axis-group kernel classifies on the code byte, the
|
|
111
|
+
# materialising paths classify on the mask. A categorical that carries only
|
|
112
|
+
# one of them answers membership two ways, silently. An Arrow dictionary
|
|
113
|
+
# carries its missingness in a validity bitmap with arbitrary code bytes —
|
|
114
|
+
# frequently 0 — so arriving masked-only is the normal import, not an edge.
|
|
84
115
|
#
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
# the caller pre-masked. Only the mask buffer is touched; the code bytes
|
|
89
|
-
# are left intact (so a pandas byte-reinterpret round-trips).
|
|
116
|
+
# #initialize marks the codes read-only, so this is also the last point at
|
|
117
|
+
# which they can be written; a read-only argument is copied rather than
|
|
118
|
+
# refused.
|
|
90
119
|
# @overload from_codes(codes, labels)
|
|
91
120
|
# Returns a {CACategorical} wrapping already-dense integer
|
|
92
121
|
# `codes` with the given `labels`, without discovery. `codes`
|
|
93
|
-
# becomes the Face's storage parent
|
|
94
|
-
#
|
|
122
|
+
# becomes the Face's storage parent, materialised first if it is
|
|
123
|
+
# a view. A cell that is masked or holds the type-max sentinel is
|
|
124
|
+
# excluded, and leaves as both.
|
|
95
125
|
# @param codes [CArray] integer code storage.
|
|
96
126
|
# @param labels [Array, CArray] category vocabulary indexed by
|
|
97
|
-
# code.
|
|
127
|
+
# code. Must be unique, and must fit the codes data type with
|
|
128
|
+
# the sentinel reserved.
|
|
98
129
|
# @return [CACategorical]
|
|
99
|
-
# @raise [ArgumentError] when `codes` is not an integer CArray
|
|
130
|
+
# @raise [ArgumentError] when `codes` is not an integer CArray,
|
|
131
|
+
# when `labels` holds duplicates or is too large for the codes
|
|
132
|
+
# data type, or when an unmasked code is outside `0...labels.size`
|
|
133
|
+
# and is not the sentinel.
|
|
100
134
|
def from_codes(codes, labels)
|
|
101
135
|
unless codes.is_a?(CArray) && SENTINEL.key?(codes.data_type)
|
|
102
136
|
got = codes.is_a?(CArray) ? codes.data_type : codes.class
|
|
103
137
|
raise ArgumentError, "from_codes: codes must be an integer CArray (got #{got})"
|
|
104
138
|
end
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
139
|
+
|
|
140
|
+
labels_arr = labels.respond_to?(:to_a) ? labels.to_a : Array(labels)
|
|
141
|
+
if labels_arr.uniq.size != labels_arr.size
|
|
142
|
+
raise ArgumentError, "from_codes: labels must be unique (got duplicates)"
|
|
143
|
+
end
|
|
144
|
+
k = labels_arr.size
|
|
145
|
+
sentinel = SENTINEL[codes.data_type]
|
|
146
|
+
max = MAX_LABELS[codes.data_type]
|
|
147
|
+
if k > max
|
|
148
|
+
raise ArgumentError,
|
|
149
|
+
"from_codes: #{k} labels do not fit #{CArray.data_type_name(codes.data_type)} " \
|
|
150
|
+
"codes, which carry at most #{max} (the top value is reserved as the " \
|
|
151
|
+
"exclusion sentinel); widen the codes data type"
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Classify on the code bytes with the mask set aside (`.value`): a
|
|
155
|
+
# comparison against a masked cell yields UNDEF, which would read as
|
|
156
|
+
# "not excluded" and let the cell through carrying a valid-looking code.
|
|
157
|
+
raw = codes.value
|
|
158
|
+
if sentinel == -1 # signed codes: -1 is the sentinel
|
|
159
|
+
out = raw.lt(0)
|
|
160
|
+
# Only test the upper bound when k is representable in the codes data
|
|
161
|
+
# type. At the very top of the range (k == max) no value can reach k
|
|
162
|
+
# anyway, and comparing against an unrepresentable literal would wrap
|
|
163
|
+
# and flag every cell as out of range.
|
|
164
|
+
out = out | raw.ge(k) if k <= max - 1
|
|
165
|
+
else # unsigned codes: type-max is the sentinel
|
|
166
|
+
out = raw.ge(k) # k <= max here, so always representable
|
|
167
|
+
end
|
|
168
|
+
masked = codes.has_mask? ? codes.is_masked : nil
|
|
169
|
+
excluded = masked ? (out | masked) : out
|
|
170
|
+
# A masked cell may hold any byte at all — that is the CArray contract —
|
|
171
|
+
# so it is never corrupt, only in need of normalising. An *unmasked* cell
|
|
172
|
+
# holding an out-of-range code that is not the sentinel is neither a
|
|
173
|
+
# category nor missingness; refuse it here, where the input is still in
|
|
174
|
+
# the caller's hands, instead of letting it surface later as an IndexError
|
|
175
|
+
# from the grouping plan or as a wrong label from a decode.
|
|
176
|
+
corrupt = out & raw.ne(sentinel)
|
|
177
|
+
corrupt = corrupt & masked.not if masked
|
|
178
|
+
if corrupt.any
|
|
179
|
+
bad = raw[corrupt].to_a.uniq.sort
|
|
180
|
+
shown = bad.first(4).join(", ") + (bad.size > 4 ? ", ..." : "")
|
|
181
|
+
raise ArgumentError,
|
|
182
|
+
"from_codes: code#{bad.size == 1 ? "" : "s"} #{shown} outside " \
|
|
183
|
+
"0...#{k} for #{k} label#{k == 1 ? "" : "s"} " \
|
|
184
|
+
"(use #{sentinel} to exclude a cell, or mask it)"
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Take real ownership, then normalise so the byte reader and the mask
|
|
188
|
+
# reader agree from here on.
|
|
189
|
+
#
|
|
190
|
+
# A view owns no bytes — its root does, and the caller still holds that
|
|
191
|
+
# root. #initialize marks what it is given read-only, which would stop
|
|
192
|
+
# `cat.codes[i] = x` but not `root[i] = x`, so the codes could still
|
|
193
|
+
# change underneath a Face that is supposed to be immutable (and
|
|
194
|
+
# underneath the grouping plan memoised against them). Marking the root
|
|
195
|
+
# instead is worse: it would freeze bytes outside the window the caller
|
|
196
|
+
# handed over. So a view is materialised. Read-only codes are copied for
|
|
197
|
+
# the adjacent reason — the normalising writes need somewhere to land.
|
|
198
|
+
#
|
|
199
|
+
# An entity that needs no rewriting is adopted verbatim, which is what
|
|
200
|
+
# keeps a wrapped memory view a view. That buffer's producer can still
|
|
201
|
+
# write it; that is the borrowed-buffer bargain, and not something this
|
|
202
|
+
# constructor can close.
|
|
203
|
+
needs_mask = masked ? (excluded & masked.not).any : excluded.any
|
|
204
|
+
needs_byte = (excluded & raw.ne(sentinel)).any
|
|
205
|
+
if !codes.entity? || ((needs_mask || needs_byte) && codes.read_only?)
|
|
206
|
+
codes = codes.copy
|
|
108
207
|
end
|
|
109
|
-
|
|
208
|
+
if needs_mask || needs_byte
|
|
209
|
+
codes.value[excluded] = sentinel if needs_byte
|
|
210
|
+
codes.mask = excluded
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
new(codes, labels_arr)
|
|
110
214
|
end
|
|
111
215
|
end
|
|
112
216
|
|
|
113
|
-
# codes : integer CArray, the storage parent
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
#
|
|
117
|
-
#
|
|
118
|
-
#
|
|
217
|
+
# codes : integer CArray, the storage parent, already normalised by the
|
|
218
|
+
# caller (categorize builds it that way; from_codes rewrites what it
|
|
219
|
+
# receives). Excluded cells are both masked AND store the type-max
|
|
220
|
+
# sentinel value (= the all-ones bit pattern, which is signed -1
|
|
221
|
+
# byte-for-byte — the pandas / Arrow missing code). Marking the codes
|
|
222
|
+
# read-only below keeps the two in step from here on, so consumers may
|
|
223
|
+
# rely on either: the mask (CArray-native) or the sentinel
|
|
224
|
+
# (axis-group's out-of-range skip, zero-copy export).
|
|
119
225
|
# labels: Array | CArray, the vocabulary; labels[code] = category.
|
|
120
226
|
# @overload initialize(codes, labels)
|
|
121
227
|
# Allocates a READONLY {CACategorical} Face whose storage is
|
|
@@ -143,8 +249,9 @@ class CACategorical < CAObject
|
|
|
143
249
|
# through views/Faces (a reshape of frozen codes is frozen) and would block
|
|
144
250
|
# the grouping cache from memoising. The flag gives the same write protection
|
|
145
251
|
# (mutations raise) while keeping the object non-frozen. One-way: it takes
|
|
146
|
-
# ownership of `codes
|
|
147
|
-
#
|
|
252
|
+
# ownership of `codes`, which from_codes has already validated, materialised
|
|
253
|
+
# if it was a view, and normalised; a caller keeping a mutable entity of its
|
|
254
|
+
# own must pass `.copy`.
|
|
148
255
|
codes.set_read_only_flag
|
|
149
256
|
end
|
|
150
257
|
|
|
@@ -188,12 +295,11 @@ class CACategorical < CAObject
|
|
|
188
295
|
other.is_a?(CACategorical) && @labels == other.labels
|
|
189
296
|
end
|
|
190
297
|
|
|
191
|
-
# Face hook: decode a per-cell code into its category label
|
|
192
|
-
#
|
|
193
|
-
# wrong category via Ruby negative indexing.
|
|
298
|
+
# Face hook: decode a per-cell code into its category label, through the one
|
|
299
|
+
# decode every code-to-label path shares (see #label_at).
|
|
194
300
|
def storage_to_scalar(raw)
|
|
195
301
|
code = raw.is_a?(String) ? raw.unpack1(UNPACK_FORMAT.fetch(parent.data_type)) : raw
|
|
196
|
-
(code
|
|
302
|
+
label_at(code)
|
|
197
303
|
end
|
|
198
304
|
|
|
199
305
|
# ---- category-space operations (by label, not code) -------------------
|
|
@@ -350,10 +456,21 @@ class CACategorical < CAObject
|
|
|
350
456
|
|
|
351
457
|
private
|
|
352
458
|
|
|
353
|
-
#
|
|
354
|
-
#
|
|
459
|
+
# The one decode from a code to its label. An out-of-range code — a negative
|
|
460
|
+
# one, or an unmasked sentinel that reached us from outside — has no label,
|
|
461
|
+
# so it decodes to nil. Writing `@labels[code]` instead would read a negative
|
|
462
|
+
# code from the *end* of the vocabulary and hand back a real label for a cell
|
|
463
|
+
# that has none. Every code-to-label path goes through here rather than
|
|
464
|
+
# indexing @labels itself, because two decodes are two chances to disagree,
|
|
465
|
+
# and disagreeing is how #unique came to contradict #to_a on the same cell.
|
|
466
|
+
def label_at (code)
|
|
467
|
+
(code < 0 || code >= @labels.size) ? nil : @labels[code]
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
# Labels for a code array. The discovery kernels skip masked cells, so what
|
|
471
|
+
# reaches here is real codes — but it is decoded like any other code.
|
|
355
472
|
def labels_for (code_array)
|
|
356
|
-
CA_OBJECT(code_array.to_a.map { |c|
|
|
473
|
+
CA_OBJECT(code_array.to_a.map { |c| label_at(c) })
|
|
357
474
|
end
|
|
358
475
|
|
|
359
476
|
# This categorical's cells as their labels; a masked cell stays masked.
|
|
@@ -521,7 +638,7 @@ class CArray
|
|
|
521
638
|
# Integer / Float keys stay distinct (eql?, so 1 and 1.0 are separate
|
|
522
639
|
# categories). The discovery path below is reserved for sort_labels (which
|
|
523
640
|
# reorders the vocabulary, desyncing the appearance-order codes), an explicit
|
|
524
|
-
# labels list, and the
|
|
641
|
+
# labels list, and the data types the factorize kernel does not take (complex).
|
|
525
642
|
if labels.nil? && !sort_labels && (integer? || float? || object? || fixlen? || boolean?)
|
|
526
643
|
codes, levels = __factorize_appearance__
|
|
527
644
|
return CACategorical.from_codes(codes, levels.to_a)
|
|
@@ -541,7 +658,7 @@ class CArray
|
|
|
541
658
|
end
|
|
542
659
|
end
|
|
543
660
|
|
|
544
|
-
# Choose a narrow unsigned code
|
|
661
|
+
# Choose a narrow unsigned code type, reserving its top value as the
|
|
545
662
|
# exclusion sentinel so it never collides with a real code 0..k-1.
|
|
546
663
|
k = labels_arr.size
|
|
547
664
|
code_type, sentinel =
|