carray 3.0.1 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +520 -0
- data/README.md +2 -2
- data/carray.gemspec +1 -1
- data/ext/ca_axis_dispatch.c +33 -4
- data/ext/ca_axis_group.c +202 -96
- data/ext/ca_categorical_iterator.c +108 -54
- data/ext/ca_kernel_iterator.c +317 -51
- data/ext/ca_kernel_iterator.h +142 -35
- data/ext/ca_obj_array.c +62 -20
- data/ext/ca_obj_block.c +4 -4
- data/ext/ca_obj_const_string.c +85 -26
- data/ext/ca_obj_face.c +24 -0
- data/ext/ca_obj_face.h +15 -0
- data/ext/ca_obj_fixlen_string.c +18 -5
- data/ext/ca_obj_meld.c +123 -25
- data/ext/ca_obj_object.c +8 -0
- data/ext/ca_obj_select.c +49 -34
- data/ext/ca_obj_stack.c +3 -8
- data/ext/ca_obj_stride.c +72 -1
- data/ext/ca_obj_string.c +8 -4
- data/ext/ca_obj_window.c +8 -2
- data/ext/ca_op_ipower.c +1 -2
- data/ext/ca_rng_normal.h +42 -0
- data/ext/ca_rng_xoshiro256pp.h +105 -0
- data/ext/ca_sweep_engine.c +307 -143
- data/ext/ca_sweep_engine.h +26 -5
- data/ext/carray.h +21 -2
- data/ext/carray_access.c +32 -20
- data/ext/carray_address_basis.c +590 -0
- data/ext/carray_broadcast.c +3 -3
- data/ext/carray_call_cfunc.c +667 -483
- data/ext/carray_cast.c +115 -41
- data/ext/carray_copy.c +55 -30
- data/ext/carray_core.c +83 -3
- data/ext/carray_count.c +9 -10
- data/ext/carray_factorize.c +46 -25
- data/ext/carray_internal.h +17 -0
- data/ext/carray_kernels_reduce_aggregate.c +168 -0
- data/ext/carray_kernels_reduce_cumulative.c +270 -1
- data/ext/carray_kernels_reduce_extreme.c +554 -8
- data/ext/carray_kernels_scan.c +4 -4
- data/ext/carray_kernels_search.c +94 -14
- data/ext/carray_loop.c +7 -1
- data/ext/carray_mask.c +23 -8
- data/ext/carray_median_percentile.c +55 -0
- data/ext/carray_operator.c +4 -4
- data/ext/carray_order.c +1 -1
- data/ext/carray_random.c +384 -40
- data/ext/carray_slab.c +13 -0
- data/ext/carray_sort.c +20 -22
- data/ext/mk_call_cfunc.rb +103 -116
- data/ext/mkkernel.rb +297 -29
- data/ext/ruby_carray.c +10 -1
- data/ext/version.h +4 -4
- data/lib/carray/autoload_carray.rb +5 -3
- data/lib/carray/autoload_method_extension.rb +12 -0
- data/lib/carray/axis_group.rb +77 -0
- data/lib/carray/basics.rb +4 -0
- data/lib/carray/block_iterator.rb +92 -16
- data/lib/carray/categorical.rb +150 -33
- data/lib/carray/categorical_iterator.rb +207 -80
- data/lib/carray/const_string.rb +131 -27
- data/lib/carray/construct.rb +40 -0
- data/lib/carray/data_type_extension.rb +3 -0
- data/lib/carray/data_type_limits.rb +91 -0
- data/lib/carray/fixlen_string.rb +1 -1
- data/lib/carray/frame/csv_parser.rb +11 -4
- data/lib/carray/frame/frame.rb +81 -10
- data/lib/carray/frame/group.rb +36 -3
- data/lib/carray/frame/io.rb +67 -15
- data/lib/carray/frame/records.rb +18 -4
- data/lib/carray/frame/verbs.rb +14 -11
- data/lib/carray/inspect.rb +42 -9
- data/lib/carray/iterator.rb +143 -0
- data/lib/carray/lazy.rb +0 -37
- data/lib/carray/mask_gap_fill.rb +3 -1
- data/lib/carray/methods/discovery_along.rb +74 -0
- data/lib/carray/methods/factorize.rb +50 -0
- data/lib/carray/methods/is_in.rb +13 -2
- data/lib/carray/methods/locate_addr.rb +75 -2
- data/lib/carray/methods/mask_duplicates.rb +35 -1
- data/lib/carray/methods/nunique.rb +22 -1
- data/lib/carray/methods/repeat.rb +110 -0
- data/lib/carray/methods/unique.rb +41 -1
- data/lib/carray/rng.rb +86 -0
- data/lib/carray/slab_iterator.rb +58 -13
- data/lib/carray/string_operation_extension.rb +5 -1
- data/lib/carray/time.rb +18 -2
- data/lib/carray/window_iterator.rb +142 -20
- data/lib/carray.rb +2 -0
- data/yard-stubs/ca_obj_block.rb +2 -7
- data/yard-stubs/ca_obj_window.rb +10 -2
- data/yard-stubs/carray_access.rb +1 -1
- data/yard-stubs/carray_broadcast.rb +1 -1
- data/yard-stubs/carray_core.rb +0 -80
- data/yard-stubs/carray_count.rb +7 -2
- data/yard-stubs/carray_lazy.rb +205 -0
- data/yard-stubs/carray_math.rb +1486 -3
- data/yard-stubs/carray_median_percentile.rb +16 -2
- data/yard-stubs/carray_order.rb +9 -69
- data/yard-stubs/carray_slab.rb +9 -7
- data/yard-stubs/carray_sort.rb +7 -5
- metadata +9 -1
|
@@ -32,6 +32,7 @@ module AutoloadMethodExtension
|
|
|
32
32
|
private
|
|
33
33
|
|
|
34
34
|
def autoload_define (target, name, library, original_spec)
|
|
35
|
+
stub = nil
|
|
35
36
|
target.define_method(name) do |*args, **kwargs, &block|
|
|
36
37
|
begin
|
|
37
38
|
require library
|
|
@@ -39,7 +40,18 @@ module AutoloadMethodExtension
|
|
|
39
40
|
raise "error in autoloading '#{library}' hooked by method " \
|
|
40
41
|
"'#{original_spec}', check gem installation."
|
|
41
42
|
end
|
|
43
|
+
# The require is supposed to have replaced this stub with the real
|
|
44
|
+
# definition. If it has not, the method does not exist anywhere, and
|
|
45
|
+
# forwarding would land straight back here and keep doing so until the
|
|
46
|
+
# stack gave out -- naming neither the method nor the library. Say
|
|
47
|
+
# which, once.
|
|
48
|
+
if target.instance_method(name) == stub
|
|
49
|
+
raise NoMethodError,
|
|
50
|
+
"'#{original_spec}' is registered for autoload from " \
|
|
51
|
+
"'#{library}', but that library defines no such method"
|
|
52
|
+
end
|
|
42
53
|
send(name, *args, **kwargs, &block)
|
|
43
54
|
end
|
|
55
|
+
stub = target.instance_method(name)
|
|
44
56
|
end
|
|
45
57
|
end
|
data/lib/carray/axis_group.rb
CHANGED
|
@@ -308,7 +308,72 @@ end
|
|
|
308
308
|
# / min / max / variance / stddev / variancep / stddevp / count /
|
|
309
309
|
# count_not_masked / all / any) bind in C to one driver; the rest of the common
|
|
310
310
|
# iterator surface that composes cheaply from those is added here.
|
|
311
|
+
# CAGroupIterator subclasses CAIterator, but the class itself is created in
|
|
312
|
+
# C, so the reopening below carries no superclass for YARD to read.
|
|
313
|
+
# @!parse class CAGroupIterator < CAIterator; end
|
|
311
314
|
class CAGroupIterator
|
|
315
|
+
# The shape one value per piece comes back in: a group slot contributes its
|
|
316
|
+
# category count, a band slot its length, in slot order. The base declares
|
|
317
|
+
# these and every other member answers them; this one inherited the readers
|
|
318
|
+
# without anything ever setting the ivars, so it answered nil -- which the
|
|
319
|
+
# base's own rule calls out as the one thing a member must not do ("a clean
|
|
320
|
+
# NotImplementedError, never a wrong answer").
|
|
321
|
+
def shape
|
|
322
|
+
spec.slot_meta.map { |m| m[:kind] == :group ? m[:k] : m[:len] }
|
|
323
|
+
end
|
|
324
|
+
alias dim shape
|
|
325
|
+
|
|
326
|
+
def ndim
|
|
327
|
+
spec.nslots
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# The C dispatcher every reduction shares takes keywords only, so the
|
|
331
|
+
# value-equality and masked forms the base declares -- count(v), count(UNDEF)
|
|
332
|
+
# -- arrived as "wrong number of arguments", which reads as a method that
|
|
333
|
+
# does not take one rather than one whose engine has no such fold. Both are
|
|
334
|
+
# composed here instead, out of folds the engine does have.
|
|
335
|
+
alias __count_present__ count
|
|
336
|
+
|
|
337
|
+
# @overload count
|
|
338
|
+
# Per-group count of present (non-masked) cells.
|
|
339
|
+
# @overload count(UNDEF)
|
|
340
|
+
# Per-group count of masked cells, as {#count_masked}.
|
|
341
|
+
# @overload count(v)
|
|
342
|
+
# Per-group count of cells equal to `v`.
|
|
343
|
+
# @return [CArray]
|
|
344
|
+
def count (*args, **kw)
|
|
345
|
+
return __count_present__(**kw) if args.empty?
|
|
346
|
+
if args.size > 1
|
|
347
|
+
raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0..1)"
|
|
348
|
+
end
|
|
349
|
+
v = args.first
|
|
350
|
+
return count_masked(**kw) if v.equal?(UNDEF)
|
|
351
|
+
# a masked cell equals nothing, and eq marks it UNDEF; drop that to false
|
|
352
|
+
# so it is simply not counted
|
|
353
|
+
hit = value.eq(v)
|
|
354
|
+
hit = hit.strip_mask(false) if hit.has_mask?
|
|
355
|
+
self.class.__build__(hit.int64, spec).sum(**kw).int64
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
alias __all_folded__ all
|
|
359
|
+
alias __any_folded__ any
|
|
360
|
+
|
|
361
|
+
# @overload all
|
|
362
|
+
# Per-group `all` over a boolean payload.
|
|
363
|
+
# @return [CArray]
|
|
364
|
+
def all (**kw)
|
|
365
|
+
boolean_payload!(:all)
|
|
366
|
+
__all_folded__(**kw)
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# @overload any
|
|
370
|
+
# Per-group `any` over a boolean payload.
|
|
371
|
+
# @return [CArray]
|
|
372
|
+
def any (**kw)
|
|
373
|
+
boolean_payload!(:any)
|
|
374
|
+
__any_folded__(**kw)
|
|
375
|
+
end
|
|
376
|
+
|
|
312
377
|
# Per-group classified cell count (mask-independent) = count on the
|
|
313
378
|
# mask-stripped value, so every classified cell is counted regardless of the
|
|
314
379
|
# value mask (unlike count / count_not_masked, which count present cells).
|
|
@@ -549,6 +614,18 @@ class CAGroupIterator
|
|
|
549
614
|
# (the shape a length-K result reshapes to). This is the classification the C
|
|
550
615
|
# scatter kernel computes on the fly, materialised once so the order
|
|
551
616
|
# statistics / iterate / sort_addr can hold each group's members together.
|
|
617
|
+
# Whether a payload folds with all / any is the core's call, not this
|
|
618
|
+
# engine's. The engine counts any non-zero numeric cell as true, which made
|
|
619
|
+
# the group the one member of the family that answered where CArray#all and
|
|
620
|
+
# every sibling refuse -- so `data.all` and `data[g].all(axis: :group)` on
|
|
621
|
+
# the same float array disagreed about whether the question was even
|
|
622
|
+
# askable. Asking a one-cell array of the same data type lets the core's own
|
|
623
|
+
# refusal through, unworded by us.
|
|
624
|
+
def boolean_payload! (op)
|
|
625
|
+
CArray.new(value.data_type, [1]).public_send(op)
|
|
626
|
+
nil
|
|
627
|
+
end
|
|
628
|
+
|
|
552
629
|
def composite_layout
|
|
553
630
|
gslots = spec.slot_meta.select { |m| m[:kind] == :group }
|
|
554
631
|
bslots = spec.slot_meta.select { |m| m[:kind] == :band }
|
data/lib/carray/basics.rb
CHANGED
|
@@ -159,6 +159,10 @@ end
|
|
|
159
159
|
# @raise [ArgumentError] when `offset.length != self.ndim`.
|
|
160
160
|
def paste (offset, src)
|
|
161
161
|
raise ArgumentError, "offset length must equal ndim" if offset.length != ndim
|
|
162
|
+
# A zero-length source covers no cell, so there is nothing to write --
|
|
163
|
+
# and the empty range it would ask for is not a window CAWindow can
|
|
164
|
+
# build. This is what lets concatenate / mosaic take an empty piece.
|
|
165
|
+
return self if src.elements.zero?
|
|
162
166
|
ranges = offset.each_with_index.map { |o, i| o...(o + src.shape[i]) }
|
|
163
167
|
self.window(*ranges)[] = src
|
|
164
168
|
self
|
|
@@ -197,16 +197,92 @@ class CABlockIterator < CAIterator
|
|
|
197
197
|
# data type, mask, empty / all-masked (identity vs UNDEF) and epsilon-close
|
|
198
198
|
# contracts unchanged. `min_count:` / `fill_value:` pass straight to the core.
|
|
199
199
|
|
|
200
|
-
#
|
|
201
|
-
# Per-tile sum.
|
|
202
|
-
#
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
#
|
|
206
|
-
#
|
|
200
|
+
# @!method sum(min_count: nil, fill_value: nil)
|
|
201
|
+
# Per-tile sum.
|
|
202
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
203
|
+
# for a result; a piece with fewer comes back masked.
|
|
204
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
205
|
+
# result instead of leaving it masked.
|
|
206
|
+
# @return [CArray] tile-grid shaped
|
|
207
|
+
# @!method accumulate(min_count: nil, fill_value: nil)
|
|
208
|
+
# Per-tile sum kept in the source's own data type, wrapping at its
|
|
209
|
+
# width, as the core `accumulate` does -- `sum` answers in the type
|
|
210
|
+
# the core promotes to (float64 for integers).
|
|
211
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
212
|
+
# for a result; a piece with fewer comes back masked.
|
|
213
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
214
|
+
# result instead of leaving it masked.
|
|
215
|
+
# @return [CArray] tile-grid shaped
|
|
216
|
+
# @!method prod(min_count: nil, fill_value: nil)
|
|
217
|
+
# Per-tile product.
|
|
218
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
219
|
+
# for a result; a piece with fewer comes back masked.
|
|
220
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
221
|
+
# result instead of leaving it masked.
|
|
222
|
+
# @return [CArray] tile-grid shaped
|
|
223
|
+
# @!method mean(min_count: nil, fill_value: nil)
|
|
224
|
+
# Per-tile arithmetic mean.
|
|
225
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
226
|
+
# for a result; a piece with fewer comes back masked.
|
|
227
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
228
|
+
# result instead of leaving it masked.
|
|
229
|
+
# @return [CArray] tile-grid shaped
|
|
230
|
+
# @!method min(min_count: nil, fill_value: nil)
|
|
231
|
+
# Per-tile minimum.
|
|
232
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
233
|
+
# for a result; a piece with fewer comes back masked.
|
|
234
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
235
|
+
# result instead of leaving it masked.
|
|
236
|
+
# @return [CArray] tile-grid shaped
|
|
237
|
+
# @!method max(min_count: nil, fill_value: nil)
|
|
238
|
+
# Per-tile maximum.
|
|
239
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
240
|
+
# for a result; a piece with fewer comes back masked.
|
|
241
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
242
|
+
# result instead of leaving it masked.
|
|
243
|
+
# @return [CArray] tile-grid shaped
|
|
244
|
+
# @!method variance(min_count: nil, fill_value: nil)
|
|
245
|
+
# Per-tile sample variance (divisor `n - 1`).
|
|
246
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
247
|
+
# for a result; a piece with fewer comes back masked.
|
|
248
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
249
|
+
# result instead of leaving it masked.
|
|
250
|
+
# @return [CArray] tile-grid shaped
|
|
251
|
+
# @!method stddev(min_count: nil, fill_value: nil)
|
|
252
|
+
# Per-tile sample standard deviation (divisor `n - 1`).
|
|
253
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
254
|
+
# for a result; a piece with fewer comes back masked.
|
|
255
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
256
|
+
# result instead of leaving it masked.
|
|
257
|
+
# @return [CArray] tile-grid shaped
|
|
258
|
+
# @!method all(min_count: nil, fill_value: nil)
|
|
259
|
+
# Whether every cell of each tile is true.
|
|
260
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
261
|
+
# for a result; a piece with fewer comes back masked.
|
|
262
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
263
|
+
# result instead of leaving it masked.
|
|
264
|
+
# @return [CArray] tile-grid shaped
|
|
265
|
+
# @!method any(min_count: nil, fill_value: nil)
|
|
266
|
+
# Whether any cell of each tile is true.
|
|
267
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
268
|
+
# for a result; a piece with fewer comes back masked.
|
|
269
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
270
|
+
# result instead of leaving it masked.
|
|
271
|
+
# @return [CArray] tile-grid shaped
|
|
272
|
+
# @!method variancep(min_count: nil, fill_value: nil)
|
|
273
|
+
# Per-tile population variance (divisor `n`).
|
|
274
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
275
|
+
# for a result; a piece with fewer comes back masked.
|
|
276
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
277
|
+
# result instead of leaving it masked.
|
|
278
|
+
# @return [CArray] tile-grid shaped
|
|
279
|
+
# @!method stddevp(min_count: nil, fill_value: nil)
|
|
280
|
+
# Per-tile population standard deviation (divisor `n`).
|
|
281
|
+
# @param min_count [Integer, nil] fewest cells that must be present
|
|
282
|
+
# for a result; a piece with fewer comes back masked.
|
|
283
|
+
# @param fill_value [Object, nil] value to put in place of a masked
|
|
284
|
+
# result instead of leaving it masked.
|
|
207
285
|
# @return [CArray] tile-grid shaped
|
|
208
|
-
# The rest are analogous: prod / mean / min / max, sample and population
|
|
209
|
-
# variance / stddev, all / any.
|
|
210
286
|
[:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
|
|
211
287
|
:variancep, :stddevp].each do |op|
|
|
212
288
|
define_method(op) do |min_count: nil, fill_value: nil|
|
|
@@ -279,11 +355,11 @@ class CABlockIterator < CAIterator
|
|
|
279
355
|
assemble { |view, _| view.minmax(axis: @tile_axes, **kw) }
|
|
280
356
|
end
|
|
281
357
|
|
|
282
|
-
#
|
|
358
|
+
# @!method min_index
|
|
283
359
|
# Per-tile position of the minimum, as a flat index within the tile (a
|
|
284
360
|
# partial edge tile indexes within its own present cells).
|
|
285
361
|
# @return [CArray] tile-grid shaped
|
|
286
|
-
#
|
|
362
|
+
# @!method max_index
|
|
287
363
|
# Per-tile position of the maximum (tile-local flat index).
|
|
288
364
|
# @return [CArray]
|
|
289
365
|
[:min_index, :max_index].each do |op|
|
|
@@ -530,19 +606,19 @@ class CABlockIterator < CAIterator
|
|
|
530
606
|
# cumcount -> int64 running count of present cells; the output data type is seeded
|
|
531
607
|
# from the first tile's scan.
|
|
532
608
|
|
|
533
|
-
#
|
|
609
|
+
# @!method cumsum
|
|
534
610
|
# Per-tile inclusive running sum (float64), source-shaped.
|
|
535
611
|
# @return [CArray]
|
|
536
|
-
#
|
|
612
|
+
# @!method cumprod
|
|
537
613
|
# Per-tile inclusive running product (float64), source-shaped.
|
|
538
614
|
# @return [CArray]
|
|
539
|
-
#
|
|
615
|
+
# @!method cummax
|
|
540
616
|
# Per-tile inclusive running maximum (value data type), source-shaped.
|
|
541
617
|
# @return [CArray]
|
|
542
|
-
#
|
|
618
|
+
# @!method cummin
|
|
543
619
|
# Per-tile inclusive running minimum (value data type), source-shaped.
|
|
544
620
|
# @return [CArray]
|
|
545
|
-
#
|
|
621
|
+
# @!method cumcount
|
|
546
622
|
# Per-tile running count of present cells (int64), source-shaped.
|
|
547
623
|
# @return [CArray]
|
|
548
624
|
[:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
|
data/lib/carray/categorical.rb
CHANGED
|
@@ -17,8 +17,12 @@
|
|
|
17
17
|
# (per-category counts), `cat.codes.count(code)`, etc.
|
|
18
18
|
# - per-cell access = decode the code into its label (`cat[i]` -> category)
|
|
19
19
|
#
|
|
20
|
-
# Exclusion (missing / out-of-vocabulary) is encoded two ways at once
|
|
21
|
-
#
|
|
20
|
+
# Exclusion (missing / out-of-vocabulary) is encoded two ways at once. Both
|
|
21
|
+
# are written when the categorical is built — categorize produces them, and
|
|
22
|
+
# from_codes normalises whatever it receives into them — and the Face is
|
|
23
|
+
# READONLY afterwards, so they stay in step. Both are needed because they have
|
|
24
|
+
# different readers: the axis-group kernel classifies on the code byte, the
|
|
25
|
+
# materialising paths classify on the mask.
|
|
22
26
|
#
|
|
23
27
|
# - the cell is MASKED -> CArray-native idiom: `cat[i]` is
|
|
24
28
|
# UNDEF, `is_masked` / mask-aware
|
|
@@ -76,46 +80,148 @@ class CACategorical < CAObject
|
|
|
76
80
|
CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => -1,
|
|
77
81
|
}.freeze
|
|
78
82
|
|
|
83
|
+
# The largest vocabulary each codes data type can carry: every valid code in
|
|
84
|
+
# 0...k has to be representable AND distinct from the exclusion sentinel. An
|
|
85
|
+
# unsigned type spends its top value on the sentinel; a signed one spends -1,
|
|
86
|
+
# which was never a valid index, so a signed type carries one more label than
|
|
87
|
+
# the unsigned type of the same width. categorize picks a width by this rule
|
|
88
|
+
# when it builds codes; from_codes has to check it when it receives them.
|
|
89
|
+
MAX_LABELS = {
|
|
90
|
+
CA_UINT8 => 0xFF, CA_INT8 => 0x80,
|
|
91
|
+
CA_UINT16 => 0xFFFF, CA_INT16 => 0x8000,
|
|
92
|
+
CA_UINT32 => 0xFFFFFFFF, CA_INT32 => 0x80000000,
|
|
93
|
+
CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => 0x8000000000000000,
|
|
94
|
+
}.freeze
|
|
95
|
+
|
|
79
96
|
class << self
|
|
80
97
|
# Wrap already-dense codes + labels with no discovery — the import receiver
|
|
81
|
-
# for a pandas Categorical or an Arrow dictionary.
|
|
82
|
-
# Face's storage parent verbatim
|
|
83
|
-
#
|
|
98
|
+
# for a pandas Categorical or an Arrow dictionary. from_codes takes
|
|
99
|
+
# ownership of `codes`, which becomes the Face's storage parent: verbatim
|
|
100
|
+
# when it is an entity that needs no rewriting, so a wrapped memory view
|
|
101
|
+
# stays a view, and materialised when it is a view over an array the caller
|
|
102
|
+
# still holds, since a view owns no bytes to take ownership of.
|
|
103
|
+
#
|
|
104
|
+
# This is the one door through which an already-built encoding enters, so
|
|
105
|
+
# it validates rather than assumes, and it normalises before handing over:
|
|
106
|
+
# a cell is excluded when it arrives masked OR holds the all-ones sentinel
|
|
107
|
+
# (type-max for unsigned codes, -1 for signed — both the pandas / Arrow
|
|
108
|
+
# missing code), and every excluded cell leaves here holding the sentinel
|
|
109
|
+
# AND masked. Writing both matters because the two encodings have different
|
|
110
|
+
# readers: the axis-group kernel classifies on the code byte, the
|
|
111
|
+
# materialising paths classify on the mask. A categorical that carries only
|
|
112
|
+
# one of them answers membership two ways, silently. An Arrow dictionary
|
|
113
|
+
# carries its missingness in a validity bitmap with arbitrary code bytes —
|
|
114
|
+
# frequently 0 — so arriving masked-only is the normal import, not an edge.
|
|
84
115
|
#
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
# the caller pre-masked. Only the mask buffer is touched; the code bytes
|
|
89
|
-
# are left intact (so a pandas byte-reinterpret round-trips).
|
|
116
|
+
# #initialize marks the codes read-only, so this is also the last point at
|
|
117
|
+
# which they can be written; a read-only argument is copied rather than
|
|
118
|
+
# refused.
|
|
90
119
|
# @overload from_codes(codes, labels)
|
|
91
120
|
# Returns a {CACategorical} wrapping already-dense integer
|
|
92
121
|
# `codes` with the given `labels`, without discovery. `codes`
|
|
93
|
-
# becomes the Face's storage parent
|
|
94
|
-
#
|
|
122
|
+
# becomes the Face's storage parent, materialised first if it is
|
|
123
|
+
# a view. A cell that is masked or holds the type-max sentinel is
|
|
124
|
+
# excluded, and leaves as both.
|
|
95
125
|
# @param codes [CArray] integer code storage.
|
|
96
126
|
# @param labels [Array, CArray] category vocabulary indexed by
|
|
97
|
-
# code.
|
|
127
|
+
# code. Must be unique, and must fit the codes data type with
|
|
128
|
+
# the sentinel reserved.
|
|
98
129
|
# @return [CACategorical]
|
|
99
|
-
# @raise [ArgumentError] when `codes` is not an integer CArray
|
|
130
|
+
# @raise [ArgumentError] when `codes` is not an integer CArray,
|
|
131
|
+
# when `labels` holds duplicates or is too large for the codes
|
|
132
|
+
# data type, or when an unmasked code is outside `0...labels.size`
|
|
133
|
+
# and is not the sentinel.
|
|
100
134
|
def from_codes(codes, labels)
|
|
101
135
|
unless codes.is_a?(CArray) && SENTINEL.key?(codes.data_type)
|
|
102
136
|
got = codes.is_a?(CArray) ? codes.data_type : codes.class
|
|
103
137
|
raise ArgumentError, "from_codes: codes must be an integer CArray (got #{got})"
|
|
104
138
|
end
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
139
|
+
|
|
140
|
+
labels_arr = labels.respond_to?(:to_a) ? labels.to_a : Array(labels)
|
|
141
|
+
if labels_arr.uniq.size != labels_arr.size
|
|
142
|
+
raise ArgumentError, "from_codes: labels must be unique (got duplicates)"
|
|
143
|
+
end
|
|
144
|
+
k = labels_arr.size
|
|
145
|
+
sentinel = SENTINEL[codes.data_type]
|
|
146
|
+
max = MAX_LABELS[codes.data_type]
|
|
147
|
+
if k > max
|
|
148
|
+
raise ArgumentError,
|
|
149
|
+
"from_codes: #{k} labels do not fit #{CArray.data_type_name(codes.data_type)} " \
|
|
150
|
+
"codes, which carry at most #{max} (the top value is reserved as the " \
|
|
151
|
+
"exclusion sentinel); widen the codes data type"
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Classify on the code bytes with the mask set aside (`.value`): a
|
|
155
|
+
# comparison against a masked cell yields UNDEF, which would read as
|
|
156
|
+
# "not excluded" and let the cell through carrying a valid-looking code.
|
|
157
|
+
raw = codes.value
|
|
158
|
+
if sentinel == -1 # signed codes: -1 is the sentinel
|
|
159
|
+
out = raw.lt(0)
|
|
160
|
+
# Only test the upper bound when k is representable in the codes data
|
|
161
|
+
# type. At the very top of the range (k == max) no value can reach k
|
|
162
|
+
# anyway, and comparing against an unrepresentable literal would wrap
|
|
163
|
+
# and flag every cell as out of range.
|
|
164
|
+
out = out | raw.ge(k) if k <= max - 1
|
|
165
|
+
else # unsigned codes: type-max is the sentinel
|
|
166
|
+
out = raw.ge(k) # k <= max here, so always representable
|
|
167
|
+
end
|
|
168
|
+
masked = codes.has_mask? ? codes.is_masked : nil
|
|
169
|
+
excluded = masked ? (out | masked) : out
|
|
170
|
+
# A masked cell may hold any byte at all — that is the CArray contract —
|
|
171
|
+
# so it is never corrupt, only in need of normalising. An *unmasked* cell
|
|
172
|
+
# holding an out-of-range code that is not the sentinel is neither a
|
|
173
|
+
# category nor missingness; refuse it here, where the input is still in
|
|
174
|
+
# the caller's hands, instead of letting it surface later as an IndexError
|
|
175
|
+
# from the grouping plan or as a wrong label from a decode.
|
|
176
|
+
corrupt = out & raw.ne(sentinel)
|
|
177
|
+
corrupt = corrupt & masked.not if masked
|
|
178
|
+
if corrupt.any
|
|
179
|
+
bad = raw[corrupt].to_a.uniq.sort
|
|
180
|
+
shown = bad.first(4).join(", ") + (bad.size > 4 ? ", ..." : "")
|
|
181
|
+
raise ArgumentError,
|
|
182
|
+
"from_codes: code#{bad.size == 1 ? "" : "s"} #{shown} outside " \
|
|
183
|
+
"0...#{k} for #{k} label#{k == 1 ? "" : "s"} " \
|
|
184
|
+
"(use #{sentinel} to exclude a cell, or mask it)"
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Take real ownership, then normalise so the byte reader and the mask
|
|
188
|
+
# reader agree from here on.
|
|
189
|
+
#
|
|
190
|
+
# A view owns no bytes — its root does, and the caller still holds that
|
|
191
|
+
# root. #initialize marks what it is given read-only, which would stop
|
|
192
|
+
# `cat.codes[i] = x` but not `root[i] = x`, so the codes could still
|
|
193
|
+
# change underneath a Face that is supposed to be immutable (and
|
|
194
|
+
# underneath the grouping plan memoised against them). Marking the root
|
|
195
|
+
# instead is worse: it would freeze bytes outside the window the caller
|
|
196
|
+
# handed over. So a view is materialised. Read-only codes are copied for
|
|
197
|
+
# the adjacent reason — the normalising writes need somewhere to land.
|
|
198
|
+
#
|
|
199
|
+
# An entity that needs no rewriting is adopted verbatim, which is what
|
|
200
|
+
# keeps a wrapped memory view a view. That buffer's producer can still
|
|
201
|
+
# write it; that is the borrowed-buffer bargain, and not something this
|
|
202
|
+
# constructor can close.
|
|
203
|
+
needs_mask = masked ? (excluded & masked.not).any : excluded.any
|
|
204
|
+
needs_byte = (excluded & raw.ne(sentinel)).any
|
|
205
|
+
if !codes.entity? || ((needs_mask || needs_byte) && codes.read_only?)
|
|
206
|
+
codes = codes.copy
|
|
108
207
|
end
|
|
109
|
-
|
|
208
|
+
if needs_mask || needs_byte
|
|
209
|
+
codes.value[excluded] = sentinel if needs_byte
|
|
210
|
+
codes.mask = excluded
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
new(codes, labels_arr)
|
|
110
214
|
end
|
|
111
215
|
end
|
|
112
216
|
|
|
113
|
-
# codes : integer CArray, the storage parent
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
#
|
|
117
|
-
#
|
|
118
|
-
#
|
|
217
|
+
# codes : integer CArray, the storage parent, already normalised by the
|
|
218
|
+
# caller (categorize builds it that way; from_codes rewrites what it
|
|
219
|
+
# receives). Excluded cells are both masked AND store the type-max
|
|
220
|
+
# sentinel value (= the all-ones bit pattern, which is signed -1
|
|
221
|
+
# byte-for-byte — the pandas / Arrow missing code). Marking the codes
|
|
222
|
+
# read-only below keeps the two in step from here on, so consumers may
|
|
223
|
+
# rely on either: the mask (CArray-native) or the sentinel
|
|
224
|
+
# (axis-group's out-of-range skip, zero-copy export).
|
|
119
225
|
# labels: Array | CArray, the vocabulary; labels[code] = category.
|
|
120
226
|
# @overload initialize(codes, labels)
|
|
121
227
|
# Allocates a READONLY {CACategorical} Face whose storage is
|
|
@@ -143,8 +249,9 @@ class CACategorical < CAObject
|
|
|
143
249
|
# through views/Faces (a reshape of frozen codes is frozen) and would block
|
|
144
250
|
# the grouping cache from memoising. The flag gives the same write protection
|
|
145
251
|
# (mutations raise) while keeping the object non-frozen. One-way: it takes
|
|
146
|
-
# ownership of `codes
|
|
147
|
-
#
|
|
252
|
+
# ownership of `codes`, which from_codes has already validated, materialised
|
|
253
|
+
# if it was a view, and normalised; a caller keeping a mutable entity of its
|
|
254
|
+
# own must pass `.copy`.
|
|
148
255
|
codes.set_read_only_flag
|
|
149
256
|
end
|
|
150
257
|
|
|
@@ -188,12 +295,11 @@ class CACategorical < CAObject
|
|
|
188
295
|
other.is_a?(CACategorical) && @labels == other.labels
|
|
189
296
|
end
|
|
190
297
|
|
|
191
|
-
# Face hook: decode a per-cell code into its category label
|
|
192
|
-
#
|
|
193
|
-
# wrong category via Ruby negative indexing.
|
|
298
|
+
# Face hook: decode a per-cell code into its category label, through the one
|
|
299
|
+
# decode every code-to-label path shares (see #label_at).
|
|
194
300
|
def storage_to_scalar(raw)
|
|
195
301
|
code = raw.is_a?(String) ? raw.unpack1(UNPACK_FORMAT.fetch(parent.data_type)) : raw
|
|
196
|
-
(code
|
|
302
|
+
label_at(code)
|
|
197
303
|
end
|
|
198
304
|
|
|
199
305
|
# ---- category-space operations (by label, not code) -------------------
|
|
@@ -350,10 +456,21 @@ class CACategorical < CAObject
|
|
|
350
456
|
|
|
351
457
|
private
|
|
352
458
|
|
|
353
|
-
#
|
|
354
|
-
#
|
|
459
|
+
# The one decode from a code to its label. An out-of-range code — a negative
|
|
460
|
+
# one, or an unmasked sentinel that reached us from outside — has no label,
|
|
461
|
+
# so it decodes to nil. Writing `@labels[code]` instead would read a negative
|
|
462
|
+
# code from the *end* of the vocabulary and hand back a real label for a cell
|
|
463
|
+
# that has none. Every code-to-label path goes through here rather than
|
|
464
|
+
# indexing @labels itself, because two decodes are two chances to disagree,
|
|
465
|
+
# and disagreeing is how #unique came to contradict #to_a on the same cell.
|
|
466
|
+
def label_at (code)
|
|
467
|
+
(code < 0 || code >= @labels.size) ? nil : @labels[code]
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
# Labels for a code array. The discovery kernels skip masked cells, so what
|
|
471
|
+
# reaches here is real codes — but it is decoded like any other code.
|
|
355
472
|
def labels_for (code_array)
|
|
356
|
-
CA_OBJECT(code_array.to_a.map { |c|
|
|
473
|
+
CA_OBJECT(code_array.to_a.map { |c| label_at(c) })
|
|
357
474
|
end
|
|
358
475
|
|
|
359
476
|
# This categorical's cells as their labels; a masked cell stays masked.
|