carray 3.0.1 → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +520 -0
  3. data/README.md +2 -2
  4. data/carray.gemspec +1 -1
  5. data/ext/ca_axis_dispatch.c +33 -4
  6. data/ext/ca_axis_group.c +202 -96
  7. data/ext/ca_categorical_iterator.c +108 -54
  8. data/ext/ca_kernel_iterator.c +317 -51
  9. data/ext/ca_kernel_iterator.h +142 -35
  10. data/ext/ca_obj_array.c +62 -20
  11. data/ext/ca_obj_block.c +4 -4
  12. data/ext/ca_obj_const_string.c +85 -26
  13. data/ext/ca_obj_face.c +24 -0
  14. data/ext/ca_obj_face.h +15 -0
  15. data/ext/ca_obj_fixlen_string.c +18 -5
  16. data/ext/ca_obj_meld.c +123 -25
  17. data/ext/ca_obj_object.c +8 -0
  18. data/ext/ca_obj_select.c +49 -34
  19. data/ext/ca_obj_stack.c +3 -8
  20. data/ext/ca_obj_stride.c +72 -1
  21. data/ext/ca_obj_string.c +8 -4
  22. data/ext/ca_obj_window.c +8 -2
  23. data/ext/ca_op_ipower.c +1 -2
  24. data/ext/ca_rng_normal.h +42 -0
  25. data/ext/ca_rng_xoshiro256pp.h +105 -0
  26. data/ext/ca_sweep_engine.c +307 -143
  27. data/ext/ca_sweep_engine.h +26 -5
  28. data/ext/carray.h +21 -2
  29. data/ext/carray_access.c +32 -20
  30. data/ext/carray_address_basis.c +590 -0
  31. data/ext/carray_broadcast.c +3 -3
  32. data/ext/carray_call_cfunc.c +667 -483
  33. data/ext/carray_cast.c +115 -41
  34. data/ext/carray_copy.c +55 -30
  35. data/ext/carray_core.c +83 -3
  36. data/ext/carray_count.c +9 -10
  37. data/ext/carray_factorize.c +46 -25
  38. data/ext/carray_internal.h +17 -0
  39. data/ext/carray_kernels_reduce_aggregate.c +168 -0
  40. data/ext/carray_kernels_reduce_cumulative.c +270 -1
  41. data/ext/carray_kernels_reduce_extreme.c +554 -8
  42. data/ext/carray_kernels_scan.c +4 -4
  43. data/ext/carray_kernels_search.c +94 -14
  44. data/ext/carray_loop.c +7 -1
  45. data/ext/carray_mask.c +23 -8
  46. data/ext/carray_median_percentile.c +55 -0
  47. data/ext/carray_operator.c +4 -4
  48. data/ext/carray_order.c +1 -1
  49. data/ext/carray_random.c +384 -40
  50. data/ext/carray_slab.c +13 -0
  51. data/ext/carray_sort.c +20 -22
  52. data/ext/mk_call_cfunc.rb +103 -116
  53. data/ext/mkkernel.rb +297 -29
  54. data/ext/ruby_carray.c +10 -1
  55. data/ext/version.h +4 -4
  56. data/lib/carray/autoload_carray.rb +5 -3
  57. data/lib/carray/autoload_method_extension.rb +12 -0
  58. data/lib/carray/axis_group.rb +77 -0
  59. data/lib/carray/basics.rb +4 -0
  60. data/lib/carray/block_iterator.rb +92 -16
  61. data/lib/carray/categorical.rb +150 -33
  62. data/lib/carray/categorical_iterator.rb +207 -80
  63. data/lib/carray/const_string.rb +131 -27
  64. data/lib/carray/construct.rb +40 -0
  65. data/lib/carray/data_type_extension.rb +3 -0
  66. data/lib/carray/data_type_limits.rb +91 -0
  67. data/lib/carray/fixlen_string.rb +1 -1
  68. data/lib/carray/frame/csv_parser.rb +11 -4
  69. data/lib/carray/frame/frame.rb +81 -10
  70. data/lib/carray/frame/group.rb +36 -3
  71. data/lib/carray/frame/io.rb +67 -15
  72. data/lib/carray/frame/records.rb +18 -4
  73. data/lib/carray/frame/verbs.rb +14 -11
  74. data/lib/carray/inspect.rb +42 -9
  75. data/lib/carray/iterator.rb +143 -0
  76. data/lib/carray/lazy.rb +0 -37
  77. data/lib/carray/mask_gap_fill.rb +3 -1
  78. data/lib/carray/methods/discovery_along.rb +74 -0
  79. data/lib/carray/methods/factorize.rb +50 -0
  80. data/lib/carray/methods/is_in.rb +13 -2
  81. data/lib/carray/methods/locate_addr.rb +75 -2
  82. data/lib/carray/methods/mask_duplicates.rb +35 -1
  83. data/lib/carray/methods/nunique.rb +22 -1
  84. data/lib/carray/methods/repeat.rb +110 -0
  85. data/lib/carray/methods/unique.rb +41 -1
  86. data/lib/carray/rng.rb +86 -0
  87. data/lib/carray/slab_iterator.rb +58 -13
  88. data/lib/carray/string_operation_extension.rb +5 -1
  89. data/lib/carray/time.rb +18 -2
  90. data/lib/carray/window_iterator.rb +142 -20
  91. data/lib/carray.rb +2 -0
  92. data/yard-stubs/ca_obj_block.rb +2 -7
  93. data/yard-stubs/ca_obj_window.rb +10 -2
  94. data/yard-stubs/carray_access.rb +1 -1
  95. data/yard-stubs/carray_broadcast.rb +1 -1
  96. data/yard-stubs/carray_core.rb +0 -80
  97. data/yard-stubs/carray_count.rb +7 -2
  98. data/yard-stubs/carray_lazy.rb +205 -0
  99. data/yard-stubs/carray_math.rb +1486 -3
  100. data/yard-stubs/carray_median_percentile.rb +16 -2
  101. data/yard-stubs/carray_order.rb +9 -69
  102. data/yard-stubs/carray_slab.rb +9 -7
  103. data/yard-stubs/carray_sort.rb +7 -5
  104. metadata +9 -1
@@ -32,6 +32,7 @@ module AutoloadMethodExtension
32
32
  private
33
33
 
34
34
  def autoload_define (target, name, library, original_spec)
35
+ stub = nil
35
36
  target.define_method(name) do |*args, **kwargs, &block|
36
37
  begin
37
38
  require library
@@ -39,7 +40,18 @@ module AutoloadMethodExtension
39
40
  raise "error in autoloading '#{library}' hooked by method " \
40
41
  "'#{original_spec}', check gem installation."
41
42
  end
43
+ # The require is supposed to have replaced this stub with the real
44
+ # definition. If it has not, the method does not exist anywhere, and
45
+ # forwarding would land straight back here and keep doing so until the
46
+ # stack gave out -- naming neither the method nor the library. Say
47
+ # which, once.
48
+ if target.instance_method(name) == stub
49
+ raise NoMethodError,
50
+ "'#{original_spec}' is registered for autoload from " \
51
+ "'#{library}', but that library defines no such method"
52
+ end
42
53
  send(name, *args, **kwargs, &block)
43
54
  end
55
+ stub = target.instance_method(name)
44
56
  end
45
57
  end
@@ -308,7 +308,72 @@ end
308
308
  # / min / max / variance / stddev / variancep / stddevp / count /
309
309
  # count_not_masked / all / any) bind in C to one driver; the rest of the common
310
310
  # iterator surface that composes cheaply from those is added here.
311
+ # CAGroupIterator subclasses CAIterator, but the class itself is created in
312
+ # C, so the reopening below carries no superclass for YARD to read.
313
+ # @!parse class CAGroupIterator < CAIterator; end
311
314
  class CAGroupIterator
315
+ # The shape one value per piece comes back in: a group slot contributes its
316
+ # category count, a band slot its length, in slot order. The base declares
317
+ # these and every other member answers them; this one inherited the readers
318
+ # without anything ever setting the ivars, so it answered nil -- which the
319
+ # base's own rule calls out as the one thing a member must not do ("a clean
320
+ # NotImplementedError, never a wrong answer").
321
+ def shape
322
+ spec.slot_meta.map { |m| m[:kind] == :group ? m[:k] : m[:len] }
323
+ end
324
+ alias dim shape
325
+
326
+ def ndim
327
+ spec.nslots
328
+ end
329
+
330
+ # The C dispatcher every reduction shares takes keywords only, so the
331
+ # value-equality and masked forms the base declares -- count(v), count(UNDEF)
332
+ # -- arrived as "wrong number of arguments", which reads as a method that
333
+ # does not take one rather than one whose engine has no such fold. Both are
334
+ # composed here instead, out of folds the engine does have.
335
+ alias __count_present__ count
336
+
337
+ # @overload count
338
+ # Per-group count of present (non-masked) cells.
339
+ # @overload count(UNDEF)
340
+ # Per-group count of masked cells, as {#count_masked}.
341
+ # @overload count(v)
342
+ # Per-group count of cells equal to `v`.
343
+ # @return [CArray]
344
+ def count (*args, **kw)
345
+ return __count_present__(**kw) if args.empty?
346
+ if args.size > 1
347
+ raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0..1)"
348
+ end
349
+ v = args.first
350
+ return count_masked(**kw) if v.equal?(UNDEF)
351
+ # a masked cell equals nothing, and eq marks it UNDEF; drop that to false
352
+ # so it is simply not counted
353
+ hit = value.eq(v)
354
+ hit = hit.strip_mask(false) if hit.has_mask?
355
+ self.class.__build__(hit.int64, spec).sum(**kw).int64
356
+ end
357
+
358
+ alias __all_folded__ all
359
+ alias __any_folded__ any
360
+
361
+ # @overload all
362
+ # Per-group `all` over a boolean payload.
363
+ # @return [CArray]
364
+ def all (**kw)
365
+ boolean_payload!(:all)
366
+ __all_folded__(**kw)
367
+ end
368
+
369
+ # @overload any
370
+ # Per-group `any` over a boolean payload.
371
+ # @return [CArray]
372
+ def any (**kw)
373
+ boolean_payload!(:any)
374
+ __any_folded__(**kw)
375
+ end
376
+
312
377
  # Per-group classified cell count (mask-independent) = count on the
313
378
  # mask-stripped value, so every classified cell is counted regardless of the
314
379
  # value mask (unlike count / count_not_masked, which count present cells).
@@ -549,6 +614,18 @@ class CAGroupIterator
549
614
  # (the shape a length-K result reshapes to). This is the classification the C
550
615
  # scatter kernel computes on the fly, materialised once so the order
551
616
  # statistics / iterate / sort_addr can hold each group's members together.
617
+ # Whether a payload folds with all / any is the core's call, not this
618
+ # engine's. The engine counts any non-zero numeric cell as true, which made
619
+ # the group the one member of the family that answered where CArray#all and
620
+ # every sibling refuse -- so `data.all` and `data[g].all(axis: :group)` on
621
+ # the same float array disagreed about whether the question was even
622
+ # askable. Asking a one-cell array of the same data type lets the core's own
623
+ # refusal through, unworded by us.
624
+ def boolean_payload! (op)
625
+ CArray.new(value.data_type, [1]).public_send(op)
626
+ nil
627
+ end
628
+
552
629
  def composite_layout
553
630
  gslots = spec.slot_meta.select { |m| m[:kind] == :group }
554
631
  bslots = spec.slot_meta.select { |m| m[:kind] == :band }
data/lib/carray/basics.rb CHANGED
@@ -159,6 +159,10 @@ end
159
159
  # @raise [ArgumentError] when `offset.length != self.ndim`.
160
160
  def paste (offset, src)
161
161
  raise ArgumentError, "offset length must equal ndim" if offset.length != ndim
162
+ # A zero-length source covers no cell, so there is nothing to write --
163
+ # and the empty range it would ask for is not a window CAWindow can
164
+ # build. This is what lets concatenate / mosaic take an empty piece.
165
+ return self if src.elements.zero?
162
166
  ranges = offset.each_with_index.map { |o, i| o...(o + src.shape[i]) }
163
167
  self.window(*ranges)[] = src
164
168
  self
@@ -197,16 +197,92 @@ class CABlockIterator < CAIterator
197
197
  # data type, mask, empty / all-masked (identity vs UNDEF) and epsilon-close
198
198
  # contracts unchanged. `min_count:` / `fill_value:` pass straight to the core.
199
199
 
200
- # @overload sum(min_count: nil, fill_value: nil)
201
- # Per-tile sum. @return [CArray] tile-grid shaped
202
- # @overload accumulate(min_count: nil, fill_value: nil)
203
- # Per-tile sum kept in the source's own data type, wrapping at its width,
204
- # as the core `accumulate` does -- `sum` answers in the type the core
205
- # promotes to (float64 for integers), so a tile count over `uint8` cells
206
- # stays one byte wide instead of eight.
200
+ # @!method sum(min_count: nil, fill_value: nil)
201
+ # Per-tile sum.
202
+ # @param min_count [Integer, nil] fewest cells that must be present
203
+ # for a result; a piece with fewer comes back masked.
204
+ # @param fill_value [Object, nil] value to put in place of a masked
205
+ # result instead of leaving it masked.
206
+ # @return [CArray] tile-grid shaped
207
+ # @!method accumulate(min_count: nil, fill_value: nil)
208
+ # Per-tile sum kept in the source's own data type, wrapping at its
209
+ # width, as the core `accumulate` does -- `sum` answers in the type
210
+ # the core promotes to (float64 for integers).
211
+ # @param min_count [Integer, nil] fewest cells that must be present
212
+ # for a result; a piece with fewer comes back masked.
213
+ # @param fill_value [Object, nil] value to put in place of a masked
214
+ # result instead of leaving it masked.
215
+ # @return [CArray] tile-grid shaped
216
+ # @!method prod(min_count: nil, fill_value: nil)
217
+ # Per-tile product.
218
+ # @param min_count [Integer, nil] fewest cells that must be present
219
+ # for a result; a piece with fewer comes back masked.
220
+ # @param fill_value [Object, nil] value to put in place of a masked
221
+ # result instead of leaving it masked.
222
+ # @return [CArray] tile-grid shaped
223
+ # @!method mean(min_count: nil, fill_value: nil)
224
+ # Per-tile arithmetic mean.
225
+ # @param min_count [Integer, nil] fewest cells that must be present
226
+ # for a result; a piece with fewer comes back masked.
227
+ # @param fill_value [Object, nil] value to put in place of a masked
228
+ # result instead of leaving it masked.
229
+ # @return [CArray] tile-grid shaped
230
+ # @!method min(min_count: nil, fill_value: nil)
231
+ # Per-tile minimum.
232
+ # @param min_count [Integer, nil] fewest cells that must be present
233
+ # for a result; a piece with fewer comes back masked.
234
+ # @param fill_value [Object, nil] value to put in place of a masked
235
+ # result instead of leaving it masked.
236
+ # @return [CArray] tile-grid shaped
237
+ # @!method max(min_count: nil, fill_value: nil)
238
+ # Per-tile maximum.
239
+ # @param min_count [Integer, nil] fewest cells that must be present
240
+ # for a result; a piece with fewer comes back masked.
241
+ # @param fill_value [Object, nil] value to put in place of a masked
242
+ # result instead of leaving it masked.
243
+ # @return [CArray] tile-grid shaped
244
+ # @!method variance(min_count: nil, fill_value: nil)
245
+ # Per-tile sample variance (divisor `n - 1`).
246
+ # @param min_count [Integer, nil] fewest cells that must be present
247
+ # for a result; a piece with fewer comes back masked.
248
+ # @param fill_value [Object, nil] value to put in place of a masked
249
+ # result instead of leaving it masked.
250
+ # @return [CArray] tile-grid shaped
251
+ # @!method stddev(min_count: nil, fill_value: nil)
252
+ # Per-tile sample standard deviation (divisor `n - 1`).
253
+ # @param min_count [Integer, nil] fewest cells that must be present
254
+ # for a result; a piece with fewer comes back masked.
255
+ # @param fill_value [Object, nil] value to put in place of a masked
256
+ # result instead of leaving it masked.
257
+ # @return [CArray] tile-grid shaped
258
+ # @!method all(min_count: nil, fill_value: nil)
259
+ # Whether every cell of each tile is true.
260
+ # @param min_count [Integer, nil] fewest cells that must be present
261
+ # for a result; a piece with fewer comes back masked.
262
+ # @param fill_value [Object, nil] value to put in place of a masked
263
+ # result instead of leaving it masked.
264
+ # @return [CArray] tile-grid shaped
265
+ # @!method any(min_count: nil, fill_value: nil)
266
+ # Whether any cell of each tile is true.
267
+ # @param min_count [Integer, nil] fewest cells that must be present
268
+ # for a result; a piece with fewer comes back masked.
269
+ # @param fill_value [Object, nil] value to put in place of a masked
270
+ # result instead of leaving it masked.
271
+ # @return [CArray] tile-grid shaped
272
+ # @!method variancep(min_count: nil, fill_value: nil)
273
+ # Per-tile population variance (divisor `n`).
274
+ # @param min_count [Integer, nil] fewest cells that must be present
275
+ # for a result; a piece with fewer comes back masked.
276
+ # @param fill_value [Object, nil] value to put in place of a masked
277
+ # result instead of leaving it masked.
278
+ # @return [CArray] tile-grid shaped
279
+ # @!method stddevp(min_count: nil, fill_value: nil)
280
+ # Per-tile population standard deviation (divisor `n`).
281
+ # @param min_count [Integer, nil] fewest cells that must be present
282
+ # for a result; a piece with fewer comes back masked.
283
+ # @param fill_value [Object, nil] value to put in place of a masked
284
+ # result instead of leaving it masked.
207
285
  # @return [CArray] tile-grid shaped
208
- # The rest are analogous: prod / mean / min / max, sample and population
209
- # variance / stddev, all / any.
210
286
  [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev, :all, :any,
211
287
  :variancep, :stddevp].each do |op|
212
288
  define_method(op) do |min_count: nil, fill_value: nil|
@@ -279,11 +355,11 @@ class CABlockIterator < CAIterator
279
355
  assemble { |view, _| view.minmax(axis: @tile_axes, **kw) }
280
356
  end
281
357
 
282
- # @overload min_index
358
+ # @!method min_index
283
359
  # Per-tile position of the minimum, as a flat index within the tile (a
284
360
  # partial edge tile indexes within its own present cells).
285
361
  # @return [CArray] tile-grid shaped
286
- # @overload max_index
362
+ # @!method max_index
287
363
  # Per-tile position of the maximum (tile-local flat index).
288
364
  # @return [CArray]
289
365
  [:min_index, :max_index].each do |op|
@@ -530,19 +606,19 @@ class CABlockIterator < CAIterator
530
606
  # cumcount -> int64 running count of present cells; the output data type is seeded
531
607
  # from the first tile's scan.
532
608
 
533
- # @overload cumsum
609
+ # @!method cumsum
534
610
  # Per-tile inclusive running sum (float64), source-shaped.
535
611
  # @return [CArray]
536
- # @overload cumprod
612
+ # @!method cumprod
537
613
  # Per-tile inclusive running product (float64), source-shaped.
538
614
  # @return [CArray]
539
- # @overload cummax
615
+ # @!method cummax
540
616
  # Per-tile inclusive running maximum (value data type), source-shaped.
541
617
  # @return [CArray]
542
- # @overload cummin
618
+ # @!method cummin
543
619
  # Per-tile inclusive running minimum (value data type), source-shaped.
544
620
  # @return [CArray]
545
- # @overload cumcount
621
+ # @!method cumcount
546
622
  # Per-tile running count of present cells (int64), source-shaped.
547
623
  # @return [CArray]
548
624
  [:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
@@ -17,8 +17,12 @@
17
17
  # (per-category counts), `cat.codes.count(code)`, etc.
18
18
  # - per-cell access = decode the code into its label (`cat[i]` -> category)
19
19
  #
20
- # Exclusion (missing / out-of-vocabulary) is encoded two ways at once, and
21
- # because the Face is READONLY they can never desync:
20
+ # Exclusion (missing / out-of-vocabulary) is encoded two ways at once. Both
21
+ # are written when the categorical is built — categorize produces them, and
22
+ # from_codes normalises whatever it receives into them — and the Face is
23
+ # READONLY afterwards, so they stay in step. Both are needed because they have
24
+ # different readers: the axis-group kernel classifies on the code byte, the
25
+ # materialising paths classify on the mask.
22
26
  #
23
27
  # - the cell is MASKED -> CArray-native idiom: `cat[i]` is
24
28
  # UNDEF, `is_masked` / mask-aware
@@ -76,46 +80,148 @@ class CACategorical < CAObject
76
80
  CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => -1,
77
81
  }.freeze
78
82
 
83
+ # The largest vocabulary each codes data type can carry: every valid code in
84
+ # 0...k has to be representable AND distinct from the exclusion sentinel. An
85
+ # unsigned type spends its top value on the sentinel; a signed one spends -1,
86
+ # which was never a valid index, so a signed type carries one more label than
87
+ # the unsigned type of the same width. categorize picks a width by this rule
88
+ # when it builds codes; from_codes has to check it when it receives them.
89
+ MAX_LABELS = {
90
+ CA_UINT8 => 0xFF, CA_INT8 => 0x80,
91
+ CA_UINT16 => 0xFFFF, CA_INT16 => 0x8000,
92
+ CA_UINT32 => 0xFFFFFFFF, CA_INT32 => 0x80000000,
93
+ CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => 0x8000000000000000,
94
+ }.freeze
95
+
79
96
  class << self
80
97
  # Wrap already-dense codes + labels with no discovery — the import receiver
81
- # for a pandas Categorical or an Arrow dictionary. `codes` becomes the
82
- # Face's storage parent verbatim (zero-copy when it is a wrapped memory
83
- # view), and from_codes takes ownership of it.
98
+ # for a pandas Categorical or an Arrow dictionary. from_codes takes
99
+ # ownership of `codes`, which becomes the Face's storage parent: verbatim
100
+ # when it is an entity that needs no rewriting, so a wrapped memory view
101
+ # stays a view, and materialised when it is a view over an array the caller
102
+ # still holds, since a view owns no bytes to take ownership of.
103
+ #
104
+ # This is the one door through which an already-built encoding enters, so
105
+ # it validates rather than assumes, and it normalises before handing over:
106
+ # a cell is excluded when it arrives masked OR holds the all-ones sentinel
107
+ # (type-max for unsigned codes, -1 for signed — both the pandas / Arrow
108
+ # missing code), and every excluded cell leaves here holding the sentinel
109
+ # AND masked. Writing both matters because the two encodings have different
110
+ # readers: the axis-group kernel classifies on the code byte, the
111
+ # materialising paths classify on the mask. A categorical that carries only
112
+ # one of them answers membership two ways, silently. An Arrow dictionary
113
+ # carries its missingness in a validity bitmap with arbitrary code bytes —
114
+ # frequently 0 — so arriving masked-only is the normal import, not an edge.
84
115
  #
85
- # Excluded cells are identified by the all-ones sentinel value (type-max
86
- # for unsigned codes, -1 for signed — both the pandas / Arrow missing code)
87
- # and masked here, so the categorical is well-formed regardless of whether
88
- # the caller pre-masked. Only the mask buffer is touched; the code bytes
89
- # are left intact (so a pandas byte-reinterpret round-trips).
116
+ # #initialize marks the codes read-only, so this is also the last point at
117
+ # which they can be written; a read-only argument is copied rather than
118
+ # refused.
90
119
  # @overload from_codes(codes, labels)
91
120
  # Returns a {CACategorical} wrapping already-dense integer
92
121
  # `codes` with the given `labels`, without discovery. `codes`
93
- # becomes the Face's storage parent; excluded cells (identified
94
- # by the type-max sentinel value) are masked automatically.
122
+ # becomes the Face's storage parent, materialised first if it is
123
+ # a view. A cell that is masked or holds the type-max sentinel is
124
+ # excluded, and leaves as both.
95
125
  # @param codes [CArray] integer code storage.
96
126
  # @param labels [Array, CArray] category vocabulary indexed by
97
- # code.
127
+ # code. Must be unique, and must fit the codes data type with
128
+ # the sentinel reserved.
98
129
  # @return [CACategorical]
99
- # @raise [ArgumentError] when `codes` is not an integer CArray.
130
+ # @raise [ArgumentError] when `codes` is not an integer CArray,
131
+ # when `labels` holds duplicates or is too large for the codes
132
+ # data type, or when an unmasked code is outside `0...labels.size`
133
+ # and is not the sentinel.
100
134
  def from_codes(codes, labels)
101
135
  unless codes.is_a?(CArray) && SENTINEL.key?(codes.data_type)
102
136
  got = codes.is_a?(CArray) ? codes.data_type : codes.class
103
137
  raise ArgumentError, "from_codes: codes must be an integer CArray (got #{got})"
104
138
  end
105
- excluded = codes.eq(SENTINEL[codes.data_type])
106
- if excluded.count(true) > 0
107
- codes.mask = codes.has_mask? ? (codes.mask | excluded) : excluded
139
+
140
+ labels_arr = labels.respond_to?(:to_a) ? labels.to_a : Array(labels)
141
+ if labels_arr.uniq.size != labels_arr.size
142
+ raise ArgumentError, "from_codes: labels must be unique (got duplicates)"
143
+ end
144
+ k = labels_arr.size
145
+ sentinel = SENTINEL[codes.data_type]
146
+ max = MAX_LABELS[codes.data_type]
147
+ if k > max
148
+ raise ArgumentError,
149
+ "from_codes: #{k} labels do not fit #{CArray.data_type_name(codes.data_type)} " \
150
+ "codes, which carry at most #{max} (the top value is reserved as the " \
151
+ "exclusion sentinel); widen the codes data type"
152
+ end
153
+
154
+ # Classify on the code bytes with the mask set aside (`.value`): a
155
+ # comparison against a masked cell yields UNDEF, which would read as
156
+ # "not excluded" and let the cell through carrying a valid-looking code.
157
+ raw = codes.value
158
+ if sentinel == -1 # signed codes: -1 is the sentinel
159
+ out = raw.lt(0)
160
+ # Only test the upper bound when k is representable in the codes data
161
+ # type. At the very top of the range (k == max) no value can reach k
162
+ # anyway, and comparing against an unrepresentable literal would wrap
163
+ # and flag every cell as out of range.
164
+ out = out | raw.ge(k) if k <= max - 1
165
+ else # unsigned codes: type-max is the sentinel
166
+ out = raw.ge(k) # k <= max here, so always representable
167
+ end
168
+ masked = codes.has_mask? ? codes.is_masked : nil
169
+ excluded = masked ? (out | masked) : out
170
+ # A masked cell may hold any byte at all — that is the CArray contract —
171
+ # so it is never corrupt, only in need of normalising. An *unmasked* cell
172
+ # holding an out-of-range code that is not the sentinel is neither a
173
+ # category nor missingness; refuse it here, where the input is still in
174
+ # the caller's hands, instead of letting it surface later as an IndexError
175
+ # from the grouping plan or as a wrong label from a decode.
176
+ corrupt = out & raw.ne(sentinel)
177
+ corrupt = corrupt & masked.not if masked
178
+ if corrupt.any
179
+ bad = raw[corrupt].to_a.uniq.sort
180
+ shown = bad.first(4).join(", ") + (bad.size > 4 ? ", ..." : "")
181
+ raise ArgumentError,
182
+ "from_codes: code#{bad.size == 1 ? "" : "s"} #{shown} outside " \
183
+ "0...#{k} for #{k} label#{k == 1 ? "" : "s"} " \
184
+ "(use #{sentinel} to exclude a cell, or mask it)"
185
+ end
186
+
187
+ # Take real ownership, then normalise so the byte reader and the mask
188
+ # reader agree from here on.
189
+ #
190
+ # A view owns no bytes — its root does, and the caller still holds that
191
+ # root. #initialize marks what it is given read-only, which would stop
192
+ # `cat.codes[i] = x` but not `root[i] = x`, so the codes could still
193
+ # change underneath a Face that is supposed to be immutable (and
194
+ # underneath the grouping plan memoised against them). Marking the root
195
+ # instead is worse: it would freeze bytes outside the window the caller
196
+ # handed over. So a view is materialised. Read-only codes are copied for
197
+ # the adjacent reason — the normalising writes need somewhere to land.
198
+ #
199
+ # An entity that needs no rewriting is adopted verbatim, which is what
200
+ # keeps a wrapped memory view a view. That buffer's producer can still
201
+ # write it; that is the borrowed-buffer bargain, and not something this
202
+ # constructor can close.
203
+ needs_mask = masked ? (excluded & masked.not).any : excluded.any
204
+ needs_byte = (excluded & raw.ne(sentinel)).any
205
+ if !codes.entity? || ((needs_mask || needs_byte) && codes.read_only?)
206
+ codes = codes.copy
108
207
  end
109
- new(codes, labels)
208
+ if needs_mask || needs_byte
209
+ codes.value[excluded] = sentinel if needs_byte
210
+ codes.mask = excluded
211
+ end
212
+
213
+ new(codes, labels_arr)
110
214
  end
111
215
  end
112
216
 
113
- # codes : integer CArray, the storage parent. Excluded cells are both
114
- # masked AND store the type-max sentinel value (= the all-ones bit
115
- # pattern, which is signed -1 byte-for-byte — the pandas / Arrow
116
- # missing code). Because the Face is READONLY the two never desync,
117
- # so consumers may rely on either: the mask (CArray-native) or the
118
- # sentinel (axis-group's out-of-range skip, zero-copy export).
217
+ # codes : integer CArray, the storage parent, already normalised by the
218
+ # caller (categorize builds it that way; from_codes rewrites what it
219
+ # receives). Excluded cells are both masked AND store the type-max
220
+ # sentinel value (= the all-ones bit pattern, which is signed -1
221
+ # byte-for-byte — the pandas / Arrow missing code). Marking the codes
222
+ # read-only below keeps the two in step from here on, so consumers may
223
+ # rely on either: the mask (CArray-native) or the sentinel
224
+ # (axis-group's out-of-range skip, zero-copy export).
119
225
  # labels: Array | CArray, the vocabulary; labels[code] = category.
120
226
  # @overload initialize(codes, labels)
121
227
  # Allocates a READONLY {CACategorical} Face whose storage is
@@ -143,8 +249,9 @@ class CACategorical < CAObject
143
249
  # through views/Faces (a reshape of frozen codes is frozen) and would block
144
250
  # the grouping cache from memoising. The flag gives the same write protection
145
251
  # (mutations raise) while keeping the object non-frozen. One-way: it takes
146
- # ownership of `codes` (categorize / from_codes build or receive it, mask
147
- # already derived above); a caller keeping a mutable array must pass `.copy`.
252
+ # ownership of `codes`, which from_codes has already validated, materialised
253
+ # if it was a view, and normalised; a caller keeping a mutable entity of its
254
+ # own must pass `.copy`.
148
255
  codes.set_read_only_flag
149
256
  end
150
257
 
@@ -188,12 +295,11 @@ class CACategorical < CAObject
188
295
  other.is_a?(CACategorical) && @labels == other.labels
189
296
  end
190
297
 
191
- # Face hook: decode a per-cell code into its category label. An out-of-range
192
- # code (e.g. an unmasked external sentinel) decodes to nil rather than a
193
- # wrong category via Ruby negative indexing.
298
+ # Face hook: decode a per-cell code into its category label, through the one
299
+ # decode every code-to-label path shares (see #label_at).
194
300
  def storage_to_scalar(raw)
195
301
  code = raw.is_a?(String) ? raw.unpack1(UNPACK_FORMAT.fetch(parent.data_type)) : raw
196
- (code < 0 || code >= @labels.size) ? nil : @labels[code]
302
+ label_at(code)
197
303
  end
198
304
 
199
305
  # ---- category-space operations (by label, not code) -------------------
@@ -350,10 +456,21 @@ class CACategorical < CAObject
350
456
 
351
457
  private
352
458
 
353
- # Labels for a code array (the discovery kernels skip masked cells, so the
354
- # code arrays reaching here hold real codes only).
459
+ # The one decode from a code to its label. An out-of-range code — a negative
460
+ # one, or an unmasked sentinel that reached us from outside — has no label,
461
+ # so it decodes to nil. Writing `@labels[code]` instead would read a negative
462
+ # code from the *end* of the vocabulary and hand back a real label for a cell
463
+ # that has none. Every code-to-label path goes through here rather than
464
+ # indexing @labels itself, because two decodes are two chances to disagree,
465
+ # and disagreeing is how #unique came to contradict #to_a on the same cell.
466
+ def label_at (code)
467
+ (code < 0 || code >= @labels.size) ? nil : @labels[code]
468
+ end
469
+
470
+ # Labels for a code array. The discovery kernels skip masked cells, so what
471
+ # reaches here is real codes — but it is decoded like any other code.
355
472
  def labels_for (code_array)
356
- CA_OBJECT(code_array.to_a.map { |c| @labels[c] })
473
+ CA_OBJECT(code_array.to_a.map { |c| label_at(c) })
357
474
  end
358
475
 
359
476
  # This categorical's cells as their labels; a masked cell stays masked.