carray 3.0.3 → 3.0.4

data/lib/carray/bincount_nd.rb

@@ -78,8 +78,13 @@ class CArray
     #     accumulators; `nil` for pure counts (int64).
     #   @return [BincountND]
     def initialize (lengths:, fiber_shape: [], weights_data_type: nil)
-      @lengths = lengths.map(&:to_i)
-      raise ArgumentError, "lengths must be a non-empty list" if @lengths.empty?
+      raise ArgumentError, "lengths must be a non-empty list" if lengths.empty?
+      lengths.each_with_index do |l, k|
+        unless l.is_a?(Integer)
+          raise TypeError, "lengths[#{k}] must be an Integer (got #{l.inspect})"
+        end
+      end
+      @lengths = lengths.dup
       @lengths.each_with_index do |l, k|
         raise ArgumentError, "lengths[#{k}] must be >= 1" if l < 1
       end
@@ -209,6 +214,7 @@ class CArray
 
       if weights
         raise ArgumentError, "weights given but accumulator is unweighted" unless @weighted
+        # coerce: the accumulator's type, declared by weights_data_type:
         weights = CArray.wrap_readonly(weights, @counts_data_type)
         expected_w_shape = chunk.shape.dup
         expected_w_shape.delete_at(channel_ax)
@@ -344,6 +350,10 @@ class CArray
   #   @return [BincountND]
   def bincount_nd (lengths:, axis: [-2, -1], weights: nil)
     raise ArgumentError, "lengths must be an Array of per-dim extents" unless lengths.is_a?(Array)
+    axis = [-2, -1] if axis.nil?
+    unless axis.is_a?(Array) && axis.size == 2
+      raise ArgumentError, "bincount_nd: axis: is a [sample, channel] pair (got #{axis.inspect})"
+    end
     sample_ax  = normalize_axis(axis[0], "bincount_nd sample axis")
     channel_ax = normalize_axis(axis[1], "bincount_nd channel axis")
     fiber_shape = shape.dup

data/lib/carray/block_iterator.rb

@@ -302,28 +302,47 @@ class CABlockIterator < CAIterator
   #   @param fill_value [Object, nil] value to put in place of a masked
   #     result instead of leaving it masked.
   #   @return [CArray] tile-grid shaped
-  [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev,
-   :variancep, :stddevp].each do |op|
-    define_method(op) do |min_count: nil, fill_value: nil|
-      if FACE_WINNER_INDEX.key?(op) && @source.face?
-        return face_winner(FACE_WINNER_INDEX[op], min_count, fill_value)
-      end
-      kw = {}
-      kw[:min_count]  = min_count  unless min_count.nil?
-      kw[:fill_value] = fill_value unless fill_value.nil?
-      fold(op, **kw)
+  CArray.send(:define_named_methods, self,
+              [:sum, :accumulate, :prod, :mean, :min, :max, :variance, :stddev,
+               :variancep, :stddevp], :fold_reduction)
+
+  private def fold_reduction (op, min_count: nil, fill_value: nil)
+    if FACE_WINNER_INDEX.key?(op) && @source.face?
+      return face_winner(FACE_WINNER_INDEX[op], min_count, fill_value)
     end
+    kw = {}
+    kw[:min_count]  = min_count  unless min_count.nil?
+    kw[:fill_value] = fill_value unless fill_value.nil?
+    fold(op, **kw)
   end
 
-  [:all, :any].each do |op|
-    define_method(op) { |**kw| fold(op, **CAIterator.reduce_keywords(op, kw)) }
+  CArray.send(:define_named_methods, self, [:all, :any], :fold_with_keywords)
+
+  private def fold_with_keywords (op, **kw)
+    fold(op, **CAIterator.reduce_keywords(op, kw))
   end
 
+  # @!method first
+  #   Per-tile first present value, in row-major order within the tile;
+  #   UNDEF for a tile with none.
+  #   @return [CArray] tile-grid shaped
+  # @!method last
+  #   Per-tile last present value; UNDEF for a tile with none.
+  #   @return [CArray] tile-grid shaped
+  # @!method nunique
+  #   Per-tile number of distinct present values (int64); 0 for a tile with
+  #   none.
+  #   @return [CArray] tile-grid shaped
+  #
+  # These take no `min_count:` or `fill_value:`, as the core ones do not.
+  CArray.send(:define_named_methods, self, [:first, :last, :nunique], :fold_with_keywords)
+
   # A Face's minimum and maximum are cells of the Face, so they are gathered
   # from the source by the winner's address.  Assembling the per-region results
   # instead copies the Face's storage into an array that no longer knows it is
   # one: a CATime came back as 8-byte strings.
   FACE_WINNER_INDEX = { min: :min_index, max: :max_index }.freeze
+  Ractor.make_shareable(FACE_WINNER_INDEX)
   private_constant :FACE_WINNER_INDEX
 
   # ---- count family and elements ----------------------------------------
@@ -362,11 +381,11 @@ class CABlockIterator < CAIterator
   #   @return [CArray]
   def count_masked (**kw)
     kw   = CAIterator.reduce_keywords(:count_masked, kw)
-    need = CArray.__min_count_need__(:count_masked, kw[:min_count], 0)
+    need = CArray.__present_cells_needed_per_result_cell__(:count_masked, kw[:min_count], 0)
     out  = elements - count_not_masked
     # A count of the mask reads every cell of the tile.
     out[elements.lt(need)] = UNDEF if need > 0
-    CArray.__reduce_fill__(out, kw[:fill_value])
+    CArray.__fill_undefined_result_cells__(out, kw[:fill_value])
   end
 
   # @overload elements
@@ -434,13 +453,14 @@ class CABlockIterator < CAIterator
   # and with `min_count` a tile with too few present cells, is a masked cell.
   def face_winner (idx_op, min_count, fill_value)
     addr = winner_addr(idx_op)
-    need = CArray.__min_count_need__(idx_op == :min_index ? :min : :max, min_count, 0)
+    need = CArray.__present_cells_needed_per_result_cell__(idx_op == :min_index ? :min : :max, min_count, 0)
     addr[count_not_masked.lt(need)] = UNDEF if need > 0
     src   = @source.reshape(@source.elements)
     empty = addr.is_masked
     return src[addr].copy unless empty.any
-    # A masked index drops the cell rather than masking it, so an empty tile
-    # needs a placeholder address.  Pointing it at a masked cell of the source
+    # A masked index would read as masked, but gathering by one gives the
+    # source a mask, so an empty tile takes a placeholder address instead.
+    # Pointing it at a masked cell of the source
     # carries the mask through the gather, which a read-only Face (whose copy
     # cannot be written) needs.  Otherwise mask the placeholder afterwards.
     if fill_value.nil? && src.has_mask? && (hole = src.is_masked.where).elements > 0
@@ -474,7 +494,7 @@ class CABlockIterator < CAIterator
       out ||= CArray.int64(*@shape)
       out[*out_ranges] = picked
     end
-    CArray.__reduce_fill__(out || CArray.int64(*@shape), kw[:fill_value])
+    CArray.__fill_undefined_result_cells__(out || CArray.int64(*@shape), kw[:fill_value])
   end
 
   public
@@ -691,24 +711,81 @@ class CABlockIterator < CAIterator
   # @!method cumcount
   #   Per-tile running count of present cells (int64), source-shaped.
   #   @return [CArray]
-  [:cumsum, :cumprod, :cummax, :cummin, :cumcount].each do |op|
-    define_method(op) { block_scan(op) }
+  CArray.send(:define_named_methods, self,
+              [:cumsum, :cumprod, :cummax, :cummin, :cumcount], :scan_tiles)
+
+  # A scan takes no arguments here.
+  private def scan_tiles (op, *args)
+    unless args.empty?
+      raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0)"
+    end
+    block_scan(op)
+  end
+
+  # @overload lag(n = 1)
+  #   Per-tile value `n` steps earlier, counting the tile's cells row-major;
+  #   UNDEF where the tile has none, so a lag never reaches into another
+  #   tile. A partial edge tile counts only the cells it has.
+  #   @return [CArray] source-shaped
+  def lag (n = 1)
+    unless n.is_a?(Integer)
+      raise TypeError, "lag: n must be an Integer (got #{n.class})"
+    end
+    tile_wise { |rows, axis| rows.lag(n, axis: axis) }
+  end
+
+  # @overload strip_mask(method:)
+  #   Per-tile fill of the masked cells, counting the tile's cells row-major:
+  #   `:forward` / `:ffill` or `:backward` / `:bfill`. A fill never reaches
+  #   into another tile, so a masked cell with nothing to carry in its tile
+  #   stays masked.
+  #   @return [CArray] source-shaped
+  def strip_mask (method:)
+    CAIterator.hold_backward?(method)
+    tile_wise { |rows, axis| rows.strip_mask(method: method, axis: axis) }
+  end
+
+  # @overload order(descending: false, method: :ordinal, kind: :quick)
+  #   Per-tile rank of each cell among the tile's cells (int64, `0` the
+  #   smallest), as `CArray#order` ranks them; a partial edge tile ranks only
+  #   the cells it has. The keywords are those of `CArray#order`.
+  #   @return [CArray] source-shaped
+  def order (**kw)
+    tile_wise { |rows, axis| rows.order(axis: axis, **kw) }
   end
 
   private
 
+  # A source-shaped result built tile by tile: the block gets a region's
+  # tiles as rows (one tile's cells, row-major, along +axis+) and gives back
+  # rows of the same shape. Each region has whole tiles of one size, so a
+  # partial edge tile is its own cells rather than cells and padding.
+  def tile_wise
+    out = nil
+    each_region do |strip_ranges, tiles, _out_ranges|
+      view  = @source[*strip_ranges].block_view(*tiles)
+      grid  = view.shape[0, @sndim]
+      found = yield(view.copy.reshape(*grid, tiles.inject(1, :*)), @sndim)
+                .reshape(*grid, *tiles)
+      out ||= tile_output(found)
+      out[*strip_ranges].block_view(*tiles)[] = tile_values(out, found)
+    end
+    # a source with no cells has no region; its one-cell rows give the type
+    out || yield(@source.copy.reshape(*@source.shape, 1), @sndim).reshape(*@source.shape)
+  end
+
   # Drive a within-tile segment scan: scan each tile flattened row-major and
   # scatter back to a source-shaped result.  The output data type is taken from the
   # first tile's scan (all tiles share it); the padded margin is UNDEF and the
   # OOB cells of a partial edge tile are cropped from the result.
-  def block_scan (op)
+  def block_scan (op, *args)
     tgv  = tile_grid_view
     nils = Array.new(@sndim, nil)
     pout = nil
     pgv  = nil
     CArray.each_index(*@shape) do |*g|
       tile    = tgv[*g, *nils]
-      scanned = tile.copy.reshape(tile.elements).public_send(op).reshape(*@sizes)
+      scanned = tile.copy.reshape(tile.elements).public_send(op, *args).reshape(*@sizes)
       unless pout
         pout = CArray.new(scanned.data_type, padded_source.shape, bytes: scanned.bytes)
         pout[] = UNDEF
@@ -719,6 +796,24 @@ class CABlockIterator < CAIterator
     pout[*@sndim.times.map { |i| 0...@source.shape[i] }].copy
   end
 
+  # A source-shaped array that can hold what #tile_wise gives: of its kind,
+  # or an object array for a read-only Face, which cannot be written cell by
+  # cell.
+  def tile_output (lagged)
+    if lagged.face? && lagged.read_only?
+      CArray.object(*@source.shape)
+    else
+      lagged.__send__(:blank_like, @source.shape)
+    end
+  end
+
+  # The values as +out+ can hold them: a Face going into an object array
+  # goes as its surface values, not its storage.
+  def tile_values (out, lagged)
+    return lagged unless out.data_type == CA_OBJECT && lagged.face?
+    CArray.object(*lagged.shape) { lagged.to_a }
+  end
+
   # A padded entity whose interior holds the source and whose partial-edge
   # margin is masked (UNDEF), sized to exactly cover the ceil tile grid.  When
   # the source already divides evenly this is the source itself (no copy), so
@@ -729,7 +824,7 @@ class CABlockIterator < CAIterator
       if pshape == @source.shape.to_a
         @source
       else
-        pad = CArray.new(@source.data_type, pshape, bytes: @source.bytes)
+        pad = @source.__send__(:blank_like, pshape)   # a Face stays one
         pad[] = UNDEF
         pad[*@sndim.times.map { |i| 0...@source.shape[i] }] = @source
         pad
@@ -742,6 +837,14 @@ class CABlockIterator < CAIterator
   def tile_grid_view
     @tile_grid_view ||= padded_source.block_view(*@sizes)
   end
+
+  # lag and strip_mask(method:) give the source's values moved or held in
+  # place, so the result shows the source's attributes, as CArray#lag and
+  # CArray#strip_mask(method:) do.
+  prepend(Module.new do
+    def lag (*args, **keywords) = @source.__send__(:carry_attrs, super)
+    def strip_mask (**keywords) = @source.__send__(:carry_attrs, super)
+  end)
 end
 
 

data/lib/carray/boolean_reduce.rb

@@ -1,6 +1,6 @@
 # Boolean folds all / any / none gain a `skip_masked:` keyword.
 #
-#   skip_masked: true  (default) -- available-case (skipna): masked cells are
+#   skip_masked: true  (default) -- available-case: masked cells are
 #                                    ignored, the result is always true/false.
 #                                    This is the existing behavior; every other
 #                                    keyword (axis:, keep_axis:, ...) forwards
@@ -14,15 +14,15 @@
 #                                    false < unknown < true).
 #
 # The Kleene fold composes from count(true) / count(false) / count_masked, so
-# there is no dedicated kernel; the C reductions keep the skipna semantics
-# under the `__*_skipna__` aliases.
+# there is no dedicated kernel; the C reductions that skip masked cells stay
+# reachable as the private `*_skipping_masked` aliases.
 
 class CArray
 
-  alias_method :__any_skipna__,  :any
-  alias_method :__all_skipna__,  :all
-  alias_method :__none_skipna__, :none
-  private :__any_skipna__, :__all_skipna__, :__none_skipna__
+  alias_method :any_skipping_masked,  :any
+  alias_method :all_skipping_masked,  :all
+  alias_method :none_skipping_masked, :none
+  private :any_skipping_masked, :all_skipping_masked, :none_skipping_masked
 
   # Whether any cell is true.
   #
@@ -36,8 +36,8 @@ class CArray
   #   `keep_axis:`, ...).
   # @return [Boolean, CArray] a scalar, or an array when an axis is given.
   def any (skip_masked: true, **opts)
-    return __any_skipna__(**opts) if skip_masked
-    __kleene_fold(:any, opts)
+    return any_skipping_masked(**opts) if skip_masked
+    kleene_fold(:any, opts)
   end
 
   # Whether every cell is true.
@@ -52,8 +52,8 @@ class CArray
   #   `keep_axis:`, ...).
   # @return [Boolean, CArray] a scalar, or an array when an axis is given.
   def all (skip_masked: true, **opts)
-    return __all_skipna__(**opts) if skip_masked
-    __kleene_fold(:all, opts)
+    return all_skipping_masked(**opts) if skip_masked
+    kleene_fold(:all, opts)
   end
 
   # Whether no cell is true.
@@ -68,8 +68,8 @@ class CArray
   #   `keep_axis:`, ...).
   # @return [Boolean, CArray] a scalar, or an array when an axis is given.
   def none (skip_masked: true, **opts)
-    return __none_skipna__(**opts) if skip_masked
-    __kleene_fold(:none, opts)
+    return none_skipping_masked(**opts) if skip_masked
+    kleene_fold(:none, opts)
   end
 
   private
@@ -79,11 +79,11 @@ class CArray
   # any, a false for all, a true for none); elsewhere a masked cell leaves the
   # answer undetermined.  Going through that reduction checks the data type,
   # the axes and the keywords exactly as `skip_masked: true` does.
-  def __kleene_fold (kind, opts)
+  def kleene_fold (kind, opts)
     r = case kind
-        when :any then __any_skipna__(**opts)
-        when :all then __all_skipna__(**opts)
-        else           __none_skipna__(**opts)
+        when :any then any_skipping_masked(**opts)
+        when :all then all_skipping_masked(**opts)
+        else           none_skipping_masked(**opts)
         end
     cm = count_masked(**opts)
     if r.is_a?(CArray)

data/lib/carray/categorical.rb

@@ -48,6 +48,7 @@
 # ----------------------------------------------------------------------------
 
 require "carray"
+require "carray/methods/is_in"   # refuse_range_set, for the set-valued verbs
 
 # Categorical column: dense integer codes plus a label vocabulary, so each
 # element is an index into a small set of categories.  Structurally the same
@@ -68,6 +69,7 @@ class CACategorical < CAObject
     CA_INT32 => "l", CA_UINT32 => "L",
     CA_INT64 => "q", CA_UINT64 => "Q",
   }.freeze
+  Ractor.make_shareable(UNPACK_FORMAT)
 
   # The exclusion sentinel per codes data_type: the all-ones bit pattern, read
   # as type-max for an unsigned data type and as -1 for a signed one. Either way it
@@ -214,6 +216,34 @@ class CACategorical < CAObject
     end
   end
 
+  # The rows' categorical for key columns of one length (CAFrame#group_by,
+  # #is_duplicate, windows(on:, by:)).  Each column is categorized on its
+  # own, so a component compares as a single key does (NaN is one value,
+  # -0.0 is 0.0, a masked cell matches nothing); with several columns a
+  # row's category is the tuple of its component codes, labelled by the
+  # tuple of the component labels.  One masked component leaves the row in
+  # no category, as a masked single key does.
+  private_class_method def self.from_keys (cols)
+    return cols.first.categorize if cols.size == 1
+    nrow  = cols.first.elements
+    cats  = cols.map(&:categorize)
+    sizes = cats.map { |c| c.labels.size }
+    tuple = if sizes.inject(1, :*) <= 2**62
+              cats.drop(1).inject(cats[0].codes.int64) { |acc, c| acc * c.labels.size + c.codes.int64 }
+            else
+              codes = cats.map(&:codes)
+              t = CArray.object(nrow) { |i| codes.map { |c| c[i] } }
+              nrow.times { |i| t[i] = UNDEF if t[i].any? { |x| UNDEF.equal?(x) } }
+              t
+            end
+    rows = tuple.categorize
+    labels = rows.labels.map do |code|
+      parts = code.is_a?(Array) ? code : sizes.reverse.map { |n| code, d = code.divmod(n); d }.reverse
+      parts.each_with_index.map { |d, j| cats[j].labels[d] }
+    end
+    from_codes(rows.codes, labels)
+  end
+
   # codes : integer CArray, the storage parent, already normalised by the
   #         caller (categorize builds it that way; from_codes rewrites what it
   #         receives). Excluded cells are both masked AND store the type-max
@@ -235,7 +265,7 @@ class CACategorical < CAObject
     # copy first so a caller's array is never frozen as a side effect; the
     # label objects themselves are left untouched (container-level freeze).
     @labels = (labels.respond_to?(:to_a) ? labels.to_a : Array(labels)).dup.freeze
-    super(CA_FIXLEN, codes.dim,
+    super(CA_FIXLEN, codes.shape,
           bytes:     codes.bytes,
           storage:   codes.data_type,
           parent:    codes,
@@ -309,14 +339,6 @@ class CACategorical < CAObject
   # the integer code. The raw codes stay reachable via #codes for code-space
   # work (interop, ML features), but they are not the everyday surface.
 
-  # Boolean mask of cells whose category == label. Excluded cells stay UNDEF
-  # (their category is unknown); an unknown label yields an all-false mask.
-  # @overload eq(label)
-  #   Returns a boolean CArray marking cells whose category equals
-  #   `label`. Excluded cells stay masked; an unknown label yields
-  #   an all-false result.
-  #   @param label [Object] category to match.
-  #   @return [CArray]
   # @overload resize(*newdim, fill_value: UNDEF)
   #   Returns a categorical resized to `newdim` (see {CArray#resize}).
   #   The new cells are excluded (UNDEF) unless `fill_value` names one of
@@ -338,16 +360,28 @@ class CACategorical < CAObject
     CACategorical.from_codes(codes.insert_block(offset, bsize) { fill }, labels)
   end
 
+  # Boolean mask of cells whose category == label. Excluded cells stay UNDEF
+  # (their category is unknown); an unknown label yields an all-false mask.
+  # @overload eq(label)
+  #   Returns a boolean CArray marking cells whose category equals
+  #   `label`. A CArray is compared cell by cell, by label: another
+  #   categorical is matched whatever order its labels are in, and a
+  #   masked cell on either side gives UNDEF. Anything else, a Ruby Array
+  #   included, is one label, as for {CArray#eq}. An unknown label matches
+  #   nothing.
+  #   @param label [Object, CArray] category to match, or one per cell.
+  #   @return [CArray]
   def eq(label)
-    codes.eq(@labels.index(label) || @labels.size)
+    codes.eq(codes_of(label))
   end
 
   # @overload ne(label)
   #   Returns the complement of {#eq}.
-  #   @param label [Object] category to compare against.
+  #   @param label [Object, CArray] category to compare against, or one per
+  #     cell.
   #   @return [CArray]
   def ne(label)
-    codes.ne(@labels.index(label) || @labels.size)
+    codes.ne(codes_of(label))
   end
 
   # @overload count(label)
@@ -446,6 +480,7 @@ class CACategorical < CAObject
   #   @param values [Array, CArray, CACategorical] labels to test against.
   #   @return [CArray]
   def is_in (values)
+    refuse_range_set(values, "is_in")
     wanted = label_list(values).filter_map { |l| @labels.index(l) }
     codes.is_in(CArray.int64(wanted.size) { |i| wanted[i] })
   end
@@ -453,18 +488,21 @@ class CACategorical < CAObject
   # @overload intersection(other, sort: false)
   #   @return [CArray] object CArray of the labels present in both.
   def intersection (other, sort: false)
+    refuse_range_set(other, "intersection")
     unique.intersection(label_array(other), sort: sort)
   end
 
   # @overload difference(other, sort: false)
   #   @return [CArray] object CArray of the labels only `self` has.
   def difference (other, sort: false)
+    refuse_range_set(other, "difference")
     unique.difference(label_array(other), sort: sort)
   end
 
   # @overload union(other, sort: false)
   #   @return [CArray] object CArray of the labels of either side.
   def union (other, sort: false)
+    refuse_range_set(other, "union")
     unique.union(label_array(other), sort: sort)
   end
 
@@ -484,6 +522,24 @@ class CACategorical < CAObject
   # that has none. Every code-to-label path goes through here rather than
   # indexing @labels itself, because two decodes are two chances to disagree,
   # and disagreeing is how #unique came to contradict #to_a on the same cell.
+  # An operand of #eq / #ne as codes of this vocabulary. A value that is not
+  # one of the labels gets labels.size, which no cell holds. A CArray is read
+  # cell by cell, a masked cell staying masked; a categorical goes through its
+  # own codes, so the two vocabularies need not agree. Anything else is one
+  # label, as CArray#eq takes it.
+  def codes_of (other)
+    return @labels.index(other) || @labels.size unless other.is_a?(CArray)
+    if other.is_a?(CACategorical)
+      table = other.labels.map { |l| @labels.index(l) || @labels.size }
+      return CA_INT64(table)[other.codes] unless table.empty?
+    end
+    memo = {}
+    list = other.flatten.to_a.map { |v|
+      UNDEF.equal?(v) ? v : memo.fetch(v) { memo[v] = @labels.index(v) || @labels.size }
+    }
+    CA_INT64(list).reshape(*other.shape)
+  end
+
   # The code a fill value stands for: UNDEF stays UNDEF, a label is its code.
   def code_for_fill (value)
     return UNDEF if UNDEF.equal?(value)
@@ -564,35 +620,17 @@ class CACategorical < CAObject
   # `value.reshape(elements)[cat.sort_addr]` produces the contiguous blocks.
   # Memoised (see the grouping-plan note above): the counting sort is the
   # dominant grouping cost, so it is computed once and shared by every payload
-  # column and by CACategoricalIterator. Do not mutate the returned array.
+  # column and by CACategoricalIterator, and kept for the categorical's
+  # lifetime (an int64 per cell). Do not mutate the returned array.
   # @overload sort_addr
   #   Returns a length-{#elements} integer CArray of flat storage
   #   addresses that order the cells by category (excluded cells last).
   #   @return [CArray]
   def sort_addr
     return @sort_addr if @sort_addr
-    n      = elements
-    k      = @labels.size
-    nvalid = category_sizes.sum
-    cur    = reduceat_index.copy                  # segment starts, consumed as cursor
-    flat   = codes.reshape(n)
-    seq    = CArray.int64(n).seq!                 # source indices, scattered as payload
-    out    = CArray.int64(n)
-    if nvalid == n
-      flat.send(:__categorical_scatter__, seq, cur, out, k) # all valid: scatter straight in
-    else
-      if nvalid > 0
-        valid = CArray.int64(nvalid)              # scatter target must be an entity, not a view
-        flat.send(:__categorical_scatter__, seq, cur, valid, k)
-        out[0...nvalid] = valid
-      end
-      # Excluded = every cell the scatter skips: code out of [0, k), OR masked
-      # (a from_codes pre-masked cell keeps a valid code but is excluded). Both
-      # must be caught or the tail slot count would not add up to n - nvalid.
-      excluded = flat.value.ge(k).or(flat.value.lt(0))
-      excluded = excluded.or(flat.is_masked) if flat.has_mask?
-      out[nvalid..-1] = seq[excluded]             # excluded cells, source order
-    end
+    out = codes.reshape(elements).__send__(:addresses_by_category,
+                                           reduceat_index.copy, @labels.size,
+                                           category_sizes.sum)
     @sort_addr = out
     out
   end
@@ -630,6 +668,75 @@ class CACategorical < CAObject
     CACategorical.from_codes(recoded.codes, recoded.labels.map { |c| label_at(c) })
   end
 
+  # The position of +label+ among the labels, found by ==, a NaN label by a
+  # NaN, as join and at match NaN keys; nil when it is not one of them.
+  private def label_position (label)
+    return @labels.index(label) unless label.is_a?(Float) && label.nan?
+    @labels.index { |l| l.is_a?(Float) && l.nan? }
+  end
+
+  # The categorical as numeric columns, one per category, for a model that
+  # takes numbers: row i holds 1 in the column of its category and 0
+  # elsewhere.  With an intercept in the model the k columns would sum to the
+  # intercept's column, so by default the column of the base category is
+  # left out and k - 1 remain; each coefficient then reads as that category
+  # minus the base.  +full: true+ keeps all k (a model without an intercept,
+  # or an input to a learner).
+  #
+  # The columns follow the labels, not the data: a category with no cells
+  # gives a column of zeros, so two categoricals with the same labels give
+  # the same columns.  A masked cell gives a masked row.
+  #
+  # @overload contrast(scheme = :dummy, base: nil, full: false)
+  #   Returns an int8 CArray of shape (n, k - 1), or (n, k) with
+  #   `full: true`, whose columns are the labels in order with the base
+  #   left out.
+  #   @param scheme [Symbol] the coding; only `:dummy` is defined.
+  #   @param base [Object] the label whose column is left out; the first
+  #     label when omitted.
+  #   @param full [Boolean] keep every column.
+  #   @return [CArray]
+  #   @raise [ArgumentError] when `scheme` is not `:dummy`, when `base` is
+  #     not a label, when `base:` and `full: true` are both given, when
+  #     there are no labels to leave one out of, or when the categorical is
+  #     not one-dimensional.
+  def contrast (scheme = :dummy, base: nil, full: false)
+    unless scheme == :dummy
+      raise ArgumentError, "contrast: unknown scheme #{scheme.inspect} (only :dummy is defined)"
+    end
+    unless ndim == 1
+      raise ArgumentError, "contrast: the categorical is #{ndim}-dimensional, not one-dimensional"
+    end
+    k = @labels.size
+    if full
+      raise ArgumentError, "contrast: base: and full: true contradict (full keeps every column)" unless base.nil?
+      kept = (0...k).to_a
+    else
+      raise ArgumentError, "contrast: there are no labels, so no base column to leave out" if k == 0
+      dropped = base.nil? ? 0 : label_position(base)
+      raise ArgumentError, "contrast: #{base.inspect} is not one of the labels #{@labels.inspect}" unless dropped
+      kept = (0...k).to_a - [dropped]
+    end
+    n   = elements
+    m   = kept.size
+    out = CArray.int8(n, m).zero
+    return out if n == 0 || m == 0
+
+    # One pass: each cell's code picks its column through colmap (-1 for the
+    # base), and a 1 is written at row * m + column.
+    colmap = CArray.int64(k) { -1 }
+    kept.each_with_index { |code, j| colmap[code] = j }
+    excluded = codes.has_mask? ? codes.is_masked : CArray.boolean(n) { false }
+    code = codes.value.to_type(CA_INT64)
+    code[excluded] = 0 if excluded.any
+    col  = colmap[code]
+    hit  = col.ge(0) & excluded.not
+    addr = CArray.int64(n).seq! * m + col
+    out.reshape(n * m)[addr[hit]] = 1 if hit.any
+    out[excluded, nil] = UNDEF if excluded.any
+    out
+  end
+
   # @overload inspect
   #   Returns a short summary showing element count, category count,
   #   and vocabulary.
@@ -667,14 +774,11 @@ class CArray
   #     duplicates.
   #   @raise [CArray::DataTypeError] when `sort_labels: true` discovers
   #     the labels of a complex array, which have no order.
-  # The order sort_labels: puts two labels in: as sort orders the cells of an
-  # array, a Float NaN after every other value; two labels <=> cannot order
-  # are refused, by name.
+  # The order sort_labels: puts two labels in: the order sort puts the cells
+  # of an array in (object_order_cmp); two labels <=> cannot order are
+  # refused, by name.
   private def label_order(a, b)
-    na = a.is_a?(Float) && a.nan?
-    nb = b.is_a?(Float) && b.nan?
-    return (na ? 1 : 0) - (nb ? 1 : 0) if na || nb
-    c = a <=> b
+    c = CArray.send(:object_order_cmp, a, b)
     if c.nil?
       raise ArgumentError,
             "categorize(sort_labels: true): cannot order the labels " \
@@ -706,7 +810,13 @@ class CArray
       # Sort the k levels as Ruby sorts them, then renumber every code through
       # the rank in one gather. The sentinel of an excluded cell is out of the
       # rank's range, so project makes it UNDEF and from_codes writes it back.
-      order = (0...levels.size).sort { |i, j| label_order(levels[i], levels[j]) }
+      # A boolean label reads as true or false, and sorts as its storage,
+      # false before true, as sort and unique(sort: true) put the cells.
+      order = if boolean?
+                (0...levels.size).sort_by { |i| levels[i] ? 1 : 0 }
+              else
+                (0...levels.size).sort { |i, j| label_order(levels[i], levels[j]) }
+              end
       rank = CArray.new(codes.data_type, [levels.size])
       order.each_with_index { |old, new_code| rank[old] = new_code }
       return CACategorical.from_codes(rank.project(codes), order.map { |i| levels[i] })
@@ -744,3 +854,15 @@ class CArray
     CACategorical.from_codes(codes, labels_arr)
   end
 end
+
+# categorize is a conversion: the categories are the values in another
+# representation, so they show the attributes the values show.  Each of the
+# three implementations returns from several places, so the attributes are
+# carried here, around all of them.
+[CArray, CACategorical, CAConstString].each do |klass|
+  klass.prepend(Module.new do
+    def categorize (labels: nil, sort_labels: false)
+      carry_attrs(super)
+    end
+  end)
+end