carray 3.0.1 → 3.0.3

data/lib/carray/boolean_reduce.rb

@@ -69,41 +69,33 @@ class CArray
   # @return [Boolean, CArray] a scalar, or an array when an axis is given.
   def none (skip_masked: true, **opts)
     return __none_skipna__(**opts) if skip_masked
-    # none = not any (Kleene): not(true)=false, not(false)=true, not(UNDEF)=UNDEF
-    r = __kleene_fold(:any, opts)
-    r.is_a?(CArray) ? r.not : (r.equal?(UNDEF) ? UNDEF : !r)
+    __kleene_fold(:none, opts)
   end
 
   private
 
-  # Three-valued fold via counts.  `kind` is :any (OR) or :all (AND).
-  # `opts` carries axis: / keep_axis: (same surface as the skipna reduction).
+  # Three-valued fold.  `kind` is :any, :all or :none.  The available-case
+  # reduction answers wherever an unmasked cell settles the fold (a true for
+  # any, a false for all, a true for none); elsewhere a masked cell leaves the
+  # answer undetermined.  Going through that reduction checks the data type,
+  # the axes and the keywords exactly as `skip_masked: true` does.
   def __kleene_fold (kind, opts)
-    axis = opts[:axis]
-    if axis.nil?
-      if kind == :any
-        return true  if count(true)  > 0     # a known TRUE settles OR
-      else
-        return false if count(false) > 0     # a known FALSE settles AND
-      end
-      return UNDEF if count_masked > 0        # undetermined
-      return( kind == :any ? false : true )
+    r = case kind
+        when :any then __any_skipna__(**opts)
+        when :all then __all_skipna__(**opts)
+        else           __none_skipna__(**opts)
+        end
+    cm = count_masked(**opts)
+    if r.is_a?(CArray)
+      settled = ( kind == :any ) ? r : r.not
+      out = r.copy
+      out[settled.not & (cm > 0)] = UNDEF
+      out
+    else
+      settled = ( kind == :any ) ? r : !r
+      return r if settled
+      ( cm > 0 ) ? UNDEF : r
     end
-
-    keep = opts[:keep_axis]
-    cnt_kw = keep ? { axis: axis, keep_axis: keep } : { axis: axis }
-    ck = ( kind == :any ) ? count(true, **cnt_kw) : count(false, **cnt_kw)
-    cm = count_masked(axis: axis)               # count_masked has no keep_axis
-    cm = cm.reshape(*ck.shape) unless cm.shape == ck.shape
-
-    # A cell is undetermined only where it is masked AND the known side does
-    # not settle it (ck counts the settling value: true for OR, false for AND).
-    # Masking only those cells avoids leaving an all-zero mask behind.
-    out = CArray.boolean(*ck.shape)
-    out[] = ( kind == :any ) ? 0 : 1            # OR default false / AND default true
-    out[ck.eq(0) & (cm > 0)] = UNDEF             # genuinely undetermined
-    out[ck > 0] = ( kind == :any ) ? 1 : 0       # known side dominates
-    out
   end
 
 end

data/lib/carray/categorical.rb

@@ -17,8 +17,12 @@
 #                         (per-category counts), `cat.codes.count(code)`, etc.
 #    - per-cell access  = decode the code into its label (`cat[i]` -> category)
 #
-#  Exclusion (missing / out-of-vocabulary) is encoded two ways at once, and
-#  because the Face is READONLY they can never desync:
+#  Exclusion (missing / out-of-vocabulary) is encoded two ways at once. Both
+#  are written when the categorical is built — categorize produces them, and
+#  from_codes normalises whatever it receives into them — and the Face is
+#  READONLY afterwards, so they stay in step. Both are needed because they have
+#  different readers: the axis-group kernel classifies on the code byte, the
+#  materialising paths classify on the mask.
 #
 #    - the cell is MASKED                  -> CArray-native idiom: `cat[i]` is
 #                                             UNDEF, `is_masked` / mask-aware
@@ -76,46 +80,148 @@ class CACategorical < CAObject
     CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => -1,
   }.freeze
 
+  # The largest vocabulary each codes data type can carry: every valid code in
+  # 0...k has to be representable AND distinct from the exclusion sentinel.  An
+  # unsigned type spends its top value on the sentinel; a signed one spends -1,
+  # which was never a valid index, so a signed type carries one more label than
+  # the unsigned type of the same width.  categorize picks a width by this rule
+  # when it builds codes; from_codes has to check it when it receives them.
+  MAX_LABELS = {
+    CA_UINT8  => 0xFF,               CA_INT8  => 0x80,
+    CA_UINT16 => 0xFFFF,             CA_INT16 => 0x8000,
+    CA_UINT32 => 0xFFFFFFFF,         CA_INT32 => 0x80000000,
+    CA_UINT64 => 0xFFFFFFFFFFFFFFFF, CA_INT64 => 0x8000000000000000,
+  }.freeze
+
   class << self
     # Wrap already-dense codes + labels with no discovery — the import receiver
-    # for a pandas Categorical or an Arrow dictionary. `codes` becomes the
-    # Face's storage parent verbatim (zero-copy when it is a wrapped memory
-    # view), and from_codes takes ownership of it.
+    # for a pandas Categorical or an Arrow dictionary. from_codes takes
+    # ownership of `codes`, which becomes the Face's storage parent: verbatim
+    # when it is an entity that needs no rewriting, so a wrapped memory view
+    # stays a view, and materialised when it is a view over an array the caller
+    # still holds, since a view owns no bytes to take ownership of.
+    #
+    # This is the one door through which an already-built encoding enters, so
+    # it validates rather than assumes, and it normalises before handing over:
+    # a cell is excluded when it arrives masked OR holds the all-ones sentinel
+    # (type-max for unsigned codes, -1 for signed — both the pandas / Arrow
+    # missing code), and every excluded cell leaves here holding the sentinel
+    # AND masked. Writing both matters because the two encodings have different
+    # readers: the axis-group kernel classifies on the code byte, the
+    # materialising paths classify on the mask. A categorical that carries only
+    # one of them answers membership two ways, silently. An Arrow dictionary
+    # carries its missingness in a validity bitmap with arbitrary code bytes —
+    # frequently 0 — so arriving masked-only is the normal import, not an edge.
     #
-    # Excluded cells are identified by the all-ones sentinel value (type-max
-    # for unsigned codes, -1 for signed — both the pandas / Arrow missing code)
-    # and masked here, so the categorical is well-formed regardless of whether
-    # the caller pre-masked. Only the mask buffer is touched; the code bytes
-    # are left intact (so a pandas byte-reinterpret round-trips).
+    # #initialize marks the codes read-only, so this is also the last point at
+    # which they can be written; a read-only argument is copied rather than
+    # refused.
     # @overload from_codes(codes, labels)
     #   Returns a {CACategorical} wrapping already-dense integer
     #   `codes` with the given `labels`, without discovery. `codes`
-    #   becomes the Face's storage parent; excluded cells (identified
-    #   by the type-max sentinel value) are masked automatically.
+    #   becomes the Face's storage parent, materialised first if it is
+    #   a view. A cell that is masked or holds the type-max sentinel is
+    #   excluded, and leaves as both.
     #   @param codes [CArray] integer code storage.
     #   @param labels [Array, CArray] category vocabulary indexed by
-    #     code.
+    #     code. Must be unique, and must fit the codes data type with
+    #     the sentinel reserved.
     #   @return [CACategorical]
-    #   @raise [ArgumentError] when `codes` is not an integer CArray.
+    #   @raise [ArgumentError] when `codes` is not an integer CArray,
+    #     when `labels` holds duplicates or is too large for the codes
+    #     data type, or when an unmasked code is outside `0...labels.size`
+    #     and is not the sentinel.
     def from_codes(codes, labels)
       unless codes.is_a?(CArray) && SENTINEL.key?(codes.data_type)
         got = codes.is_a?(CArray) ? codes.data_type : codes.class
         raise ArgumentError, "from_codes: codes must be an integer CArray (got #{got})"
       end
-      excluded = codes.eq(SENTINEL[codes.data_type])
-      if excluded.count(true) > 0
-        codes.mask = codes.has_mask? ? (codes.mask | excluded) : excluded
+
+      labels_arr = labels.respond_to?(:to_a) ? labels.to_a : Array(labels)
+      if labels_arr.uniq.size != labels_arr.size
+        raise ArgumentError, "from_codes: labels must be unique (got duplicates)"
+      end
+      k        = labels_arr.size
+      sentinel = SENTINEL[codes.data_type]
+      max      = MAX_LABELS[codes.data_type]
+      if k > max
+        raise ArgumentError,
+              "from_codes: #{k} labels do not fit #{CArray.data_type_name(codes.data_type)} " \
+              "codes, which carry at most #{max} (the top value is reserved as the " \
+              "exclusion sentinel); widen the codes data type"
+      end
+
+      # Classify on the code bytes with the mask set aside (`.value`): a
+      # comparison against a masked cell yields UNDEF, which would read as
+      # "not excluded" and let the cell through carrying a valid-looking code.
+      raw = codes.value
+      if sentinel == -1                 # signed codes: -1 is the sentinel
+        out = raw.lt(0)
+        # Only test the upper bound when k is representable in the codes data
+        # type. At the very top of the range (k == max) no value can reach k
+        # anyway, and comparing against an unrepresentable literal would wrap
+        # and flag every cell as out of range.
+        out = out | raw.ge(k) if k <= max - 1
+      else                              # unsigned codes: type-max is the sentinel
+        out = raw.ge(k)                 # k <= max here, so always representable
       end
-      new(codes, labels)
+      masked   = codes.has_mask? ? codes.is_masked : nil
+      excluded = masked ? (out | masked) : out
+      # A masked cell may hold any byte at all — that is the CArray contract —
+      # so it is never corrupt, only in need of normalising. An *unmasked* cell
+      # holding an out-of-range code that is not the sentinel is neither a
+      # category nor missingness; refuse it here, where the input is still in
+      # the caller's hands, instead of letting it surface later as an IndexError
+      # from the grouping plan or as a wrong label from a decode.
+      corrupt = out & raw.ne(sentinel)
+      corrupt = corrupt & masked.not if masked
+      if corrupt.any
+        bad = raw[corrupt].to_a.uniq.sort
+        shown = bad.first(4).join(", ") + (bad.size > 4 ? ", ..." : "")
+        raise ArgumentError,
+              "from_codes: code#{bad.size == 1 ? "" : "s"} #{shown} outside " \
+              "0...#{k} for #{k} label#{k == 1 ? "" : "s"} " \
+              "(use #{sentinel} to exclude a cell, or mask it)"
+      end
+
+      # Take real ownership, then normalise so the byte reader and the mask
+      # reader agree from here on.
+      #
+      # A view owns no bytes — its root does, and the caller still holds that
+      # root. #initialize marks what it is given read-only, which would stop
+      # `cat.codes[i] = x` but not `root[i] = x`, so the codes could still
+      # change underneath a Face that is supposed to be immutable (and
+      # underneath the grouping plan memoised against them). Marking the root
+      # instead is worse: it would freeze bytes outside the window the caller
+      # handed over. So a view is materialised. Read-only codes are copied for
+      # the adjacent reason — the normalising writes need somewhere to land.
+      #
+      # An entity that needs no rewriting is adopted verbatim, which is what
+      # keeps a wrapped memory view a view. That buffer's producer can still
+      # write it; that is the borrowed-buffer bargain, and not something this
+      # constructor can close.
+      needs_mask = masked ? (excluded & masked.not).any : excluded.any
+      needs_byte = (excluded & raw.ne(sentinel)).any
+      if !codes.entity? || ((needs_mask || needs_byte) && codes.read_only?)
+        codes = codes.copy
+      end
+      if needs_mask || needs_byte
+        codes.value[excluded] = sentinel if needs_byte
+        codes.mask = excluded
+      end
+
+      new(codes, labels_arr)
     end
   end
 
-  # codes : integer CArray, the storage parent. Excluded cells are both
-  #         masked AND store the type-max sentinel value (= the all-ones bit
-  #         pattern, which is signed -1 byte-for-byte — the pandas / Arrow
-  #         missing code). Because the Face is READONLY the two never desync,
-  #         so consumers may rely on either: the mask (CArray-native) or the
-  #         sentinel (axis-group's out-of-range skip, zero-copy export).
+  # codes : integer CArray, the storage parent, already normalised by the
+  #         caller (categorize builds it that way; from_codes rewrites what it
+  #         receives). Excluded cells are both masked AND store the type-max
+  #         sentinel value (= the all-ones bit pattern, which is signed -1
+  #         byte-for-byte — the pandas / Arrow missing code). Marking the codes
+  #         read-only below keeps the two in step from here on, so consumers may
+  #         rely on either: the mask (CArray-native) or the sentinel
+  #         (axis-group's out-of-range skip, zero-copy export).
   # labels: Array | CArray, the vocabulary; labels[code] = category.
   # @overload initialize(codes, labels)
   #   Allocates a READONLY {CACategorical} Face whose storage is
@@ -143,8 +249,9 @@ class CACategorical < CAObject
     # through views/Faces (a reshape of frozen codes is frozen) and would block
     # the grouping cache from memoising. The flag gives the same write protection
     # (mutations raise) while keeping the object non-frozen. One-way: it takes
-    # ownership of `codes` (categorize / from_codes build or receive it, mask
-    # already derived above); a caller keeping a mutable array must pass `.copy`.
+    # ownership of `codes`, which from_codes has already validated, materialised
+    # if it was a view, and normalised; a caller keeping a mutable entity of its
+    # own must pass `.copy`.
     codes.set_read_only_flag
   end
 
@@ -188,12 +295,11 @@ class CACategorical < CAObject
     other.is_a?(CACategorical) && @labels == other.labels
   end
 
-  # Face hook: decode a per-cell code into its category label. An out-of-range
-  # code (e.g. an unmasked external sentinel) decodes to nil rather than a
-  # wrong category via Ruby negative indexing.
-  def storage_to_scalar(raw)
+  # Face hook: decode a per-cell code into its category label, through the one
+  # decode every code-to-label path shares (see #label_at).
+  def storage_to_element(raw)
     code = raw.is_a?(String) ? raw.unpack1(UNPACK_FORMAT.fetch(parent.data_type)) : raw
-    (code < 0 || code >= @labels.size) ? nil : @labels[code]
+    label_at(code)
   end
 
   # ---- category-space operations (by label, not code) -------------------
@@ -211,6 +317,27 @@ class CACategorical < CAObject
   #   an all-false result.
   #   @param label [Object] category to match.
   #   @return [CArray]
+  # @overload resize(*newdim, fill_value: UNDEF)
+  #   Returns a categorical resized to `newdim` (see {CArray#resize}).
+  #   The new cells are excluded (UNDEF) unless `fill_value` names one of
+  #   the labels, since a zero code would read as the first label.
+  #   @raise [ArgumentError] when `fill_value` is not a label.
+  def resize (*newdim, fill_value: UNDEF)
+    CACategorical.from_codes(codes.resize(*newdim, fill_value: code_for_fill(fill_value)),
+                             labels)
+  end
+
+  # @overload insert_block(offset, bsize, &block)
+  #   Returns a categorical with a block inserted (see
+  #   {CArray#insert_block}). The inserted cells are excluded (UNDEF)
+  #   unless the block returns one of the labels.
+  #   @raise [ArgumentError] when the block returns something that is not
+  #     a label.
+  def insert_block (offset, bsize, &block)
+    fill = block ? code_for_fill(block.call) : UNDEF
+    CACategorical.from_codes(codes.insert_block(offset, bsize) { fill }, labels)
+  end
+
   def eq(label)
     codes.eq(@labels.index(label) || @labels.size)
   end
@@ -250,11 +377,11 @@ class CACategorical < CAObject
   #   so `labels.zip(category_sizes.to_a)` always pairs up.
   #   @return [CArray]
   def category_sizes
-    return @_category_sizes if @_category_sizes
+    return @category_sizes if @category_sizes
     bc  = codes.bincount
-    out = CArray.new(bc.data_type, [@labels.size])   # new zero-fills
+    out = CArray.new(bc.data_type, [@labels.size], bytes: bc.bytes)   # new zero-fills
     out[0...bc.elements] = bc if bc.elements > 0
-    @_category_sizes = out
+    @category_sizes = out
     out
   end
 
@@ -350,10 +477,29 @@ class CACategorical < CAObject
 
   private
 
-  # Labels for a code array (the discovery kernels skip masked cells, so the
-  # code arrays reaching here hold real codes only).
+  # The one decode from a code to its label. An out-of-range code — a negative
+  # one, or an unmasked sentinel that reached us from outside — has no label,
+  # so it decodes to nil. Writing `@labels[code]` instead would read a negative
+  # code from the *end* of the vocabulary and hand back a real label for a cell
+  # that has none. Every code-to-label path goes through here rather than
+  # indexing @labels itself, because two decodes are two chances to disagree,
+  # and disagreeing is how #unique came to contradict #to_a on the same cell.
+  # The code a fill value stands for: UNDEF stays UNDEF, a label is its code.
+  def code_for_fill (value)
+    return UNDEF if UNDEF.equal?(value)
+    code = labels.index(value)
+    raise ArgumentError, "#{value.inspect} is not one of the labels #{labels.inspect}" unless code
+    code
+  end
+
+  def label_at (code)
+    (code < 0 || code >= @labels.size) ? nil : @labels[code]
+  end
+
+  # Labels for a code array. The discovery kernels skip masked cells, so what
+  # reaches here is real codes — but it is decoded like any other code.
   def labels_for (code_array)
-    CA_OBJECT(code_array.to_a.map { |c| @labels[c] })
+    CA_OBJECT(code_array.to_a.map { |c| label_at(c) })
   end
 
   # This categorical's cells as their labels; a masked cell stays masked.
@@ -424,13 +570,11 @@ class CACategorical < CAObject
   #   addresses that order the cells by category (excluded cells last).
   #   @return [CArray]
   def sort_addr
-    return @_sort_addr if @_sort_addr
+    return @sort_addr if @sort_addr
     n      = elements
     k      = @labels.size
-    cs     = category_sizes                       # per-category counts
-    nvalid = cs.sum
-    cur    = CArray.int64(k > 0 ? k : 0)          # segment starts, consumed as cursor
-    cur[1..-1] = cs.cumsum.int64[0..-2] if k > 1
+    nvalid = category_sizes.sum
+    cur    = reduceat_index.copy                  # segment starts, consumed as cursor
     flat   = codes.reshape(n)
     seq    = CArray.int64(n).seq!                 # source indices, scattered as payload
     out    = CArray.int64(n)
@@ -449,7 +593,7 @@ class CACategorical < CAObject
       excluded = excluded.or(flat.is_masked) if flat.has_mask?
       out[nvalid..-1] = seq[excluded]             # excluded cells, source order
     end
-    @_sort_addr = out
+    @sort_addr = out
     out
   end
 
@@ -464,17 +608,26 @@ class CACategorical < CAObject
   #   offsets aligned to {#labels}. Pair with {#category_sizes} for lengths.
   #   @return [CArray]
   def reduceat_index
-    return @_reduceat_index if @_reduceat_index
-    k = @labels.size
-    out = CArray.int64(k > 0 ? k : 0)
-    if k > 1
-      out[0] = 0
-      out[1..-1] = category_sizes.cumsum.int64[0..-2]
-    elsif k == 1
-      out[0] = 0
+    return @reduceat_index if @reduceat_index
+    # Every boundary but the last; the copy makes it an entity the
+    # reduceat kernels can read directly.
+    @reduceat_index = CArray.segment_offsets(lengths: category_sizes)[0...-1].copy
+  end
+
+  # @overload categorize(labels: nil, sort_labels: false)
+  #   Returns a {CACategorical} keyed on the labels, as categorizing any
+  #   key array does: the labels that occur, in first-appearance order (or
+  #   sorted with `sort_labels:`), or the fixed `labels:` vocabulary.  A
+  #   category of `self` with no cells does not carry over; a masked cell
+  #   is excluded.
+  #   @return [CACategorical]
+  def categorize (labels: nil, sort_labels: false)
+    if labels || sort_labels
+      return label_values.categorize(labels: labels, sort_labels: sort_labels)
     end
-    @_reduceat_index = out
-    out
+    # Distinctness rides the codes; only the k labels cross the surface.
+    recoded = codes.categorize
+    CACategorical.from_codes(recoded.codes, recoded.labels.map { |c| label_at(c) })
   end
 
   # @overload inspect
@@ -512,6 +665,24 @@ class CArray
   #   @return [CACategorical]
   #   @raise [ArgumentError] when explicit `labels` contain
   #     duplicates.
+  #   @raise [CArray::DataTypeError] when `sort_labels: true` discovers
+  #     the labels of a complex array, which have no order.
+  # The order sort_labels: puts two labels in: as sort orders the cells of an
+  # array, a Float NaN after every other value; two labels <=> cannot order
+  # are refused, by name.
+  private def label_order(a, b)
+    na = a.is_a?(Float) && a.nan?
+    nb = b.is_a?(Float) && b.nan?
+    return (na ? 1 : 0) - (nb ? 1 : 0) if na || nb
+    c = a <=> b
+    if c.nil?
+      raise ArgumentError,
+            "categorize(sort_labels: true): cannot order the labels " \
+            "#{a.inspect} and #{b.inspect}"
+    end
+    c
+  end
+
   def categorize(labels: nil, sort_labels: false)
     # Automatic appearance-order vocabulary: one linear pass (C
     # __factorize_appearance__) returns both codes and levels directly, over the
@@ -519,12 +690,26 @@ class CArray
     # lane). Distinctness is the hash-key judgement shared with the discovery
     # family: Float NaN collapses to one category and -0.0 == +0.0, while mixed
     # Integer / Float keys stay distinct (eql?, so 1 and 1.0 are separate
-    # categories). The discovery path below is reserved for sort_labels (which
-    # reorders the vocabulary, desyncing the appearance-order codes), an explicit
-    # labels list, and the data types the factorize kernel does not take (complex).
-    if labels.nil? && !sort_labels && (integer? || float? || object? || fixlen? || boolean?)
+    # categories). Complex values are distinct by value with the same NaN and
+    # -0.0 rules, and have no order, so sort_labels refuses them. The discovery
+    # path below is reserved for an explicit labels list.
+    if labels.nil? && sort_labels && complex?
+      raise CArray::DataTypeError,
+            "categorize(sort_labels: true): a #{data_type_name} value has no " \
+            "order (complex numbers are not ordered); without sort_labels: the " \
+            "labels are in first-appearance order"
+    end
+    if labels.nil? && (integer? || float? || complex? || object? || fixlen? || boolean?)
       codes, levels = __factorize_appearance__
-      return CACategorical.from_codes(codes, levels.to_a)
+      levels = levels.to_a
+      return CACategorical.from_codes(codes, levels) unless sort_labels
+      # Sort the k levels as Ruby sorts them, then renumber every code through
+      # the rank in one gather. The sentinel of an excluded cell is out of the
+      # rank's range, so project makes it UNDEF and from_codes writes it back.
+      order = (0...levels.size).sort { |i, j| label_order(levels[i], levels[j]) }
+      rank = CArray.new(codes.data_type, [levels.size])
+      order.each_with_index { |old, new_code| rank[old] = new_code }
+      return CACategorical.from_codes(rank.project(codes), order.map { |i| levels[i] })
     end
 
     if labels.nil?
@@ -533,7 +718,7 @@ class CArray
       # masked keys stay excluded), so the non-masked cells are the levels.
       # Only the final list is Ruby, since labels are Ruby objects.
       labels_arr = mask_duplicates[:is_not_masked].to_a
-      labels_arr.sort! if sort_labels
+      labels_arr.sort! { |a, b| label_order(a, b) } if sort_labels
     else
       labels_arr = labels.respond_to?(:to_a) ? labels.to_a : Array(labels)
       if labels_arr.uniq.size != labels_arr.size