carray 3.0.1 → 3.0.2

data/lib/carray/const_string.rb

@@ -5,12 +5,15 @@
 #  CAConstString high-level construction + conversion surface.
 #
 #  CAConstString itself (the Face + tail buffer + fetch decode + numeric gate) lives
-#  in ext/ca_obj_const_string.c.  This file provides the ergonomic builders that pack
-#  Ruby Strings into the internal length-prefix buffer + int64 offset entity
-#  and wrap them via the CAConstString.wrap C primitive.
+#  in ext/ca_obj_const_string.c.  This file provides the ergonomic builders that
+#  pack Ruby Strings into the shared buffer + (start,end) pair entity, via the
+#  CAConstString.__build__ C primitive.
 #
-#  Internal buffer format: concatenation of `[int32 length][bytes...]` records
-#  (self-delimiting → one offset per element, permutation-safe view).
+#  Internal buffer format: a pure concatenation of the element bytes, with no
+#  per-record length prefix (= the Arrow values buffer).  Each element carries
+#  its own `(start, end)` byte range in the storage, so it is self-describing
+#  and a gather / sort / select view over the pairs decodes correctly with no
+#  buffer copy.
 #
 # ----------------------------------------------------------------------------
 
@@ -89,21 +92,118 @@ class CAConstString
   # override the L1 generics where present.
   include CArray::StringOperationMixin
 
-  # @overload sort
-  #   Returns a byte-order sorted view built by gathering the
-  #   offsets with `sort_index`. Neither bytes nor offsets are
-  #   copied.
+  # ---- ordering family ---------------------------------------------------
+  #
+  # A storage cell is a `(start, end)` byte range into the shared buffer, so
+  # storage order is the order the strings were packed in, not the order they
+  # compare in.  Without these overrides the family sorted the offsets: the
+  # answers came back well-formed and wrong -- `%w[pear apple].sort_addr` gave
+  # the identity, and `partition_copy` gave NUL bytes.
+  #
+  # Every member therefore reads the bytes.  The flat forms do it natively
+  # (a byte-memcmp scan over the packed buffer, an order of magnitude faster
+  # than decoding a Ruby String per cell), the per-axis forms through
+  # {#to_string}, where a cell is the string and CArray's own kernels apply.
+  # Arguments are forwarded verbatim, so `kind:` / `masked_position:` /
+  # `keep_axis:` mean here what they mean on CArray.
+  #
+  # Anything whose cells are strings comes back a CAConstString, gathered
+  # over the same buffer where it can be (sort / sort_copy) and rebuilt where
+  # it cannot.  Indices, addresses and counts need no conversion.
+
+  # @overload sort(axis: nil, kind: :quick, masked_position: :last)
+  #   Returns a sorted {CAConstString} view, gathered over the same buffer
+  #   and offsets.  With no `axis:` the array is flattened first, as
+  #   `CArray#sort` does.
   #   @return [CAConstString]
-  def sort
-    self[sort_index]
+  def sort (*args, **kw)
+    addr = sort_addr(*args, **kw)
+    kw[:axis].nil? ? self[addr.flatten] : self[addr]
   end
 
-  # @overload sort_copy
-  #   Returns an owned sorted {CAConstString}; the materialised
-  #   counterpart to {#sort}.
+  # @overload sort_copy(axis: nil, kind: :quick, masked_position: :last)
+  #   Returns an owned sorted {CAConstString}; the materialised counterpart
+  #   to {#sort}.
   #   @return [CAConstString]
-  def sort_copy
-    self[sort_index].copy
+  def sort_copy (*args, **kw)
+    sort(*args, **kw).copy
+  end
+
+  # @overload sort_addr(axis: nil, kind: :quick, masked_position: :last)
+  #   Returns the view-flat addresses that index a sort by string order.
+  #   @return [CArray] `:int64` addresses.
+  def sort_addr (*args, **kw)
+    # The native scan has no notion of an incomparable sentinel, so a masked
+    # column goes the same way a per-axis one does.
+    return __sort_addr_bytes__ if args.empty? && kw.empty? && ! has_mask?
+    to_string.sort_addr(*args, **kw)
+  end
+
+  # @overload sort_index(axis: nil, kind: :quick, masked_position: :last)
+  #   Returns the per-fiber indices that index a sort by string order.
+  #   @return [CArray] `:int64` indices.
+  def sort_index (*args, **kw)
+    to_string.sort_index(*args, **kw)
+  end
+
+  # @overload rank_index(axis: nil)
+  #   @return [CArray] each cell's rank in string order.
+  def rank_index (*args, **kw)
+    to_string.rank_index(*args, **kw)
+  end
+
+  # @overload order(*args)
+  #   @return [CArray] the ordering of the cells by string order.
+  def order (*args, **kw)
+    to_string.order(*args, **kw)
+  end
+
+  # @overload min(axis: nil, keep_axis: false)
+  #   Returns the byte-smallest string, skipping masked cells; UNDEF when
+  #   every cell is masked or the array is empty.
+  #   @return [String, CAConstString]
+  def min (*args, **kw)
+    return __min_bytes__ || UNDEF if args.empty? && kw.empty?
+    const_string_lift(to_string.min(*args, **kw))
+  end
+
+  # @overload max(axis: nil, keep_axis: false)
+  #   Byte-largest counterpart of {#min}.
+  #   @return [String, CAConstString]
+  def max (*args, **kw)
+    return __max_bytes__ || UNDEF if args.empty? && kw.empty?
+    const_string_lift(to_string.max(*args, **kw))
+  end
+
+  # @overload minmax(axis: nil, keep_axis: false)
+  #   @return [Array] `[min, max]`.
+  def minmax (*args, **kw)
+    return [min, max] if args.empty? && kw.empty?
+    to_string.minmax(*args, **kw).map { |r| const_string_lift(r) }
+  end
+
+  # @overload min_index(axis: nil)
+  #   @return [Integer, CArray] where the byte-smallest string sits.
+  def min_index (*args, **kw)
+    to_string.min_index(*args, **kw)
+  end
+
+  # @overload max_index(axis: nil)
+  #   @return [Integer, CArray] where the byte-largest string sits.
+  def max_index (*args, **kw)
+    to_string.max_index(*args, **kw)
+  end
+
+  # @overload partition_copy(kth, axis: nil)
+  #   @return [CAConstString] partitioned about the `kth` string in order.
+  def partition_copy (*args, **kw)
+    const_string_lift(to_string.partition_copy(*args, **kw))
+  end
+
+  # @overload partition_index(kth, axis: nil)
+  #   @return [CArray] the indices that partition about the `kth` string.
+  def partition_index (*args, **kw)
+    to_string.partition_index(*args, **kw)
   end
 
   # CAConstString is the one Face with three separate entities -- storage
@@ -144,14 +244,14 @@ class CAConstString
   # @overload unique(sort: false)
   #   @return [CAConstString] the distinct strings.
   def unique (sort: false)
-    to_string.unique(sort: sort).to_const_string
+    const_string_lift(to_string.unique(sort: sort))
   end
 
   # @overload value_counts(sort: false)
   #   @return [Array(CAConstString, CArray)] `[values, counts]`.
   def value_counts (sort: false)
     values, counts = to_string.value_counts(sort: sort)
-    [values.to_const_string, counts]
+    [const_string_lift(values), counts]
   end
 
   # @overload nunique(axis: nil, keep_axis: false)
@@ -164,11 +264,7 @@ class CAConstString
   #   @return [CAConstString, Array] the most frequent string(s).
   def mode (axis: nil)
     r = to_string.mode(axis: axis)
-    case r
-    when CArray then r.to_const_string
-    when Array  then r.map { |c| c.is_a?(CArray) ? c.to_const_string : c }
-    else r
-    end
+    r.is_a?(Array) ? r.map { |c| const_string_lift(c) } : const_string_lift(r)
   end
 
   # @overload is_mode(axis: nil)
@@ -180,7 +276,7 @@ class CAConstString
   # @overload mask_duplicates(axis: nil)
   #   @return [CAConstString] a copy with every repeat occurrence masked.
   def mask_duplicates (axis: nil)
-    to_string.mask_duplicates(axis: axis).to_const_string
+    const_string_lift(to_string.mask_duplicates(axis: axis))
   end
 
   # @overload is_in(values)
@@ -192,19 +288,19 @@ class CAConstString
   # @overload intersection(other, sort: false)
   #   @return [CAConstString] the distinct strings present in both.
   def intersection (other, sort: false)
-    to_string.intersection(string_operand(other), sort: sort).to_const_string
+    const_string_lift(to_string.intersection(string_operand(other), sort: sort))
   end
 
   # @overload difference(other, sort: false)
   #   @return [CAConstString] the distinct strings only `self` has.
   def difference (other, sort: false)
-    to_string.difference(string_operand(other), sort: sort).to_const_string
+    const_string_lift(to_string.difference(string_operand(other), sort: sort))
   end
 
   # @overload union(other, sort: false)
   #   @return [CAConstString] the distinct strings of either side.
   def union (other, sort: false)
-    to_string.union(string_operand(other), sort: sort).to_const_string
+    const_string_lift(to_string.union(string_operand(other), sort: sort))
   end
 
   # @overload locate_addr(ref)
@@ -225,4 +321,12 @@ class CAConstString
     other.is_a?(CAConstString) ? other.to_string : other
   end
 
+  # Pack a string-bearing result back into a column of this one's encoding.
+  # The encoding has to be carried: the builder checks each element against
+  # the column's, so a Shift_JIS column rebuilt as the UTF-8 default raised
+  # rather than coming back.
+  private def const_string_lift (x)
+    x.is_a?(CArray) ? CArray.const_string(x, encoding: encoding) : x
+  end
+
 end

data/lib/carray/construct.rb

@@ -256,6 +256,46 @@ class CArray
   TypeSymbol = nil
   # @!visibility private
   DataType   = nil
+
+  # @overload empty(data_type, dim, bytes: nil)
+  #   Returns a new CArray of `data_type` with the shape `dim`, whose
+  #   contents are **undefined**. This is {CArray.new} without the fill:
+  #   the arguments are read the same way, and the cost of zeroing the
+  #   buffer is not paid. Use it when every cell is written before it is
+  #   read; otherwise use `CArray.new`, which zero-fills.
+  #
+  #   `CA_OBJECT` is the exception: its cells are zero-initialised
+  #   anyway, because the garbage collector walks them.
+  #
+  #   @param data_type [Symbol, String, Integer] element type.
+  #   @param dim [Array<Integer>] shape of the new CArray.
+  #   @param bytes [Integer, nil] element width, for `:fixlen` only.
+  #   @return [CArray]
+  #   @raise [ArgumentError] if a block is given -- filling an array is
+  #     what `CArray.new` is for.
+  # @overload empty(*shape)
+  #   (Numo / NumPy compatibility) Returns an uninitialised CArray of
+  #   the given shape. The element type comes from the receiver, so
+  #   `CArray` itself gives `CA_FLOAT64`; see
+  #   {DataTypeExtension#empty}.
+  #   @param shape [Array<Integer>, Array<Array<Integer>>] shape.
+  #   @return [CArray]
+  def self.empty (*args, **options, &block)
+    if block
+      raise ArgumentError,
+            "CArray.empty does not take a block " \
+            "(its contents are left undefined); use CArray.new to fill."
+    end
+    if args.size == 2 and args[1].is_a?(Array)
+      return __empty__(*args, **options)
+    end
+    unless options.empty?
+      raise ArgumentError,
+            "CArray.empty(*shape) takes no options; bytes: belongs to " \
+            "CArray.empty(data_type, dim)"
+    end
+    super(*args)
+  end
 end
 
 class CArray

data/lib/carray/data_type_extension.rb

@@ -237,6 +237,9 @@ class CArray
     #   **uninitialised**. The caller must overwrite the array
     #   before reading from it. `CA_OBJECT` silently falls back to
     #   a zero-VALUE init required for GC safety.
+    #
+    #   carray's own spelling is `CArray.empty(data_type, dim)`, which
+    #   names the element type instead of taking it from the receiver.
     #   @param shape [Array<Integer>, Array<Array<Integer>>]
     #   @return [CArray]
     def empty (*args)

data/lib/carray/data_type_limits.rb

@@ -0,0 +1,91 @@
+#  What each numeric data type can hold, as constants on the typed
+#  classes.
+#
+#      CArray::Int32::MAX        #  =>  2147483647
+#      CArray::Float32::EPSILON  #  =>  1.1920928955078125e-07
+#
+#  The names follow NumPy's iinfo / finfo rather than Ruby's Float, so
+#  that MIN and MAX bracket the range for every numeric type alike:
+#
+#      CArray::Float64::MIN      #  => -1.7976931348623157e+308
+#      Float::MIN                #  =>  2.2250738585072014e-308
+#
+#  Those are not the same number and not the same question.  Ruby's
+#  Float::MIN is the smallest positive normal, which is TINY here; MIN
+#  is the bottom of the range, which for an integer type is the only
+#  thing MIN could mean.  Code that has to bracket a type without
+#  knowing whether it is integer or float reads MIN and MAX and is
+#  right either way, and that is the reason for the choice.
+#
+#  Not every data type has limits.  boolean, fixlen and object have no
+#  numeric range, so they carry none of these constants -- asking gives
+#  a NameError rather than an answer that would have to be invented.
+
+class CArray
+
+  #  The integer widths come from the extension rather than from a table
+  #  written here, so a platform where a type is not the usual width is
+  #  described correctly instead of confidently mis-described.
+
+  {
+    Int8   => true,  Int16  => true,  Int32  => true,  Int64  => true,
+    UInt8  => false, UInt16 => false, UInt32 => false, UInt64 => false,
+  }.each do |klass, signed|
+    bits = 8 * CArray.sizeof(klass::TypeSymbol)
+    if signed
+      klass.const_set(:MIN, -(2 ** (bits - 1)))
+      klass.const_set(:MAX, 2 ** (bits - 1) - 1)
+    else
+      klass.const_set(:MIN, 0)
+      klass.const_set(:MAX, 2 ** bits - 1)
+    end
+  end
+
+  #  float32 and float64 are IEEE-754 binary32 and binary64, and the four
+  #  values below follow from the format: with `p` significand bits (the
+  #  implicit one included) and `emax` the exponent of the largest finite
+  #  value,
+  #
+  #      EPSILON = 2 ** (1 - p)          the step from 1.0 to the next float
+  #      MAX     = (2 - EPSILON) * 2 ** emax
+  #      MIN     = -MAX                  every format here is symmetric
+  #      TINY    = 2 ** (1 - emax)       the smallest positive normal
+  #
+  #  All four are exactly representable as a Ruby Float, binary32
+  #  included, so nothing is rounded on the way in.  The width is
+  #  checked rather than assumed: on a platform where C float or double
+  #  is not one of these formats the arithmetic below would be wrong,
+  #  and a wrong limit is worse than a missing one.
+
+  {
+    Float32 => [4, 24, 127],
+    Float64 => [8, 53, 1023],
+  }.each do |klass, (bytes, precision, max_exponent)|
+    actual = CArray.sizeof(klass::TypeSymbol)
+    unless actual == bytes
+      raise "#{klass} is #{actual} bytes wide, not the #{bytes * 8}-bit IEEE-754 " \
+            "format its limits are derived from"
+    end
+    epsilon = 2.0 ** (1 - precision)
+    max     = (2.0 - epsilon) * 2.0 ** max_exponent
+    klass.const_set(:EPSILON, epsilon)
+    klass.const_set(:MAX,     max)
+    klass.const_set(:MIN,     -max)
+    klass.const_set(:TINY,    2.0 ** (1 - max_exponent))
+  end
+
+  #  A complex type is a pair of floats, so its limits are its
+  #  component's -- MIN and MAX bound the real and the imaginary part
+  #  separately, not any magnitude of the pair.  This is what
+  #  np.finfo(np.complex64) answers too.
+
+  {
+    Complex64  => Float32,
+    Complex128 => Float64,
+  }.each do |klass, component|
+    [:MIN, :MAX, :TINY, :EPSILON].each do |name|
+      klass.const_set(name, component.const_get(name))
+    end
+  end
+
+end

data/lib/carray/fixlen_string.rb

@@ -3,7 +3,7 @@
 #  CAFixlenString high-level construction surface.
 #
 #  CAFixlenString itself (the string interpretation of CA_FIXLEN storage) lives
-#  in ext/ca_obj_string.c.  This file provides the ergonomic builder that packs
+#  in ext/ca_obj_fixlen_string.c.  This file provides the ergonomic builder that packs
 #  Ruby Strings into a fixed-width CA_FIXLEN entity and wraps it via
 #  CAFixlenString.wrap.
 #

data/lib/carray/frame/csv_parser.rb

@@ -91,13 +91,19 @@ class CAFrame
         @recno    = 0
       end
 
-      # Fields of the next non-blank record, or nil at EOF.
-      def read(io)
+      # Fields of the next record, or nil at EOF.  A blank line carries no
+      # separator, so it cannot be a row of a file with more than one column
+      # and is skipped as noise between records.  In a single-column file it
+      # is the only spelling a missing single field has -- which is what
+      # to_csv writes for a masked cell -- so the caller passes
+      # +blank_is_row: true+ once the column count is known to be one, and
+      # the empty record becomes a row of no fields for build_frame to pad.
+      def read(io, blank_is_row: false)
         loop do
           rec = CSVParser.read_record(io, @quote)
           return nil if rec.nil?
           @recno += 1
-          next if rec.empty?
+          next if rec.empty? && !blank_is_row
           return rec.count(@quote).zero? ? simple(rec) : scan(rec)
         end
       end
@@ -195,7 +201,8 @@ class CAFrame
     # Consume the remaining records as data rows.
     def body
       rows = []
-      while (fields = @tok.read(@io))
+      blank_is_row = @names && @names.size == 1
+      while (fields = @tok.read(@io, blank_is_row: blank_is_row))
         rows << fields
       end
       @rows = rows

data/lib/carray/frame/frame.rb

@@ -45,14 +45,21 @@ class CAFrame
     @columns   = {}
     @axis_name = axis_name || DEFAULT_AXIS_NAME
     @index     = nil
+    # The row axis name this frame had before set_index promoted a column over
+    # it, so reset_index can put it back.  A frame built with an index never
+    # had one, and neither does a frame derived from an indexed one: both are
+    # born indexed, so there is nothing to restore and the default stands.
+    @axis_name_before_index = nil
 
     n = nil
+    n_from = nil
     columns.each do |name, col|
       key = name.to_s
       ca  = coerce_column(col)
       len = ca.shape[0]
       if n.nil?
-        n = len
+        n      = len
+        n_from = key
       elsif len != n
         raise ArgumentError,
               "column #{key.inspect} has axis-0 length #{len}, expected #{n}"
@@ -67,8 +74,12 @@ class CAFrame
         raise ArgumentError, "index must be a 1-D column (got ndim #{idx.ndim})"
       end
       if n && idx.shape[0] != n
+        # Name the column as well as the index: when the index is the frame's
+        # existing one and a column is the new arrival, blaming the index alone
+        # points at the side the caller cannot change.
         raise ArgumentError,
-              "index length #{idx.shape[0]} does not match nrow #{n}"
+              "column #{n_from.inspect} has axis-0 length #{n}, " \
+              "but the index has length #{idx.shape[0]}"
       end
       @index = idx
       @nrow  = idx.shape[0] if n.nil?
@@ -234,9 +245,10 @@ class CAFrame
     end
   end
 
-  # Bind +key+ to +column+, adding the name when it is new.  This is the one
-  # place a column enters an existing frame, so it is where the axis-0 length
-  # invariant is enforced (§12-A).  It is a replacement rather than an edit,
+  # Bind +key+ to +column+, adding the name when it is new.  This is where a
+  # column arriving from outside the frame has its axis-0 length checked
+  # (§12-A); the conversions in verbs.rb rebind +@columns+ directly and rely on
+  # being length-preserving instead.  It is a replacement rather than an edit,
   # which is what sets it apart from the rest: +fill+ / +mask_eq+ /
   # +df[rows] = UNDEF+ write to the shared column and are therefore visible
   # wherever it is held, while this binds the name to a different column and
@@ -307,6 +319,12 @@ class CAFrame
   # value kept, so the row stays identifiable), leaving a later, better-informed
   # pass to re-judge them. Definitely-true rows carry their values through
   # unchanged in both modes.
+  #
+  # The default is a view-frame. +keep_masked: true+ returns a **materialized**
+  # frame instead -- carrying the UNDEF forward means writing it into the
+  # result, which a view cannot do without masking the parent's rows. That holds
+  # whether or not the selector actually carries a masked cell, so the same call
+  # site does not switch between sharing and copying with the data.
   def filter(keep_masked: false)
     mask = yield(self)
     unless mask.is_a?(CArray) && mask.data_type == :boolean
@@ -330,7 +348,18 @@ class CAFrame
     unless idx.ndim == 1
       raise ArgumentError, "index must be a 1-D column (got ndim #{idx.ndim})"
     end
+    # Only the name held before the frame had an index: a second set_index must
+    # not overwrite it with the first index's name.
+    @axis_name_before_index = @axis_name if @index.nil?
     @columns.delete(key)
+    if @index
+      # Re-indexing replaces the index, so the one being replaced goes back to
+      # being a column -- the same demotion reset_index performs, in the same
+      # position, which makes set_index over an existing index equivalent to
+      # reset_index followed by set_index.  Dropping it would lose the column it
+      # was made from.
+      @columns = { @axis_name => @index }.merge(@columns)
+    end
     @index     = idx
     @axis_name = key
     @nrow      = idx.shape[0]
@@ -344,7 +373,8 @@ class CAFrame
     return self unless @index
     @columns   = { @axis_name => @index }.merge(@columns)
     @index     = nil
-    @axis_name = DEFAULT_AXIS_NAME
+    @axis_name = @axis_name_before_index || DEFAULT_AXIS_NAME
+    @axis_name_before_index = nil
     self
   end
 
@@ -370,6 +400,17 @@ class CAFrame
   # multi-row, frame-returning path). Positional access is +df[i]+.
   def at(label)
     raise ArgumentError, "at requires an index (set one with set_index)" unless @index
+    # An index may hold a masked cell -- an outer join and align both build one
+    # -- but an undefined label identifies no row, and two undefined labels are
+    # not the same label. The addressing primitives already say this (a masked
+    # key matches nothing, not another masked key), so at says it too rather
+    # than answering through eq(UNDEF), which asks about the mask and not about
+    # the label at all.
+    if UNDEF.equal?(label)
+      raise ArgumentError,
+            "at(UNDEF): an undefined label identifies no row; " \
+            "use filter { |f| f.index.is_masked } for the rows with no label"
+    end
     pos = @index.eq(label).where
     case pos.elements
     when 0
@@ -483,6 +524,19 @@ class CAFrame
   # and index are untouched; the selected cells of every column go to UNDEF.
   # The selector is forwarded to the column indexer, which classifies it.
   private def mask_rows(selector)
+    # Decide before changing anything. A read-only column (a categorical, whose
+    # codes are read-only) refuses the write, and refusing half way through
+    # would leave the frame masked in whichever columns happened to come first.
+    refusing = @columns.find { |_, col| col.read_only? }
+    if refusing
+      raise "can not modify read-only array: column #{refusing[0].inspect} " \
+            "refuses the write, so no column was masked"
+    end
+    # The selector is classified by the column indexer, which is also what
+    # bound-checks it -- so with no column to forward it to, nothing would.
+    # Build the row mask purely to have the frame's own row axis refuse an
+    # out-of-range row, as every other row form does.
+    selected_row_mask(selector) if @columns.empty?
     @columns.each_value do |col|
       col[selector, *([nil] * (col.ndim - 1))] = UNDEF
     end
@@ -540,7 +594,16 @@ class CAFrame
     new_index = splice_index(other, lo, hi)
     @columns  = new_cols
     @index    = new_index
-    @nrow     = @nrow - (hi - lo) + other.nrow
+    # Read the new count off something the frame now holds, rather than
+    # computing it. For every column the pieces welded above are head (lo) +
+    # other's rows + tail (nrow - hi), so the arithmetic and the objects agree
+    # -- except when there is nothing to weld: a frame with no columns and no
+    # index would otherwise come away claiming rows that nothing backs.
+    @nrow = if (witness = new_cols.each_value.first || new_index)
+              witness.shape[0]
+            else
+              0
+            end
     self
   end
 
@@ -609,8 +672,11 @@ class CAFrame
   end
 
   private def select_rows(selector, keep_masked: false)
-    if keep_masked && selector.is_a?(CArray) &&
-       selector.data_type == :boolean && selector.has_mask?
+    if keep_masked && selector.is_a?(CArray) && selector.data_type == :boolean
+      # Not gated on selector.has_mask?: gating there would make the same call
+      # site return a view-frame or a materialized one depending on whether that
+      # run's data happened to produce an undetermined cell (the reason splice
+      # always snapshots -- see CAFrame.md section 3).
       return select_rows_keep_masked(selector)
     end
     cols = {}
@@ -631,7 +697,12 @@ class CAFrame
       g[undet_kept, *tail] = UNDEF
       cols[name] = g
     end
-    new_index = @index ? @index[keep] : nil
+    # The index is copied along with the columns. Its values are carried over
+    # unmasked (an undetermined row keeps its label, as mask_rows leaves the
+    # index alone), but a materialized frame has to be materialized whole: an
+    # aliased index would write through to the parent from a frame whose columns
+    # do not.
+    new_index = @index ? @index[keep].copy : nil
     CAFrame.new(cols, axis_name: @axis_name, index: new_index)
   end
 

data/lib/carray/frame/group.rb

@@ -13,7 +13,7 @@ class CAFrame
     raise ArgumentError, "group_by needs at least one key" if keys.empty?
     cat  = grouping_categorical(keys)
     axis = keys.size == 1 && keys.first.is_a?(String) ? keys.first : "group"
-    GroupedFrame.new(self, cat, axis)
+    GroupedFrame.new(self, cat, axis, keys.grep(String))
   end
 
   # Number of rows currently selected — used by group per-group view-frames
@@ -28,7 +28,14 @@ class CAFrame
       # values, categorized by content (the codes are composed from the
       # per-column keys).
       n = nrow
-      CArray.object(n) { |i| cols.map { |c| c[i] } }.categorize
+      key = CArray.object(n) { |i| cols.map { |c| c[i] } }
+      # One undetermined component makes the whole tuple undetermined, the same
+      # answer a single masked key cell gets. Left as a value, the UNDEF inside
+      # the tuple would intern as an ordinary distinct key and the row would
+      # form a group of its own.
+      undetermined = CArray.boolean(n) { |i| key[i].any? { |v| UNDEF.equal?(v) } }
+      key[undetermined] = UNDEF if undetermined.any
+      key.categorize
     end
   end
 
@@ -56,10 +63,14 @@ end
 #   aggregate    -> declarative per-column reductions into a new frame
 #   table { |g| }-> cross-column Ruby escape, g is a per-group view-frame
 class GroupedFrame
-  def initialize(frame, cat, axis_name)
+  # +key_names+ are the frame columns the grouping was keyed on. They become
+  # the result's index, so the reduction shortcuts must not also return them as
+  # reduced columns; an external CArray key contributes no name.
+  def initialize(frame, cat, axis_name, key_names = [])
     @frame     = frame
     @cat       = cat
     @axis_name = axis_name
+    @key_names = key_names
     @labels    = cat.labels          # group values, in code order
   end
 
@@ -124,6 +135,23 @@ class GroupedFrame
 
   # Convenience reductions over every numeric scalar column (memo §6-4
   # "grp.mean"). Non-numeric / N-D columns are skipped.
+  #
+  # @!method sum
+  #   Returns a frame of the per-group sum of every numeric one-dimensional
+  #   column. Non-numeric and multi-dimensional columns are left out.
+  #   @return [CAFrame] one row per group, indexed by the group labels.
+  # @!method mean
+  #   Returns a frame of the per-group arithmetic mean of every numeric
+  #   one-dimensional column, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method min
+  #   Returns a frame of the per-group minimum of every numeric
+  #   one-dimensional column, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method max
+  #   Returns a frame of the per-group maximum of every numeric
+  #   one-dimensional column, as {#sum} does.
+  #   @return [CAFrame] one row per group.
   [:sum, :mean, :min, :max].each do |red|
     define_method(red) { reduce_numeric(red) }
   end
@@ -143,6 +171,11 @@ class GroupedFrame
   private def reduce_numeric(reduction)
     cols = {}
     @frame.variable_names.each do |name|
+      # A key column is the index here, not a result column. Filtering it out
+      # by name rather than by data type is what makes a numeric key behave
+      # like a string one: NON_NUMERIC is about which columns a reduction can
+      # apply to, which happened to cover string keys and nothing else.
+      next if @key_names.include?(name)
       col = @frame[name]
       next unless col.ndim == 1 && !NON_NUMERIC.include?(col.data_type)
       cols[name] = col.group_by_category(@cat).public_send(reduction)