carray 3.0.3 → 3.0.5

data/lib/carray/frame/group.rb

@@ -9,12 +9,72 @@
 class CAFrame
   # Group rows by one or more keys. Each key is a column name (String) or an
   # external length-N CArray. Returns a GroupedFrame.
+  #
+  # With one key, a summary's index is the key's value of each group. With
+  # several, each key is a column of a summary, holding its part of each
+  # group's key, and the summary has no index; a CArray key among several
+  # names its column by its position, "key0", "key1", ....
   def group_by(*keys)
     raise ArgumentError, "group_by needs at least one key" if keys.empty?
+    CAFrame.__send__(:refuse_repeated_names, keys.grep(String), "group_by")
     cat  = grouping_categorical(keys)
-    axis = keys.size == 1 && keys.first.is_a?(String) ? keys.first : "group"
-    key  = keys.size == 1 ? key_column(keys.first) : nil
-    GroupedFrame.new(self, cat, axis, keys.grep(String), key: key)
+    if keys.size == 1
+      axis = keys.first.is_a?(String) ? keys.first : "group"
+      return GroupedFrame.new(self, cat, axis, keys.grep(String), key: key_column(keys.first))
+    end
+    names = keys.each_with_index.map { |k, i| k.is_a?(String) ? k : "key#{i}" }
+    CAFrame.__send__(:refuse_repeated_names, names, "group_by")
+    GroupedFrame.new(self, cat, "group", keys.grep(String),
+                     key_columns: names.zip(keys.map { |k| key_column(k) }).to_h)
+  end
+
+  # Whether each row repeats another row: a boolean column, true for the rows
+  # whose values in the named columns (every column when none are named)
+  # appear on more than one row, except the one +except:+ leaves out --
+  #
+  #   except: :first   every occurrence but the first (the default)
+  #   except: :last    every occurrence but the last
+  #   except: :none    every occurrence
+  #
+  # so +df[df.is_duplicate.not]+ keeps one row of each, and
+  # +is_duplicate(except: :none).not+ is the rows that appear only once.
+  # Values compare as group_by compares keys (NaN is one value, -0.0 is 0.0),
+  # and a row with a masked cell in those columns is never a duplicate, as a
+  # masked key is in no group. The index is compared only when named
+  # (by the row axis name, as group_by takes it).
+  #
+  #   df[df.is_duplicate.not]                # the rows without repeats
+  #   df.is_duplicate("station", "time")     # repeats of the pair
+  #   df.is_duplicate(except: :none).not     # the rows that appear once
+  #
+  # @param names [Array<String>] the columns to compare; default every column.
+  # @param except [Symbol] the occurrence left unmarked: :first, :last or :none.
+  # @return [CArray] boolean, one value per row.
+  # @raise [KeyError] for a name that is no column.
+  # @raise [ArgumentError] when a compared column is N-D, there is no column
+  #   to compare, or +except:+ is not one of the three.
+  def is_duplicate(*names, except: :first)
+    unless %i[first last none].include?(except)
+      raise ArgumentError,
+            "is_duplicate: except: must be :first, :last or :none (got #{except.inspect})"
+    end
+    names = column_keys(names, "is_duplicate")
+    names = column_names if names.empty?
+    raise ArgumentError, "is_duplicate: the frame has no column to compare" if names.empty?
+    cols = names.map { |n| key_column(n, "is_duplicate") }
+    cat   = CACategorical.send(:from_keys, cols)
+    codes = cat.codes
+    out =
+      case except
+      when :first then codes.mask_duplicates.is_masked
+      when :last  then codes.flip(0).mask_duplicates.is_masked.flip(0).copy
+      when :none
+        index = codes.value.int64
+        index[codes.is_masked] = 0 if codes.has_mask?
+        sizes = cat.category_sizes
+        sizes.elements > 0 ? sizes[index].gt(1) : CArray.boolean(nrow) { false }
+      end
+    codes.has_mask? ? out & codes.is_not_masked : out
   end
 
   # Group rows into time bins of length +unit+ along the time column (or the
@@ -38,17 +98,28 @@ class CAFrame
   # as an empty reduction does: UNDEF for +mean+, 0 for +count+ and +sum+.
   # A row whose time is masked belongs to no bin.
   #
+  # +by:+ (a column name or an Array of them) bins each group of rows on its
+  # own: a group is the rows sharing the +by:+ values and a bin. The result
+  # then has no index: the +by:+ columns and the bin times (named +name+)
+  # are its first columns, the groups in the order their +by:+ values first
+  # appear and the bins of each in time order. With +fill: true+ every group
+  # has every bin from the first to the last of the whole frame. A row whose
+  # +by:+ value is masked belongs to no group.
+  #
+  #   df.resample("date", "1 week", by: "shop").sum
+  #
   # @param name [String] the time column, or the index's axis name.
   # @param unit [String, Symbol, CATime::Resolution] the bin length.
   # @param origin [nil, Time, String, CATime::Element] where bins start.
   # @param label [:left, :right] which end of a bin names it and is closed.
   # @param fill [Boolean] whether bins with no rows are rows of the result.
+  # @param by [nil, String, Array<String>] columns whose groups are binned apart.
   # @return [GroupedFrame]
   # @raise [ArgumentError] when the column is not a CATime, or +label:+ is
-  #   neither :left nor :right.
-  def resample(name, unit, origin: nil, label: :left, fill: false)
-    name = name.to_s
-    time = column_or_index(name)
+  #   neither :left nor :right, or +by:+ names the time column.
+  def resample(name, unit, origin: nil, label: :left, fill: false, by: nil)
+    name = column_key(name, "resample")
+    time = key_column(name, "resample")
     unless time.is_a?(CATime)
       raise ArgumentError, "resample: #{name.inspect} is not a time column (got #{time.class})"
     end
@@ -59,18 +130,62 @@ class CAFrame
       else
         raise ArgumentError, "resample: label: must be :left or :right (got #{label.inspect})"
       end
+    # key-labels: the bins are the index, given the column's attributes below.
     grid = fill && bins.count_not_masked > 0 ? resample_grid(bins, unit) : bins.unique(sort: true)
+    return resample_by(name, time, bins, grid, by, fill) unless by.nil?
     cat = CACategorical.from_codes(bins.locate_addr(grid), grid.to_a)
-    GroupedFrame.new(self, cat, name, @columns.key?(name) ? [name] : [], index: grid)
+    GroupedFrame.new(self, cat, name, @columns.key?(name) ? [name] : [],
+                     index: CAFrame.__send__(:key_labels, time, grid))
+  end
+
+  # resample with by:. A group is a (by values, bin) pair, coded as
+  # group * bins + bin, so the codes run through the groups in the order their
+  # by values first appear and through each group's bins in time order.
+  private def resample_by(name, time, bins, grid, by, fill)
+    by_names = column_keys(Array(by), "resample")
+    raise ArgumentError, "resample: by: names no column" if by_names.empty?
+    if by_names.include?(name)
+      raise ArgumentError, "resample: by: names the time column #{name.inspect}"
+    end
+    by_cols = by_names.map { |n| key_column(n, "resample") }
+    gcat = CACategorical.send(:from_keys, by_cols)
+    ng   = gcat.labels.size
+    nb   = grid.elements
+    comb = gcat.codes.int64 * nb + bins.locate_addr(grid).int64
+    # key-labels: the used (group, bin) codes, not labels; those go through key_labels below
+    occupied = fill ? CArray.int64(ng * nb).seq! : comb.unique(sort: true)
+    codes = fill ? comb : comb.locate_addr(occupied)
+    g_of = nb > 0 ? occupied / nb : occupied
+    b_of = nb > 0 ? occupied % nb : occupied
+    # Each group's by values, from its first row.
+    order = gcat.group_addr
+    first = order[CArray.segment_offsets(lengths: gcat.category_sizes)[0...ng]]
+    columns = {}
+    by_names.zip(by_cols).each do |n, col|
+      columns[n] = CAFrame.__send__(:key_labels, col, col.project(first))[g_of].copy
+    end
+    columns[name] = CAFrame.__send__(:key_labels, time, grid[b_of].copy)
+    glabels = gcat.labels
+    blabels = grid.to_a
+    labels = occupied.to_a.map do |c|
+      g, b = c.divmod(nb)
+      parts = by_names.size == 1 ? [glabels[g]] : glabels[g]
+      parts + [blabels[b]]
+    end
+    cat = CACategorical.from_codes(codes, labels)
+    key_names = by_names + (@columns.key?(name) ? [name] : [])
+    GroupedFrame.new(self, cat, "group", key_names, label_columns: columns)
   end
 
-  # Every bin from the first occupied one to the last. A fixed-length bin
-  # steps from the first bin, so an origin's phase is kept; a month or a year
-  # cannot be stepped on a finer grid and is laid on its own unit, where
-  # bins start on the calendar boundary anyway.
+  # Every bin from the first occupied one to the last, stepped from the
+  # first bin so an origin's phase (or label: :right) is kept. A month or a
+  # year cannot be stepped on a finer grid, so it is stepped on the one-month
+  # or one-year grid and then given the bins' unit, as the labels without
+  # fill: are.
   private def resample_grid(bins, unit)
-    if %i[Y M].include?(CATime::Resolution.parse(unit).base)
-      CArray.time_range(bins.min, bins.max, unit: unit)
+    base = CATime::Resolution.parse(unit).base
+    if %i[Y M].include?(base)
+      CArray.time_range(bins.min, bins.max, unit: base, step: unit).to_unit(bins.unit)
     else
       CArray.time_range(bins.min, bins.max, unit: bins.unit, step: unit)
     end
@@ -80,50 +195,32 @@ class CAFrame
   # and elsewhere; already provided by the core (attr_reader :nrow).
 
   private def grouping_categorical(keys)
-    cols = keys.map { |k| key_column(k) }
-    if cols.size == 1
-      cols.first.categorize
-    else
-      composite_categorical(cols)
-    end
+    CACategorical.send(:from_keys, keys.map { |k| key_column(k) })
   end
 
-  # A composite key.  Each column is categorized on its own, so a component
-  # compares exactly as a single key does (NaN is one value, -0.0 is 0.0, a
-  # masked cell matches nothing); a row's group is then the tuple of its
-  # component codes, and a group's label the tuple of the component labels.
-  # One masked component leaves the row in no group, as a masked single key.
-  private def composite_categorical(cols)
-    cats  = cols.map(&:categorize)
-    sizes = cats.map { |c| c.labels.size }
-    tuple = if sizes.inject(1, :*) <= 2**62
-              cats.drop(1).inject(cats[0].codes.int64) { |acc, c| acc * c.labels.size + c.codes.int64 }
-            else
-              codes = cats.map(&:codes)
-              t = CArray.object(nrow) { |i| codes.map { |c| c[i] } }
-              nrow.times { |i| t[i] = UNDEF if t[i].any? { |x| UNDEF.equal?(x) } }
-              t
-            end
-    rows = tuple.categorize
-    labels = rows.labels.map do |code|
-      parts = code.is_a?(Array) ? code : sizes.reverse.map { |n| code, d = code.divmod(n); d }.reverse
-      parts.each_with_index.map { |d, j| cats[j].labels[d] }
-    end
-    CACategorical.from_codes(rows.codes, labels)
-  end
-
-  # The column a group key names, checked: +verb+ is the method the caller
-  # called, for the messages.
+  # The column a key names, checked -- the one reading of a key every verb
+  # that takes one goes through (group_by, is_duplicate, sort_by_key, join,
+  # join_asof, align, pivot, resample, windows): a column of that name, or
+  # the index by the row axis name; or a CArray of one value per row.  It
+  # gives each row one value.  +verb+ is the method the caller called, for
+  # the messages.
   private def key_column(key, verb = "group_by")
     col = case key
           when String
-            @columns.fetch(key) { raise KeyError, "#{verb}: no column #{key.inspect}" }
+            # The index names the rows, so it is a key as a column is, by the
+            # row axis name, as resample and pivot take it.
+            @columns.fetch(key) do
+              next @index if @index && @axis_name == key
+              raise KeyError, "#{verb}: no column #{key.inspect}"
+            end
           when CArray
             unless key.shape[0] == nrow
               raise ArgumentError,
                     "#{verb}: the key has #{key.shape[0]} values for #{nrow} rows"
             end
             key
+          when Symbol
+            column_key(key, verb)
           else
             raise ArgumentError, "#{verb}: a group key is a column name or a CArray (got #{key.class})"
           end
@@ -133,7 +230,7 @@ class CAFrame
       name = key.is_a?(String) ? "#{key.inspect} " : ""
       raise ArgumentError,
             "#{verb}: the key #{name}has shape #{col.shape.inspect}, not one value per row; " \
-            "group by one of its components (e.g. df[name][nil, 0])"
+            "use one of its components (e.g. df[name][nil, 0])"
     end
     col
   end
@@ -142,11 +239,13 @@ end
 # GroupedFrame — the result of +CAFrame#group_by+ (memo §11.6, §16).
 #
 # Holds the grouping categorical once (grouping is per-key, computed once
-# and shared across every column). Three surfaces:
-#   grp["col"]   -> the CArray group iterator (raw exposure)
-#   aggregate    -> declarative per-column reductions into a new frame
-#   table { |g| }-> cross-column Ruby escape, g is a per-group view-frame
+# and shared across every column). The surfaces:
+#   grp["col"], grp[array] -> the CArray group iterator of a column or array
+#   aggregate(spec)        -> declarative per-column reductions into a new frame
+#   aggregate { |grp| }    -> columns built from iterators, run once
+#   table { |g| }          -> Ruby escape, g is a per-group view-frame
 class GroupedFrame
+  include CAFrame::ColumnVerbs
   # +key_names+ are the frame columns the grouping was keyed on. They become
   # the result's index, so the reduction shortcuts must not also return them as
   # reduced columns; an external CArray key contributes no name.
@@ -154,17 +253,27 @@ class GroupedFrame
   # The result's index is +index:+ when given (one entry per group, in code
   # order); otherwise, for a single +key:+, the key's value at each group's
   # first row, so it keeps the key's data type and Face; otherwise an object
-  # array of the labels (a composite key's tuples).
+  # array of the labels.
+  #
+  # With several keys the result has no index: +key_columns:+ (name => the
+  # key, one value per row) become its first columns, each holding the key's
+  # value at each group's first row; +label_columns:+ (name => one value per
+  # group, in code order) give them directly, for groups that may have no
+  # rows (resample with fill: true).
   #
   # The grouping and the index are taken here, so a later change to the key
   # does not rename the groups; the values are read when a reduction is.
-  def initialize(frame, cat, axis_name, key_names = [], index: nil, key: nil)
+  def initialize(frame, cat, axis_name, key_names = [], index: nil, key: nil,
+                 key_columns: nil, label_columns: nil)
     @frame     = frame
     @cat       = cat
     @axis_name = axis_name
     @key_names = key_names
     @labels    = cat.labels          # group values, in code order
-    @index     = index || (key && key.project(group_perm[group_bounds[0...ngroup]]))
+    @index     = index ||
+                 (key && CAFrame.__send__(:key_labels, key, key.project(first_rows)))
+    @label_columns = label_columns ||
+                     key_columns&.transform_values { |col| CAFrame.__send__(:key_labels, col, col.project(first_rows)) }
   end
 
   # Number of groups.
@@ -177,9 +286,40 @@ class GroupedFrame
     @labels.dup
   end
 
-  # Raw exposure: the CArray group iterator for one column (memo §11.6).
-  def [](name)
-    @frame[name].group_by_category(@cat)
+  # The CArray group iterator of a column, or of any CArray with one entry
+  # per row of the frame: a value computed from columns, or a column with
+  # some cells masked to leave them out. Its reductions answer one value
+  # per group, and take the arguments a CArray's reductions take.
+  #
+  #   grp["temp"].percentile(90)
+  #   grp[df["temp"] * df["w"]].sum / grp["w"].sum
+  #   grp[df["temp"].mask_where(df["qc"].ne(0))].mean   # without the flagged rows
+  #
+  # The array is matched to the rows by position, as a CArray key of
+  # +group_by+ is. An N-D one is grouped along its first axis; pass
+  # +axis: 0+ to its reductions to keep the trailing shape.
+  #
+  # @param source [String, CArray] a column name, or a CArray whose first
+  #   axis has one entry per row.
+  # @return [CACategoricalIterator]
+  # @raise [KeyError] for a name that is no column.
+  # @raise [ArgumentError] when the CArray's first axis is not the rows.
+  def [](source)
+    refuse_changed_rows("[]")
+    values =
+      case source
+      when CArray
+        rows = source.ndim == 0 ? 0 : source.shape[0]
+        unless source.ndim >= 1 && rows == @cat.elements
+          raise ArgumentError,
+                "[]: the array has #{source.ndim == 0 ? "no axis" : "#{rows} rows"} " \
+                "for the frame's #{@cat.elements}"
+        end
+        source
+      else
+        @frame[@frame.__send__(:column_key, source, "[]")]
+      end
+    values.group_by_category(@cat)
   end
 
   # Declarative aggregation (memo §11.6). Spec maps an output column name to
@@ -188,10 +328,46 @@ class GroupedFrame
   # with the group's column slice). A Symbol reduction takes its arguments
   # after it, as in a call: +["temp", :percentile, 90.0]+, and its keywords
   # last: +["temp", :mean, min_count: 6]+.
-  def aggregate(spec)
+  #
+  # With +keep_rows: true+ the rows are not collapsed: each row of the frame
+  # gets the result of its own group, so the result has the frame's rows and
+  # index, and a row whose key is masked (in no group) is UNDEF. The same
+  # keyword works for +table+ and the reductions below.
+  #
+  #   df.group_by("station").aggregate("temp_mean" => ["temp", :mean], keep_rows: true)
+  #
+  # Given a block instead of a spec, aggregate yields the grouped frame once
+  # and the block returns a Hash of output name => a CArray with one entry
+  # per group. The block computes the columns with the grouped frame's
+  # reductions, +grp[...]+ iterators and CArray arithmetic, so nothing runs
+  # per group in Ruby:
+  #
+  #   df.group_by("station").aggregate do |g|
+  #     {
+  #       "temp"  => g["temp"].mean,
+  #       "range" => g["temp"].max - g["temp"].min,
+  #       "wmean" => g[df["temp"] * df["w"]].sum / g["w"].sum,
+  #       "n"     => g["temp"].group_sizes,
+  #     }
+  #   end
+  #
+  # +keep_rows: true+ lays each column over the rows as for a spec.
+  #
+  # @raise [ArgumentError] when both a spec and a block are given, or the
+  #   block returns something other than a Hash of per-group CArrays.
+  def aggregate(spec = nil, keep_rows: false, **named, &block)
+    refuse_changed_rows("aggregate")
+    if block
+      unless spec.nil? && named.empty?
+        raise ArgumentError, "aggregate: give a spec or a block, not both"
+      end
+      return result_frame(block_columns(yield(self)), keep_rows)
+    end
+    spec = (spec || {}).merge(named)
     cols = {}
     spec.each do |out_name, (in_name, reduction, *args)|
-      out = out_name.to_s
+      out = @frame.__send__(:column_name_arg, out_name, "aggregate")
+      @frame.__send__(:column_key, in_name, "aggregate")
       options = args.last.is_a?(Hash) ? args.pop : {}
       cols[out] =
         case reduction
@@ -207,29 +383,34 @@ class GroupedFrame
                 "reduction must be a Symbol or Proc (got #{reduction.class})"
         end
     end
-    CAFrame.new(cols, axis_name: @axis_name, index: label_index)
+    result_frame(cols, keep_rows)
   end
 
   # Cross-column Ruby escape (memo §11.6). The block receives a per-group
   # view-frame and returns a Hash of output-name => value; the values are
-  # collected column-wise across groups into a new frame.
-  def table
-    collected = {}
-    order = nil
+  # collected column-wise across groups into a new frame. +keep_rows: true+
+  # gives each row its group's values, as for +aggregate+.
+  #
+  # The columns are the names any group's Hash has, in the order they
+  # first appear; a group whose Hash lacks a name is UNDEF there, as
+  # +from_records+ reads a record that lacks a key.
+  def table(keep_rows: false)
+    refuse_changed_rows("table")
+    outs = []
     each_group_frame do |g|
       out = yield(g)
       unless out.is_a?(Hash)
         raise ArgumentError, "table block must return a Hash (got #{out.class})"
       end
-      order ||= out.keys.map(&:to_s)
-      out.each { |k, v| (collected[k.to_s] ||= []) << v }
+      out.each_key { |k| @frame.__send__(:column_name_arg, k, "table") }
+      outs << out
     end
+    names = CAFrame.__send__(:record_key_union, outs)
     cols = {}
-    (order || []).each do |name|
-      vals = collected[name]
-      cols[name] = CArray.object(vals.size) { |i| vals[i] }
+    names.each do |name|
+      cols[name] = column_of_values(outs.map { |out| out.fetch(name, nil) })
     end
-    CAFrame.new(cols, axis_name: @axis_name, index: label_index)
+    result_frame(cols, keep_rows)
   end
 
   # Convenience reductions over every column that defines the reduction. A
@@ -240,28 +421,249 @@ class GroupedFrame
   # its trailing shape: a (N, L) column gives (groups, L). Each
   # takes +min_count:+ and +fill_value:+ as a core reduction does: a group
   # with fewer than min_count present values is UNDEF, and fill_value fills
-  # the UNDEF cells.
+  # the UNDEF cells. One fill_value goes to every column and has to suit each
+  # (a number into a string column raises); a Hash of column name => value
+  # fills those columns only. +keep_rows: true+ gives each row its group's value
+  # instead of one row per group, as for +aggregate+.
   #
-  # @!method sum(min_count: nil, fill_value: nil)
+  # Given column names, they reduce those columns only and return them as
+  # +df[...]+ does: one name gives the CArray, several an Array of them.
+  #
+  #   grp.mean                                   # a frame, one row per group
+  #   grp.mean("temp")                           # CArray, one value per group
+  #   df["temp"] - grp.mean("temp", keep_rows: true)   # anomaly from the group mean
+  #
+  # @!method sum(*names, min_count: nil, fill_value: nil, keep_rows: false)
   #   Returns a frame of the per-group sum of every column whose type has a
   #   sum, an N-D one per group along the rows. Other columns are left out.
-  #   @return [CAFrame] one row per group, indexed by the group labels.
-  # @!method mean(min_count: nil, fill_value: nil)
+  #   @return [CAFrame, CArray, Array<CArray>] a frame with one row per
+  #     group, indexed by the group labels, or with keep_rows: true the
+  #     frame's rows and index; given names, the CArray (one) or the Array of
+  #     them (several).
+  # @!method mean(*names, min_count: nil, fill_value: nil, keep_rows: false)
   #   Returns a frame of the per-group mean of every column whose type has
   #   a mean, as {#sum} does.
   #   @return [CAFrame] one row per group.
-  # @!method min(min_count: nil, fill_value: nil)
+  # @!method min(*names, min_count: nil, fill_value: nil, keep_rows: false)
   #   Returns a frame of the per-group minimum of every column whose type has
   #   a minimum, as {#sum} does.
   #   @return [CAFrame] one row per group.
-  # @!method max(min_count: nil, fill_value: nil)
+  # @!method max(*names, min_count: nil, fill_value: nil, keep_rows: false)
   #   Returns a frame of the per-group maximum of every column whose type has
   #   a maximum, as {#sum} does.
   #   @return [CAFrame] one row per group.
-  [:sum, :mean, :min, :max].each do |red|
-    define_method(red) do |min_count: nil, fill_value: nil|
-      reduce_columns(red, min_count: min_count, fill_value: fill_value)
+  # @!method prod(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group product, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method accumulate(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group sum kept in each column's own data type, as
+  #   +CArray#accumulate+, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method median(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group median, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method variance(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group sample variance (divisor n - 1), as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method stddev(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group sample standard deviation, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method variancep(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group population variance (divisor n), as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method stddevp(*names, min_count: nil, fill_value: nil, keep_rows: false)
+  #   Returns the per-group population standard deviation, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  CArray.send(:define_named_methods, self,
+              [:sum, :mean, :min, :max, :prod, :accumulate, :median,
+               :variance, :stddev, :variancep, :stddevp], :group_reduction)
+
+  private def group_reduction (red, *names, min_count: nil, fill_value: nil, keep_rows: false)
+    refuse_changed_rows(red)
+    reduce_columns(red, names, keep_rows, min_count: min_count, fill_value: fill_value)
+  end
+
+  # The same convenience for the reductions that take no min_count: or
+  # fill_value:, as the core ones take none: a value of the group (first,
+  # last), a count (nunique, count) or a boolean fold (all, any).
+  #
+  # @!method first(*names, keep_rows: false)
+  #   Returns each group's first present value in row order, UNDEF for a
+  #   group with none, as {#sum} returns its sums.
+  #   @return [CAFrame] one row per group.
+  # @!method last(*names, keep_rows: false)
+  #   Returns each group's last present value in row order.
+  #   @return [CAFrame] one row per group.
+  # @!method nunique(*names, keep_rows: false)
+  #   Returns each group's number of distinct present values (int64).
+  #   @return [CAFrame] one row per group.
+  # @!method count(*names, keep_rows: false)
+  #   Returns each group's number of present values (int64), a masked cell
+  #   left out.
+  #   @return [CAFrame] one row per group.
+  # @!method all(*names, keep_rows: false)
+  #   Returns whether every present value of the group is true, for the
+  #   boolean columns.
+  #   @return [CAFrame] one row per group.
+  # @!method any(*names, keep_rows: false)
+  #   Returns whether any present value of the group is true, for the
+  #   boolean columns.
+  #   @return [CAFrame] one row per group.
+  CArray.send(:define_named_methods, self,
+              [:first, :last, :nunique, :count, :all, :any], :group_value)
+
+  private def group_value (red, *names, keep_rows: false)
+    refuse_changed_rows(red)
+    reduce_columns(red, names, keep_rows)
+  end
+
+  # Values computed within each group and kept on the frame's rows, in row
+  # order: the value some rows back in the same group, a rank among the
+  # group's rows, a running statistic. A row in no group is UNDEF. Without
+  # names they give a frame of every column whose type has the operation
+  # (the key columns left out), on the frame's rows and index; given names,
+  # the CArray (one) or the Array of them (several), as +df[...]+ returns
+  # columns. An N-D column runs along the rows.
+  #
+  #   df["dtemp"] = df["temp"] - grp.lag("temp")   # change since the station's previous row
+  #   grp.order("temp", descending: true)          # rank within the station
+  #   grp.cumsum("rain")                           # running total per station
+  #
+  # "Previous" is the previous row of the frame: sort the frame first
+  # (+sort_by_key("time")+) when its rows are not in time order.
+  #
+  # @!method lag(*names, n: 1)
+  #   Returns each row's value +n+ rows back in its own group, UNDEF where
+  #   the group has none, as +CArray#lag+; a negative +n+ looks ahead.
+  #   @return [CAFrame, CArray, Array<CArray>]
+  # @!method order(*names, descending: false, method: :ordinal, kind: :quick)
+  #   Returns each row's rank among its group's rows (int64, 0 the
+  #   smallest), as +CArray#order+ ranks them.
+  #   @return [CAFrame, CArray, Array<CArray>]
+  # @!method cumsum(*names)
+  #   Returns the running sum within each group, as +CArray#cumsum+.
+  #   @return [CAFrame, CArray, Array<CArray>]
+  # @!method cumprod(*names)
+  #   Returns the running product within each group.
+  #   @return [CAFrame, CArray, Array<CArray>]
+  # @!method cummax(*names)
+  #   Returns the running maximum within each group.
+  #   @return [CAFrame, CArray, Array<CArray>]
+  # @!method cummin(*names)
+  #   Returns the running minimum within each group.
+  #   @return [CAFrame, CArray, Array<CArray>]
+  # @!method cumcount(*names)
+  #   Returns the running count of present values within each group (int64).
+  #   @return [CAFrame, CArray, Array<CArray>]
+  def lag (*names, n: 1)
+    transform_columns(:lag, names, [n], {})
+  end
+
+  def order (*names, descending: false, method: :ordinal, kind: :quick)
+    transform_columns(:order, names, [], { descending: descending, method: method, kind: kind })
+  end
+
+  CArray.send(:define_named_methods, self,
+              [:cumsum, :cumprod, :cummax, :cummin, :cumcount], :group_scan)
+
+  private def group_scan (op, *names)
+    transform_columns(op, names, [], {})
+  end
+
+  # Fill the masked cells of a column from the rows of the same group, in
+  # place, as +CAFrame#fill+ does for the whole column: +:ffill+ carries the
+  # group's last present value forward in row order, +:bfill+ its next one
+  # back. A fill never reaches across into another group, so a cell with
+  # nothing to carry in its group stays masked, and a row whose key is
+  # masked (in no group) is left as it is. A constant fills every masked
+  # cell, as +CAFrame#fill+ does. Returns self.
+  #
+  #   df.group_by("station").fill("temp", :ffill)
+  #
+  # Sort the frame first (+sort_by_key("time")+) when its rows are not in
+  # time order.
+  #
+  # @param name [String] the column.
+  # @param method_or_value [Symbol, Object] +:ffill+, +:bfill+, or a constant.
+  # @return [GroupedFrame] self
+  # @raise [ArgumentError] for +:linear+, which within groups is not
+  #   provided, or another Symbol.
+  def fill (name, method_or_value)
+    refuse_changed_rows("fill")
+    case method_or_value
+    when :ffill, :bfill
+      @frame.__send__(:refuse_if_frozen)
+      key = @frame.__send__(:column_key, name, "fill")
+      col = @frame[key]
+      # A row in no group is UNDEF in the fill, so writing it changes nothing.
+      held = group_transform(col, :strip_mask, [], { method: method_or_value })
+      target = col.is_masked
+      col[target] = held[target] if target.any
+    when :linear
+      raise ArgumentError,
+            "fill(#{name.inspect}, :linear): an interpolation within each group " \
+            "is not provided; use :ffill or :bfill, or interpolate one group at a " \
+            "time with CAFrame#fill"
+    when Symbol
+      raise ArgumentError,
+            "fill: unknown method #{method_or_value.inspect} " \
+            "(:ffill | :bfill, or a constant fill value)"
+    else
+      @frame.fill(name, method_or_value)
+    end
+    self
+  end
+
+  # Keep the rows of the groups the block chooses. The block receives the
+  # grouped frame and returns a boolean CArray with one value per group,
+  # built from the group reductions, as +CAFrame#filter+'s block returns one
+  # per row:
+  #
+  #   grp.filter { |g| g["temp"].count_not_masked >= 20 }   # stations with 20 readings
+  #   grp.filter { |g| g.mean("temp") > 15.0 }
+  #
+  # The rows of a chosen group are kept in the frame's order, as a sub-frame
+  # sharing the frame's columns. A group whose value is UNDEF is
+  # undetermined, as is a row whose key is masked (it belongs to no group):
+  # such rows are dropped, or with +keep_masked: true+ carried with their
+  # cells masked, as +CAFrame#filter+ does.
+  #
+  # @param keep_masked [Boolean] carry the undetermined rows forward masked.
+  # @yieldparam grouped [GroupedFrame] self.
+  # @yieldreturn [CArray] boolean, one value per group.
+  # @return [CAFrame]
+  # @raise [ArgumentError] when the block does not return a boolean CArray of
+  #   shape [ngroup].
+  def filter (keep_masked: false)
+    refuse_changed_rows("filter")
+    chosen = yield(self)
+    unless chosen.is_a?(CArray) && chosen.data_type == :boolean
+      raise ArgumentError,
+            "filter block must return a boolean CArray (got #{chosen.is_a?(CArray) ? "a #{CArray::Inspector.type_label(chosen)} CArray" : chosen.class})"
+    end
+    unless chosen.shape == [ngroup]
+      raise ArgumentError,
+            "filter mask has shape #{chosen.shape.inspect}, " \
+            "expected [#{ngroup}] (one value per group)"
     end
+    rows = spread_to_rows(chosen)
+    @frame.filter(keep_masked: keep_masked) { rows }
+  end
+
+  # Returns the grouped rows as a frame: the frame the grouping was made
+  # from, its rows, columns and index, as a new frame sharing its columns.
+  # Nothing is copied, so a write to a cell reaches the grouped frame's
+  # rows; adding or dropping a column changes the new frame only.
+  #
+  #   def warm_stations (grp)
+  #     grp.filter { |g| g.mean("temp") > 15.0 }   # the rows of some groups
+  #   end
+  #   grp.to_frame                                  # all of its rows
+  #
+  # @return [CAFrame]
+  def to_frame
+    refuse_changed_rows("to_frame")
+    @frame.__send__(:rebuild, @frame.column_names.to_h { |name| [name, @frame[name]] })
   end
 
   # @return [String]
@@ -269,22 +671,81 @@ class GroupedFrame
     "#<GroupedFrame ngroup=#{ngroup} by=#{@axis_name.inspect}>"
   end
 
+  # A within-group transform of the named columns, or of every column whose
+  # type has it. The arguments are checked by the core first: below, a
+  # refusal reads as "this type has no such operation", and a wrong keyword
+  # would otherwise leave every column out without a word.
+  private def transform_columns(op, names, args, keywords)
+    refuse_changed_rows(op)
+    CArray.float64(1).public_send(op, *args, **keywords)
+    out = over_named_columns(op, names, @key_names, args, keywords) do |col|
+      group_transform(col, op, args, keywords)
+    end
+    return out unless names.empty?
+    CAFrame.new(out, axis_name: @frame.axis_name, index: @frame.index)
+  end
+
+  # A column's within-group transform: an N-D column runs along the rows.
+  private def group_transform(col, op, args, keywords)
+    it = col.group_by_category(@cat)
+    keywords = keywords.merge(axis: 0) if col.ndim > 1
+    it.public_send(op, *args, **keywords)
+  end
+
   # Each result gets its own index.
   private def label_index
     return @index.copy if @index
     CArray.object(@labels.size) { |i| @labels[i] }
   end
 
-  private def reduce_columns(reduction, **keywords)
-    cols = {}
-    @frame.column_names.each do |name|
-      # A key column is the index here, not a result column.
-      next if @key_names.include?(name)
-      col = @frame[name]
-      next unless answers?(col, reduction)
-      cols[name] = group_reduce(col, reduction, **keywords)
+  # A key column is the index here, not a result column.  Named columns
+  # come back as df[...] returns them, each on the frame's rows with
+  # +keep_rows+.
+  private def reduce_columns(reduction, names, keep_rows, **keywords)
+    keywords = keywords.compact
+    fills = keywords.delete(:fill_value)
+    if fills.is_a?(Hash)
+      unknown = fills.keys - @frame.column_names
+      unless unknown.empty?
+        raise ArgumentError,
+              "#{reduction}: fill_value names no column #{unknown.map(&:inspect).join(', ')}"
+      end
     end
-    CAFrame.new(cols, axis_name: @axis_name, index: label_index)
+    out = over_named_columns(reduction, names, @key_names) do |col, name|
+      fill = fills.is_a?(Hash) ? fills[name] : fills
+      kw = fill.nil? ? keywords : keywords.merge(fill_value: fill)
+      col = group_reduce(col, reduction, **kw)
+      keep_rows && !names.empty? ? spread_to_rows(col) : col
+    end
+    return out unless names.empty?
+    result_frame(out, keep_rows)
+  end
+
+  # One row per group, indexed by the labels; or, with +keep_rows+, the
+  # frame's rows, each holding its group's result.
+  private def result_frame(cols, keep_rows)
+    unless keep_rows
+      return CAFrame.new(cols, axis_name: @axis_name, index: label_index) unless @label_columns
+      clash = cols.keys & @label_columns.keys
+      unless clash.empty?
+        raise ArgumentError,
+              "a result column has the name of a key column: #{clash.map(&:inspect).join(', ')}"
+      end
+      return CAFrame.new(@label_columns.transform_values(&:copy).merge(cols))
+    end
+    CAFrame.new(cols.transform_values { |col| spread_to_rows(col) },
+                axis_name: @frame.axis_name, index: @frame.index)
+  end
+
+  # A per-group result (one entry per group along axis 0) laid back over the
+  # frame's rows through the group codes. A row in no group is UNDEF.
+  private def spread_to_rows(result)
+    # A row in no group has a masked index cell, which x[i] answers as a
+    # hole: so even with no group at all, the result keeps its type.
+    codes = group_codes
+    index = codes.value.int64
+    index[codes.is_masked] = UNDEF
+    result[index, *[nil] * (result.ndim - 1)].copy
   end
 
   # A column's group reduction. An N-D column is reduced along the rows, so
@@ -298,22 +759,46 @@ class GroupedFrame
     end
   end
 
-  # Whether the column's type defines the reduction, asked of the core on one
-  # cell of it so that the answer is about the type and not the data: a
-  # refusal there is the core saying the reduction does not apply, while an
-  # error in the grouped reduction itself is left to raise. A column with no
-  # rows is asked as it is: the refusal does not depend on the cells.
-  private def answers?(col, reduction)
-    (col.elements > 0 ? col[0..0] : col).public_send(reduction)
-    true
-  rescue CArray::DataTypeError, TypeError, ArgumentError
-    false
+  # The columns an aggregate block returned: a Hash of names to CArrays
+  # with one entry per group along their first axis.
+  private def block_columns(out)
+    unless out.is_a?(Hash)
+      raise ArgumentError, "aggregate block must return a Hash (got #{out.class})"
+    end
+    out.each_with_object({}) do |(name, col), cols|
+      name = @frame.__send__(:column_name_arg, name, "aggregate")
+      unless col.is_a?(CArray) && col.ndim >= 1 && col.shape[0] == ngroup
+        got = col.is_a?(CArray) ? "shape #{col.shape.inspect}" : col.class.to_s
+        raise ArgumentError,
+              "aggregate: #{name.inspect} must be a CArray with one value per group " \
+              "(#{ngroup}), got #{got}"
+      end
+      cols[name] = col
+    end
   end
 
   private def per_group_column(in_name, proc)
     results = []
     each_group_slice(in_name) { |slice| results << proc.call(slice) }
-    CArray.object(results.size) { |i| results[i] }
+    column_of_values(results)
+  end
+
+  # The grouping holds one code per row of the frame as it was when grouped.
+  # A frame whose row count changed since (a splice, a deletion) has rows
+  # the grouping does not know, so every verb refuses it.
+  private def refuse_changed_rows(verb)
+    return if @frame.nrow == @cat.elements
+    raise ArgumentError,
+          "#{verb}: the frame has #{@frame.nrow} rows, #{@cat.elements} when it " \
+          "was grouped; group it again"
+  end
+
+  # A column of the Ruby values the groups gave, typed as from_records types
+  # a record's values: Integers int64, numbers float64, Strings text, time
+  # elements a time column, arrays of one shape an N-D column, anything else
+  # object. UNDEF and nil are missing.
+  private def column_of_values(values)
+    CAFrame.__send__(:build_record_column, values, values.size)
   end
 
   private def each_group_slice(in_name)
@@ -327,6 +812,23 @@ class GroupedFrame
     ngroup.times { |k| yield @frame[group_address(k)] }
   end
 
+  # The first row of each group, in code order.
+  # With few groups each row's number is folded into its group's slot by the
+  # smallest, which needs no sort of the rows (a numeric column then reduces
+  # without one too); a row in no group (masked code) is in no slot, and a
+  # group without rows keeps no row (UNDEF). With more groups the columns
+  # are gathered through the sort anyway, so the first rows are read off it.
+  private def first_rows
+    if ngroup > CACategoricalIterator::SCATTER_CATEGORIES_MAX
+      return group_perm[group_bounds[0...ngroup]]
+    end
+    n     = @cat.elements
+    first = CArray.int64(ngroup).fill(n)
+    first.scatter_min!(@cat.codes, CArray.int64(n).seq!)
+    first[first.eq(n)] = UNDEF if ngroup > 0
+    first
+  end
+
   private def group_address(k)
     group_perm[group_bounds[k]...group_bounds[k + 1]]
   end
@@ -336,7 +838,7 @@ class GroupedFrame
   end
 
   private def group_perm
-    @group_perm ||= @cat.sort_addr
+    @group_perm ||= @cat.group_addr
   end
 
   private def group_bounds