carray 3.0.1 → 3.0.3

data/lib/carray/frame/describe.rb

@@ -0,0 +1,79 @@
+# CAFrame#describe: one row of statistics per column, for a first look at a
+# table that has just been read.
+
+class CAFrame
+  # Summarize the columns, one row each, as a new frame indexed by column
+  # name:
+  #
+  #   puts df.describe.to_table
+  #
+  #   column   type     count  masked  unique  min         max         mean        stddev
+  #   -------  -------  -----  ------  ------  ----------  ----------  ----------  --------
+  #   station  object       3       0       2  _           _           _           _
+  #   temp     float64      2       1       2  19.0        22.1        20.55       2.192031
+  #   time     CATime       3       0       3  2024-01-01  2024-01-03  2024-01-02  1D
+  #
+  # +type+ is the data type, or the Face's class for a Face column, with the
+  # trailing shape of an N-D column in brackets (+float64[3]+). +count+ and
+  # +masked+ count cells, so an N-D column counts every cell of every row;
+  # +unique+ is the number of distinct values among the present cells.
+  #
+  # The other statistics apply only where the column's kind gives them a
+  # meaning, and are UNDEF elsewhere:
+  #
+  #   real numbers, boolean   min  max  mean  stddev
+  #   complex                           mean  stddev   (no order)
+  #   CATime, CATimedelta     min  max  mean  stddev   (as times / durations)
+  #   anything else (text, categorical, records ...)   unique only
+  #
+  # A boolean column counts true as 1, so its mean is the share of trues. A
+  # column with no present cell has UNDEF statistics. The index is not a
+  # column and is not summarized.
+  #
+  # @param names [Array<String>] the columns to summarize; default all.
+  # @return [CAFrame] a new frame with one row per column.
+  def describe(*names)
+    names = names.empty? ? column_names : names.map(&:to_s)
+    rows = names.map { |name| describe_column(self[name]) }
+    stats = %w[type count masked unique min max mean stddev]
+    cols = stats.to_h do |stat|
+      values = rows.map { |row| row[stat] }
+      column =
+        case stat
+        when "count", "masked", "unique" then CArray.int64(values.size)
+        else CArray.object(values.size)
+        end
+      column[] = UNDEF
+      values.each_with_index { |v, i| column[i] = v unless UNDEF.equal?(v) }
+      [stat, column]
+    end
+    CAFrame.new(cols, axis_name: "column", index: CA_OBJECT(names))
+  end
+
+  DESCRIBE_TIME_FACES = %w[CATime CATimedelta].freeze
+  private_constant :DESCRIBE_TIME_FACES
+
+  private def describe_column(col)
+    kind =
+      if col.face?
+        DESCRIBE_TIME_FACES.include?(col.class.name) ? :ordered : :other
+      elsif col.complex?
+        :complex
+      elsif col.numeric? || col.boolean?
+        :ordered
+      else
+        :other
+      end
+    type = col.face? ? col.class.name : col.data_type_name
+    type += col.shape[1..].inspect if col.ndim > 1
+    row = { "type" => type,
+            "count" => col.count_not_masked, "masked" => col.count_masked,
+            "unique" => col.nunique }
+    ordered = kind == :ordered
+    row["min"]    = ordered ? col.min : UNDEF
+    row["max"]    = ordered ? col.max : UNDEF
+    row["mean"]   = kind == :other ? UNDEF : col.mean
+    row["stddev"] = kind == :other ? UNDEF : col.stddev
+    row
+  end
+end

data/lib/carray/frame/frame.rb

@@ -11,6 +11,29 @@
 # storage, and +copy+ is the way to an independent frame.
 
 class CAFrame
+  # Raised when the contents of a column cannot be read as asked: a cell
+  # that does not read as the type a +cast+ (or +from_csv+'s +types:+) asks
+  # for under on_error: :raise, a cell +parse_to_time+ cannot read, a column
+  # whose time format +infer_time_format+ / parse_to_time(name, :infer)
+  # cannot find, or a float column +to_time+ cannot read as whole counts.
+  # The message names the column and the cell. The arguments were fine; the
+  # column's contents are what did not read.
+  class UnreadableColumn < StandardError; end
+
+  # Raised when +infer_time_format+ / parse_to_time(name, :infer) finds more
+  # than one format every cell fits ("01/02/2024" and "03/04/2024" are both
+  # day-first and month-first). The text reads in each of them, so it is
+  # not an UnreadableColumn; +formats+ gives the candidates, to choose one and
+  # pass it as the format.
+  class AmbiguousTimeFormat < StandardError
+    attr_reader :formats
+
+    def initialize(message = nil, formats: [])
+      @formats = formats
+      super(message)
+    end
+  end
+
   # Integer data_types that select rows positionally when used as a df[] key.
   INTEGER_TYPES = [:int8, :int16, :int32, :int64,
                    :uint8, :uint16, :uint32, :uint64].freeze
@@ -45,14 +68,21 @@ class CAFrame
     @columns   = {}
     @axis_name = axis_name || DEFAULT_AXIS_NAME
     @index     = nil
+    # The row axis name this frame had before set_index promoted a column over
+    # it, so reset_index can put it back.  A frame built with an index never
+    # had one, and neither does a frame derived from an indexed one: both are
+    # born indexed, so there is nothing to restore and the default stands.
+    @axis_name_before_index = nil
 
     n = nil
+    n_from = nil
     columns.each do |name, col|
       key = name.to_s
       ca  = coerce_column(col)
       len = ca.shape[0]
       if n.nil?
-        n = len
+        n      = len
+        n_from = key
       elsif len != n
         raise ArgumentError,
               "column #{key.inspect} has axis-0 length #{len}, expected #{n}"
@@ -67,8 +97,12 @@ class CAFrame
         raise ArgumentError, "index must be a 1-D column (got ndim #{idx.ndim})"
       end
       if n && idx.shape[0] != n
+        # Name the column as well as the index: when the index is the frame's
+        # existing one and a column is the new arrival, blaming the index alone
+        # points at the side the caller cannot change.
         raise ArgumentError,
-              "index length #{idx.shape[0]} does not match nrow #{n}"
+              "column #{n_from.inspect} has axis-0 length #{n}, " \
+              "but the index has length #{idx.shape[0]}"
       end
       @index = idx
       @nrow  = idx.shape[0] if n.nil?
@@ -101,7 +135,7 @@ class CAFrame
     if keys.size > 1
       unless keys.all? { |k| k.is_a?(String) }
         raise ArgumentError,
-              "multi-key df[...] escapes columns; every key must be a String " \
+              "df[...] with several keys returns columns; every key must be a String " \
               "(use df.select(...) for a subset frame)"
       end
       return keys.map { |k| @columns.fetch(k) { raise KeyError, "no column #{k.inspect}" } }
@@ -131,7 +165,7 @@ class CAFrame
               "CArray df[] key must be boolean or integer (got #{key.data_type})"
       end
     when Symbol
-      raise NotImplementedError, "df[symbol] is reserved for predicate keys"
+      raise ArgumentError, "df[symbol] is reserved for predicate keys"
     else
       raise ArgumentError, "unsupported df[] key: #{key.class}"
     end
@@ -186,6 +220,7 @@ class CAFrame
   # write path always propagates to derived views (unchanged from
   # previous behaviour).
   def []=(*keys, value)
+    refuse_if_frozen
     if keys.size != 1
       raise ArgumentError,
             "df[...] = takes one key (a column name or a row selector); " \
@@ -211,6 +246,52 @@ class CAFrame
     value
   end
 
+  # Freezing a frame fixes its columns, rows and index; the columns' cells are
+  # their own arrays' to guard, as an Array's elements are.  The check comes
+  # first so a refused edit leaves the frame as it was: the edits change the
+  # column table before any instance variable.
+  private def refuse_if_frozen
+    raise FrozenError.new("can't modify frozen CAFrame: #{inspect}", receiver: self) if frozen?
+  end
+
+  # Make this frame read-only and return it.  It is +freeze+ extended to the
+  # cells: the column set, the rows and the index are fixed as by +freeze+,
+  # and the cells and masks of every column and of the index refuse writes
+  # made through this frame.
+  #
+  # Each column (and the index) is replaced by a read-only view of itself, so
+  # the arrays the frame was built from are left as they were: whoever holds
+  # them can still write, and the writes show through this frame.  Frames
+  # derived from a protected frame by views (+select+, +filter+, a row slice)
+  # read through those views and refuse cell writes too, but are not frozen.
+  # +copy+ gives a frame that can be written again.
+  #
+  # Calling it on a protected frame returns the frame unchanged.  A frame
+  # frozen with +freeze+ cannot be protected afterwards, because its columns
+  # can no longer be replaced: raises +FrozenError+.
+  def protect
+    return self if protected?
+    refuse_if_frozen
+    @columns = @columns.transform_values { |col| read_only_view(col) }
+    @index   = read_only_view(@index) if @index
+    @protected = true
+    freeze
+  end
+
+  # True if this frame was made read-only by +protect+.  A +dup+ is not frozen
+  # and so not protected, though its columns stay the read-only views it
+  # shares; a +clone+ keeps both.
+  def protected?
+    frozen? && @protected == true
+  end
+
+  private def read_only_view(col)
+    return col if col.read_only?
+    view = col.refer
+    view.set_read_only_flag
+    view
+  end
+
   # df["name"] = ... : the column forms of []=.  Unlike the verbs, +[]=+ can
   # only ever mutate the receiver -- Ruby hands the right-hand side back as
   # the value of an assignment, so there is no way to return a new frame.
@@ -227,16 +308,17 @@ class CAFrame
       mask_column(key)
     elsif value.is_a?(CAFrame)
       raise ArgumentError,
-            "df[name] = takes a column, not a CAFrame; escape one with " \
+            "df[name] = takes a column, not a CAFrame; take one with " \
             "other[\"name\"], or splice rows with df[rows] = other"
     else
       rebind_column(key, value)
     end
   end
 
-  # Bind +key+ to +column+, adding the name when it is new.  This is the one
-  # place a column enters an existing frame, so it is where the axis-0 length
-  # invariant is enforced (§12-A).  It is a replacement rather than an edit,
+  # Bind +key+ to +column+, adding the name when it is new.  This is where a
+  # column arriving from outside the frame has its axis-0 length checked
+  # (§12-A); the conversions in verbs.rb rebind +@columns+ directly and rely on
+  # being length-preserving instead.  It is a replacement rather than an edit,
   # which is what sets it apart from the rest: +fill+ / +mask_eq+ /
   # +df[rows] = UNDEF+ write to the shared column and are therefore visible
   # wherever it is held, while this binds the name to a different column and
@@ -307,15 +389,21 @@ class CAFrame
   # value kept, so the row stays identifiable), leaving a later, better-informed
   # pass to re-judge them. Definitely-true rows carry their values through
   # unchanged in both modes.
+  #
+  # The default is a view-frame. +keep_masked: true+ returns a **materialized**
+  # frame instead -- carrying the UNDEF forward means writing it into the
+  # result, which a view cannot do without masking the parent's rows. That holds
+  # whether or not the selector actually carries a masked cell, so the same call
+  # site does not switch between sharing and copying with the data.
   def filter(keep_masked: false)
     mask = yield(self)
     unless mask.is_a?(CArray) && mask.data_type == :boolean
       raise ArgumentError,
             "filter block must return a boolean CArray (got #{mask.class})"
     end
-    unless mask.shape[0] == @nrow
+    unless mask.shape == [@nrow]
       raise ArgumentError,
-            "filter mask has axis-0 length #{mask.shape[0]}, expected nrow #{@nrow}"
+            "filter mask has shape #{mask.shape.inspect}, expected [#{@nrow}] (one value per row)"
     end
     select_rows(mask, keep_masked: keep_masked)
   end
@@ -324,13 +412,25 @@ class CAFrame
   # An index-role change: the data is unchanged, so this mutates self and
   # returns it (memo §3.8). Any existing index is replaced.
   def set_index(name)
+    refuse_if_frozen
     key = name.to_s
     raise KeyError, "no column #{key.inspect}" unless @columns.key?(key)
     idx = @columns[key]
     unless idx.ndim == 1
       raise ArgumentError, "index must be a 1-D column (got ndim #{idx.ndim})"
     end
+    # Only the name held before the frame had an index: a second set_index must
+    # not overwrite it with the first index's name.
+    @axis_name_before_index = @axis_name if @index.nil?
     @columns.delete(key)
+    if @index
+      # Re-indexing replaces the index, so the one being replaced goes back to
+      # being a column -- the same demotion reset_index performs, in the same
+      # position, which makes set_index over an existing index equivalent to
+      # reset_index followed by set_index.  Dropping it would lose the column it
+      # was made from.
+      @columns = { @axis_name => @index }.merge(@columns)
+    end
     @index     = idx
     @axis_name = key
     @nrow      = idx.shape[0]
@@ -341,10 +441,12 @@ class CAFrame
   # Also an index-role change, so it mutates self (memo §3.8). The former
   # index becomes the first column, named after the row axis.
   def reset_index
+    refuse_if_frozen
     return self unless @index
     @columns   = { @axis_name => @index }.merge(@columns)
     @index     = nil
-    @axis_name = DEFAULT_AXIS_NAME
+    @axis_name = @axis_name_before_index || DEFAULT_AXIS_NAME
+    @axis_name_before_index = nil
     self
   end
 
@@ -364,22 +466,50 @@ class CAFrame
 
   # The single row whose index label equals +label+, as a Ruby Hash (memo
   # §13.2b). +label+ is matched exactly against the index (+index.eq+), so any
-  # orderable / object / datetime / categorical index works. The return type is
+  # orderable / object / datetime / categorical index works, and +label+ is
+  # whatever +index.eq+ takes -- for a time index, a Time, Date or a String
+  # such as "2024-01-02".  A NaN label finds the NaN label, as join and
+  # group_by match NaN keys.  A numeric index takes a number; a String or
+  # true / false raises. The return type is
   # a row Hash and stays that way: zero matches raise KeyError, and duplicate
   # labels raise (go through +filter { |f| f.index.eq(label) }+ for the
   # multi-row, frame-returning path). Positional access is +df[i]+.
   def at(label)
     raise ArgumentError, "at requires an index (set one with set_index)" unless @index
-    pos = @index.eq(label).where
+    # An index may hold a masked cell -- an outer join and align both build one
+    # -- but an undefined label identifies no row, and two undefined labels are
+    # not the same label. The addressing primitives already say this (a masked
+    # key matches nothing, not another masked key), so at says it too rather
+    # than answering through eq(UNDEF), which asks about the mask and not about
+    # the label at all.
+    if UNDEF.equal?(label)
+      raise ArgumentError,
+            "at(UNDEF): an undefined label identifies no row; " \
+            "use filter { |f| f.index.is_masked } for the rows with no label"
+    end
+    # A NaN label is the NaN value, as join and group_by take it: eq answers
+    # false for NaN against anything, itself included.
+    # A numeric index is looked up with a number: a String or true/false would
+    # be read differently by each numeric type, or not at all.
+    if !@index.face? && (@index.integer? || @index.float? || @index.complex?) &&
+       !label.is_a?(Numeric)
+      raise ArgumentError,
+            "at: the index is #{@index.data_type}, so the label must be a number " \
+            "(got #{label.inspect}); convert it first, e.g. at(Float(s)) or at(Integer(s))"
+    end
+    nan = label.is_a?(Float) && label.nan?
+    hit = nan ? @index.is_in([label]) : @index.eq(label)
+    pos = hit.where
     case pos.elements
     when 0
       raise KeyError, "no row with index label #{label.inspect}"
     when 1
       row(pos[0])
     else
+      test = nan ? "f.index.is_nan" : "f.index.eq(label)"
       raise ArgumentError,
             "index label #{label.inspect} matches #{pos.elements} rows; " \
-            "use filter { |f| f.index.eq(label) } for duplicate labels"
+            "use filter { |f| #{test} } for duplicate labels"
     end
   end
 
@@ -403,20 +533,20 @@ class CAFrame
   # --- metadata readers (memo §13.2) ------------------------------------
   # Each returns a fresh object; the live columns Hash is never exposed.
 
-  # Variable (column) names in column order.  Returns +Array<String>+.
-  def variable_names
+  # Column names in column order.  Returns +Array<String>+.
+  def column_names
     @columns.keys
   end
 
-  # Variables (raw column CArrays) in column order.  Returns
-  # +Array<CArray>+.  Equivalent to +df[*variable_names]+ but built
-  # directly from the columns Hash.
-  def variables
+  # The columns (raw CArrays) in column order.  Returns +Array<CArray>+.
+  # Equivalent to +df[*column_names]+ but built directly from the columns
+  # Hash.
+  def columns
     @columns.values
   end
 
-  # Number of variables (columns).
-  def nvar
+  # Number of columns.
+  def ncol
     @columns.size
   end
 
@@ -444,7 +574,7 @@ class CAFrame
     if col.is_a?(CAFrame)
       raise ArgumentError,
             "a CAFrame cannot be a column (it answers to_ca as a 2-D matrix); " \
-            "escape a column with df[\"name\"] or df.to_ca first"
+            "take a column with df[\"name\"] or df.to_ca first"
     end
     unless col.respond_to?(:to_ca)
       raise ArgumentError, "column must be a CArray or Array (got #{col.class})"
@@ -457,10 +587,11 @@ class CAFrame
   end
 
   private def row(i)
-    i += @nrow if i < 0
-    unless i >= 0 && i < @nrow
+    pos = i < 0 ? i + @nrow : i
+    unless pos >= 0 && pos < @nrow
       raise IndexError, "row index #{i} out of range (nrow=#{@nrow})"
     end
+    i = pos
     h = {}
     h[@axis_name] = elem_at(@index, i) if @index
     @columns.each { |name, col| h[name] = elem_at(col, i) }
@@ -483,6 +614,19 @@ class CAFrame
   # and index are untouched; the selected cells of every column go to UNDEF.
   # The selector is forwarded to the column indexer, which classifies it.
   private def mask_rows(selector)
+    # Decide before changing anything. A read-only column (a categorical, whose
+    # codes are read-only) refuses the write, and refusing half way through
+    # would leave the frame masked in whichever columns happened to come first.
+    refusing = @columns.find { |_, col| col.read_only? }
+    if refusing
+      raise "can not modify read-only array: column #{refusing[0].inspect} " \
+            "refuses the write, so no column was masked"
+    end
+    # The selector is classified by the column indexer, which is also what
+    # bound-checks it -- so with no column to forward it to, nothing would.
+    # Build the row mask purely to have the frame's own row axis refuse an
+    # out-of-range row, as every other row form does.
+    selected_row_mask(selector) if @columns.empty?
     @columns.each_value do |col|
       col[selector, *([nil] * (col.ndim - 1))] = UNDEF
     end
@@ -508,10 +652,10 @@ class CAFrame
   # promote and the row count shifts by other.nrow - span.
   private def splice_rows(selector, other)
     lo, hi = contiguous_span(selector)
-    unless other.variable_names.sort == variable_names.sort
+    unless other.column_names.sort == column_names.sort
       raise ArgumentError,
-            "splice frame has columns #{other.variable_names.inspect}, " \
-            "expected the same set as #{variable_names.inspect}"
+            "splice frame has columns #{other.column_names.inspect}, " \
+            "expected the same set as #{column_names.inspect}"
     end
     new_cols = {}
     @columns.each do |name, col|
@@ -540,7 +684,16 @@ class CAFrame
     new_index = splice_index(other, lo, hi)
     @columns  = new_cols
     @index    = new_index
-    @nrow     = @nrow - (hi - lo) + other.nrow
+    # Read the new count off something the frame now holds, rather than
+    # computing it. For every column the pieces welded above are head (lo) +
+    # other's rows + tail (nrow - hi), so the arithmetic and the objects agree
+    # -- except when there is nothing to weld: a frame with no columns and no
+    # index would otherwise come away claiming rows that nothing backs.
+    @nrow = if (witness = new_cols.each_value.first || new_index)
+              witness.shape[0]
+            else
+              0
+            end
     self
   end
 
@@ -609,8 +762,11 @@ class CAFrame
   end
 
   private def select_rows(selector, keep_masked: false)
-    if keep_masked && selector.is_a?(CArray) &&
-       selector.data_type == :boolean && selector.has_mask?
+    if keep_masked && selector.is_a?(CArray) && selector.data_type == :boolean
+      # Not gated on selector.has_mask?: gating there would make the same call
+      # site return a view-frame or a materialized one depending on whether that
+      # run's data happened to produce an undetermined cell (the reason splice
+      # always snapshots -- see CAFrame.md section 3).
       return select_rows_keep_masked(selector)
     end
     cols = {}
@@ -625,13 +781,27 @@ class CAFrame
     keep       = selector.strip_mask(1)   # true where selected OR undetermined
     undet_kept = selector.is_masked[keep] # among kept rows, which were undetermined
     cols = {}
+    any_undet = undet_kept.any
     @columns.each do |name, col|
       tail = [nil] * (col.ndim - 1)
-      g = col[keep, *tail].copy
-      g[undet_kept, *tail] = UNDEF
-      cols[name] = g
+      g = col[keep, *tail]
+      if any_undet
+        # mask_where copies and masks in one step, so a read-only Face
+        # (categorical, CAConstString) takes the mask as any column does.
+        cells = CArray.boolean(*g.shape)
+        cells[] = false
+        cells[undet_kept, *tail] = true
+        cols[name] = g.mask_where(cells)
+      else
+        cols[name] = g.copy
+      end
     end
-    new_index = @index ? @index[keep] : nil
+    # The index is copied along with the columns. Its values are carried over
+    # unmasked (an undetermined row keeps its label, as mask_rows leaves the
+    # index alone), but a materialized frame has to be materialized whole: an
+    # aliased index would write through to the parent from a frame whose columns
+    # do not.
+    new_index = @index ? @index[keep].copy : nil
     CAFrame.new(cols, axis_name: @axis_name, index: new_index)
   end