carray 3.0.2 → 3.0.4

data/lib/carray/frame/column_verbs.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+class CAFrame
+
+  # The one rule of a frame-wide verb over the frame's columns -- the group
+  # reductions and transforms, the windowed reductions -- kept in one place
+  # so that every verb answers in the same shape:
+  #
+  # - given names, the CArray (one) or the Array of them (several), as
+  #   +df[...]+ returns columns; a named column whose type has no such
+  #   operation is refused with CArray::DataTypeError, as the core refuses
+  #   it, since it was asked for;
+  # - given none, every column whose type has it, the key columns left out,
+  #   as a Hash of name to result that the caller makes a frame of.
+  #
+  # Whether a type has the operation is asked of the core on two copies of
+  # the column's first row (two blank cells of its kind when it has no
+  # rows), so the answer is about the type and not the data: a refusal
+  # there means the operation does not apply, while an error in the verb
+  # itself is left to raise. An N-D column (each row a profile) is asked,
+  # and computed, along the rows.
+  module ColumnVerbs
+
+    private
+
+    def over_named_columns (verb, names, key_names, args = [], keywords = {})
+      if names.empty?
+        cols = {}
+        @frame.column_names.each do |name|
+          next if key_names.include?(name)
+          col = @frame[name]
+          next unless column_has?(col, verb, args, keywords)
+          cols[name] = yield(col)
+        end
+        return cols
+      end
+      results = names.map do |name|
+        col = @frame[@frame.__send__(:column_key, name, verb)]
+        unless column_has?(col, verb, args, keywords)
+          raise CArray::DataTypeError,
+                "#{verb}: column #{name.inspect} " \
+                "(#{CArray::Inspector.type_label(col)}) has no #{verb}"
+        end
+        yield(col)
+      end
+      results.size == 1 ? results.first : results
+    end
+
+    # count with no value counts the present cells, which every type has;
+    # a Face's own count takes a value (CACategorical#count(label)), so the
+    # type is asked about count_not_masked instead.
+    def column_has? (col, verb, args, keywords)
+      verb = :count_not_masked if verb == :count && args.empty?
+      # Two cells: one cell does not reach what combines two (a String's
+      # median interpolates, its sum adds), and an empty column answers
+      # with the identity without asking the type at all.
+      tail = [nil] * (col.ndim - 1)
+      probe = if col.shape[0] > 0
+                col[CA_INT64([0, 0]), *tail]
+              else
+                col.resize(2, *col.shape[1..])
+              end
+      keywords = keywords.merge(axis: 0) if col.ndim > 1
+      probe.public_send(verb, *args, **keywords)
+      true
+    rescue CArray::DataTypeError, TypeError, ArgumentError
+      false
+    end
+  end
+end

data/lib/carray/frame/concat.rb

@@ -9,6 +9,9 @@
 #
 # Both are class methods (symmetric N-ary; no frame is privileged), like
 # +CArray.meld+ / +CArray.concatenate+, not instance verbs.
+#
+# +CAFrame.stack+ and +CAFrame#unstack+ add and take away a layer axis
+# instead of rows.
 
 class CAFrame
   # Weld frames along the row axis, view-style.  Each output column is
@@ -28,6 +31,13 @@ class CAFrame
   #
   # The index is welded too when every frame has one (their +axis_name+
   # must agree); if none do, the result has no index; a mix raises.
+  #
+  # A frame with no rows adds no values, so it takes no part in the result
+  # past these checks: its column set, its N-D columns' trailing dimensions
+  # and its index still have to agree, but the
+  # data types are the other frames' (a header-only CSV reads as object
+  # columns, which would otherwise refuse or demote the rest).  When every
+  # frame is empty, the first one stands for them.
   # Column-set mismatch raises — a union-with-UNDEF mode is a possible
   # future opt-in, kept out here to stay explicit (memo §4.2).
   #
@@ -37,8 +47,11 @@ class CAFrame
     frames = frames.flatten
     check_concat_inputs(frames, verb: "meld")
     first = frames.first
-    names = first.variable_names
+    names = first.column_names
     check_column_sets(frames, names, verb: "meld")
+    index_pieces(frames, verb: "meld")
+    check_trailing_shapes(frames, names, verb: "meld")
+    frames = frames_with_rows(frames)
     cols = {}
     names.each do |name|
       cols[name] = CArray.meld(frames.map { |f| f[name] }, axis: 0)
@@ -54,8 +67,9 @@ class CAFrame
   # For a view frame that shares storage with the inputs (strict same data type
   # per column, chain composability preserved) use {CAFrame.meld}.
   #
-  # Column matching, index handling, and column-set / index-mix rules
-  # match {CAFrame.meld}.
+  # Column matching, index handling, column-set / index-mix rules and the
+  # treatment of a frame with no rows match {CAFrame.meld}: an empty frame's
+  # data types take no part in the common type.
   #
   #   CAFrame.concatenate(jan, feb, mar)     # eager, independent result
   #   CAFrame.concatenate([jan, feb, mar])   # an Array is accepted too
@@ -63,8 +77,11 @@ class CAFrame
     frames = frames.flatten
     check_concat_inputs(frames, verb: "concatenate")
     first = frames.first
-    names = first.variable_names
+    names = first.column_names
     check_column_sets(frames, names, verb: "concatenate")
+    index_pieces(frames, verb: "concatenate")
+    check_trailing_shapes(frames, names, verb: "concatenate")
+    frames = frames_with_rows(frames)
     cols = {}
     names.each do |name|
       cols[name] = CArray.concatenate(frames.map { |f| f[name] })
@@ -72,6 +89,108 @@ class CAFrame
     new(cols, axis_name: first.axis_name, index: concatenate_index(frames))
   end
 
+  # Stack frames of the same shape as layers.  Each output column is
+  # +CArray.stack+ of that column across the frames along +axis+, counted in
+  # the column's own axes (0 is the row axis, so +axis+ starts at 1): a
+  # scalar column of K frames becomes a column of shape (nrow, K).  The rows
+  # stay as they are, so row verbs work on the stack unchanged and a layer is
+  # a column axis -- +s["temp"][nil, k]+ is frame k's column, and
+  # +s["temp"].mean(axis: 1)+ reduces across the layers.
+  #
+  # The result is a view frame: its columns are CAStack views over the
+  # frames' columns, and writes reach them.  Call +copy+ for an independent
+  # frame.  The layers carry no labels; keep them alongside (the dates of the
+  # files, say) as an array of your own.
+  #
+  # Every frame must have the same columns, the same number of rows and the
+  # same index (by value) under the same axis name, or none; the result
+  # takes the first frame's.  +unstack+ is the inverse:
+  #
+  #   s = CAFrame.stack(jan1, jan2, jan3)    # temp:float64 -> temp:float64[3]
+  #   CAFrame.stack(*s.unstack(axis: 1))       # the same frame again
+  def self.stack(*frames, axis: 1)
+    frames = frames.flatten
+    check_concat_inputs(frames, verb: "stack")
+    check_layer_axis(axis, "stack")
+    first = frames.first
+    names = first.column_names
+    check_column_sets(frames, names, verb: "stack")
+    frames.each_with_index do |f, i|
+      next if i.zero?
+      unless f.nrow == first.nrow
+        raise ArgumentError,
+              "stack: frame #{i} has #{f.nrow} rows, frame 0 has #{first.nrow}"
+      end
+    end
+    indexes, = index_pieces(frames, verb: "stack")
+    if indexes && ! indexes.all? { |x| same_index_values?(x, indexes.first) }
+      raise ArgumentError, "stack: the frames' indexes differ; stacked rows have to be the same rows"
+    end
+    cols = {}
+    names.each do |name|
+      parts = frames.map { |f| f[name] }
+      if axis > parts.first.ndim
+        raise ArgumentError,
+              "stack: axis #{axis} out of range for column #{name.inspect} " \
+              "(a #{parts.first.ndim}-D column takes 1 to #{parts.first.ndim})"
+      end
+      cols[name] = CArray.stack(parts, axis: axis)
+    end
+    new(cols, axis_name: first.axis_name, index: first.index)
+  end
+
+  # Split the frame along a column axis into an Array of frames, one per
+  # position on that axis -- the inverse of {CAFrame.stack}.  +axis+ counts
+  # the columns' own axes (0 is the row axis, so it starts at 1).  Every
+  # column must have that axis, and the same length along it: a column
+  # without it has no layer to give each frame.  Each frame's columns are
+  # views of this frame's, and each has this frame's index.
+  #
+  #   s.unstack(axis: 1)    # => [frame of layer 0, frame of layer 1, ...]
+  def unstack(axis:)
+    self.class.send(:check_layer_axis, axis, "unstack")
+    if @columns.empty?
+      raise ArgumentError, "unstack: the frame has no columns to unstack"
+    end
+    length = nil
+    @columns.each do |name, col|
+      unless axis < col.ndim
+        raise ArgumentError,
+              "unstack: column #{name.inspect} is #{col.ndim}-D and has no axis #{axis}"
+      end
+      length ||= col.shape[axis]
+      unless col.shape[axis] == length
+        raise ArgumentError,
+              "unstack: column #{name.inspect} has #{col.shape[axis]} along axis #{axis}, " \
+              "the others #{length}"
+      end
+    end
+    (0...length).map do |k|
+      cols = @columns.to_h do |name, col|
+        idx = [nil] * col.ndim
+        idx[axis] = k
+        [name, col[*idx]]
+      end
+      CAFrame.new(cols, axis_name: @axis_name, index: @index)
+    end
+  end
+
+  def self.check_layer_axis(axis, verb)
+    unless axis.is_a?(Integer)
+      raise TypeError, "#{verb}: axis must be an Integer (got #{axis.inspect})"
+    end
+    if axis == 0
+      raise ArgumentError,
+            "#{verb}: axis 0 is the row axis; rows are joined by meld or concatenate"
+    end
+    if axis < 0
+      raise ArgumentError,
+            "#{verb}: axis counts a column's axes from 1 (got #{axis}); " \
+            "a negative axis would mean a different axis in columns of different ndim"
+    end
+  end
+  private_class_method :check_layer_axis
+
   # ---- shared validators -------------------------------------------------
 
   def self.check_concat_inputs(frames, verb:)
@@ -86,15 +205,47 @@ class CAFrame
     expected = names.sort
     frames.each_with_index do |f, i|
       next if i.zero?
-      if f.variable_names.sort != expected
+      if f.column_names.sort != expected
         raise ArgumentError,
-              "#{verb}: frame #{i} has columns #{f.variable_names.inspect}, " \
+              "#{verb}: frame #{i} has columns #{f.column_names.inspect}, " \
               "expected the same set as #{names.inspect}"
       end
     end
   end
   private_class_method :check_column_sets
 
+  # A column's trailing dimensions are its shape, not a type the other
+  # frames can settle, so a frame with no rows has to agree on them too.
+  def self.check_trailing_shapes(frames, names, verb:)
+    names.each do |name|
+      trailing = frames.first[name].shape[1..]
+      frames.each_with_index do |f, i|
+        next if i.zero? || f[name].shape[1..] == trailing
+        raise ArgumentError,
+              "#{verb}: column #{name.inspect} has trailing dimensions " \
+              "#{f[name].shape[1..].inspect} in frame #{i}, #{trailing.inspect} in frame 0"
+      end
+    end
+  end
+  private_class_method :check_trailing_shapes
+
+  # Two indexes hold the same rows when their values agree cell by cell; a
+  # NaN label matches a NaN label, and a masked cell a masked cell.
+  def self.same_index_values?(a, b)
+    return false unless a.shape == b.shape
+    a.to_a.zip(b.to_a).all? do |x, y|
+      x.equal?(y) || x == y || (x.is_a?(Float) && y.is_a?(Float) && x.nan? && y.nan?)
+    end
+  end
+  private_class_method :same_index_values?
+
+  # The frames that add rows, or the first frame when none does.
+  def self.frames_with_rows(frames)
+    with_rows = frames.reject { |f| f.nrow.zero? }
+    with_rows.empty? ? [frames.first] : with_rows
+  end
+  private_class_method :frames_with_rows
+
   # ---- index helpers -----------------------------------------------------
 
   # Index welded via CArray.meld (view).  Every frame must agree: all

data/lib/carray/frame/convert.rb

@@ -16,21 +16,25 @@ class CAFrame
   # column name => value; a scalar cell is a Ruby value, an N-D cell is a Ruby
   # Array, and a masked cell (UNDEF) is +nil+. Normalizing UNDEF -> nil and
   # CArray -> Array (unlike +each_row+, which yields the raw view with UNDEF /
-  # CArray slices) is what makes it round-trip: +from_records(df.to_records)+
-  # rebuilds the same typing, and the result is JSON-serializable.
+  # CArray slices) makes the result JSON-serializable and lets
+  # +from_records(df.to_records)+ give back the column names, the values and
+  # the mask. The rest is not in a record: +from_records+ rebuilds each type
+  # from the values (an int32 comes back int64, a time column as an object
+  # column of its elements), the index comes back as an ordinary first
+  # column, and a frame with no rows gives +[]+, which has no column names.
   def to_records
     each_row.map { |row| row.transform_values { |v| record_value(v) } }
   end
 
   # Hand the frame over as a CArray with the minimum work (memo §11.9): a
-  # 2-D **view** of shape (nrow, nvar), one column per variable in column
+  # 2-D **view** of shape (nrow, ncol), one per column in column
   # order (§12-C). Nothing is materialised — the result is a +CAStack+ over
   # the stored columns, so reads gather from them and writes flow back
   # (§3.6). Call +copy+ on it for an independent, owned matrix; +CArray.tabulate+
   # is the eager sibling that builds one directly.
   #
   # Only same-shape scalar (1-D) columns qualify. An N-D column has no single
-  # matrix form and raises — escape it per column with +df["name"]+. A mixed
+  # matrix form and raises — take it per column with +df["name"]+. A mixed
   # data type set is promoted to a common type (+result_type+, §12-F) through
   # lazy cast lanes, so the promotion costs no buffer either.
   #
@@ -48,12 +52,12 @@ class CAFrame
     if nd
       raise ArgumentError,
             "to_ca needs all-scalar (1-D) columns; #{nd.first.inspect} is " \
-            "#{nd.last.ndim}-D — escape per column with df[name]"
+            "#{nd.last.ndim}-D — take it per column with df[name]"
     end
     refuse_unshared_columns(cols) if writable
     begin
       CArray.stack(cols, axis: 1)
-    rescue ArgumentError, RuntimeError => e
+    rescue ArgumentError, TypeError, CArray::DataTypeError => e
       # The columns have no common type (a text or Face column beside a
       # numeric one). Point at the frame-level verb that gives them one.
       raise e.class,
@@ -87,9 +91,80 @@ class CAFrame
     if UNDEF.equal?(v)
       nil
     elsif v.is_a?(CArray)
-      v.to_a.map { |e| UNDEF.equal?(e) ? nil : e }
+      record_value(v.to_a)
+    elsif v.is_a?(Array)
+      v.map { |e| record_value(e) }
     else
       v
     end
   end
 end
+
+class CArray
+  # @overload to_frame(*names, axis: -1)
+  #   Shows this array as a table: a CAFrame whose rows run along axis 0 and
+  #   whose columns are the positions along `axis`. Column `j` is the view of
+  #   `self` with `j` at `axis` and every other axis whole, so its shape is
+  #   `self`'s shape without `axis`. A 2-D array gives 1-D columns, an array
+  #   of more dimensions gives N-D columns:
+  #
+  #   ```
+  #   a = CArray.float64(3, 2).seq!
+  #   df = a.to_frame("x", "y")
+  #   df["y"]          # => a[nil, 1], shape [3]
+  #
+  #   img = CArray.uint8(480, 640, 3)
+  #   img.to_frame("r", "g", "b")["r"].shape   # => [480, 640]
+  #   ```
+  #
+  #   Nothing is copied: each column is a view, so writes through the frame
+  #   reach `self` and writes to `self` show in the frame, values and mask
+  #   alike. Use `copy` on the frame for an independent one.
+  #
+  #   Without names the columns are called `"c0"`, `"c1"`, ... A 1-D array
+  #   has no axis to take columns from; give it one with `a[nil, :_]`.
+  #
+  #   @param names [Array<String>] column names, one per position along `axis`.
+  #   @param axis [Integer] the axis the columns are taken along; it cannot
+  #     be axis 0, which holds the rows.
+  #   @return [CAFrame]
+  #   @raise [TypeError] when a name is not a String or `axis` is not an
+  #     Integer.
+  #   @raise [ArgumentError] when `axis` is out of range or is the row axis,
+  #     when the number of names differs from the length of `axis`, or when
+  #     a name repeats.
+  def to_frame (*names, axis: -1)
+    axis = -1 if axis.nil?
+    if ndim < 2
+      raise ArgumentError,
+            "to_frame: a #{ndim}-D array has no axis to take columns from " \
+            "besides the row axis (axis 0); give it one with a[nil, :_]"
+    end
+    ax = normalize_axis(axis, "to_frame")
+    if ax == 0
+      raise ArgumentError,
+            "to_frame: axis #{axis} is the row axis (axis 0); columns are " \
+            "taken along another axis"
+    end
+    ncol = shape[ax]
+    if names.empty?
+      names = Array.new(ncol) { |j| "c#{j}" }
+    else
+      names.each { |name| CAFrame.__send__(:column_key, name, "to_frame") }
+      raise ArgumentError, "to_frame: a column name can not be empty" if names.include?("")
+      unless names.size == ncol
+        raise ArgumentError,
+              "to_frame: #{names.size} names for #{ncol} columns " \
+              "(axis #{axis} has length #{ncol})"
+      end
+      CAFrame.__send__(:refuse_repeated_names, names, "to_frame")
+    end
+    index = Array.new(ndim, nil)
+    columns = {}
+    names.each_with_index do |name, j|
+      index[ax] = j
+      columns[name] = self[*index]
+    end
+    CAFrame.new(columns)
+  end
+end