carray 3.0.2 → 3.0.3

data/lib/carray/complex.rb

@@ -52,6 +52,14 @@ class CArray
   # framework's read-only chain does not replace the CAField mutable view).
   # ---------------------------------------------------------------------------
 
+  # Whether a view kept by {#real} / {#imag} was made for this array.
+  # `dup` and `clone` copy instance variables, so a copy arrives holding the
+  # original's view, whose parent is the original; the copy makes its own.
+  def part_of_self? (view)
+    view && view.parent.equal?(self)
+  end
+  private :part_of_self?
+
   # @overload real
   #   Returns the real part of `self` as a zero-copy view. For a
   #   complex array the view is a mutable {CAField} into the
@@ -59,19 +67,16 @@ class CArray
   #   over `self`. Writing to the view updates `self` in place.
   #   @return [CArray]
   def real
-    if not @__real__
-      if complex?
-        @__real__ = case data_type
-                    when CA_CMPLX64
-                      field(0, CA_FLOAT32)
-                    when CA_CMPLX128
-                      field(0, CA_FLOAT64)
-                    end
-      else
-        @__real__ = self[]
-      end
-    end
-    @__real__
+    return @__real__ if part_of_self?(@__real__)
+    view = if complex?
+             case data_type
+             when CA_CMPLX64  then field(0, CA_FLOAT32)
+             when CA_CMPLX128 then field(0, CA_FLOAT64)
+             end
+           else
+             self[]
+           end
+    frozen? ? view : (@__real__ = view)
   end
 
   # @overload real=(val)
@@ -85,35 +90,37 @@ class CArray
   # @overload imag
   #   Returns the imaginary part of `self`. For a complex array the
   #   result is a mutable {CAField} view of the imaginary slot;
-  #   writing to it updates `self` in place. For a real array the
-  #   result is a fresh independent CArray filled with 0.
+  #   writing to it updates `self` in place. For an object array each
+  #   cell's `imaginary`; for a real array a fresh independent CArray
+  #   filled with 0. Masked cells stay masked.
   #   @return [CArray]
   def imag
-    if not @__imag__
-      if complex?
-        @__imag__ = case data_type
-                    when CA_CMPLX64
-                      field(4, CA_FLOAT32)
-                    when CA_CMPLX128
-                      field(8, CA_FLOAT64)
-                    end
-      else
-        @__imag__ = self.template { 0 }
-      end
+    if object?
+      return imag_i
+    end
+    if not complex?
+      zero = template { 0 }
+      zero[is_masked] = UNDEF if has_mask?
+      return zero
     end
-    return @__imag__
+    return @__imag__ if part_of_self?(@__imag__)
+    view = case data_type
+           when CA_CMPLX64  then field(4, CA_FLOAT32)
+           when CA_CMPLX128 then field(8, CA_FLOAT64)
+           end
+    frozen? ? view : (@__imag__ = view)
   end
 
   # @overload imag=(val)
   #   Sets the imaginary-part slot to `val` (complex arrays only).
   #   @param val [CArray, Numeric] value to broadcast.
   #   @return [Object] `val`.
-  #   @raise [RuntimeError] when `self` is not a complex array.
+  #   @raise [CArray::DataTypeError] when `self` is not a complex array.
   def imag= (val)
     if complex?
       imag[] = val
     else
-      raise "not a complex array"
+      raise CArray::DataTypeError, "not a complex array"
     end
   end
 

data/lib/carray/conditional.rb

@@ -6,7 +6,7 @@
 #  same "pick a value per cell based on a boolean condition" contract but
 #  differ in where the condition lives and how the branches are supplied:
 #
-#      cond.then_else(x, y)                  # boolean receiver, 2 values (eager)
+#      cond.then_else(x, y)                  # boolean receiver, 2 values (lazy if an operand is)
 #      self.replace_where(cond, b)           # value receiver, 1 replacement (eager)
 #      self.conditional(cond, f_then, f_else) # value receiver, 2 callables (per-region)
 #
@@ -25,10 +25,20 @@ class CArray
   #   reading as "if `self` then `x` else `y`". The boolean 2-way
   #   case of {#choose} that additionally propagates `self`'s mask
   #   (`UNDEF` in `self` produces `UNDEF` in the result).
+  #   The result's data type is the common type of the two branches; a
+  #   branch given as `UNDEF` masks its cells and leaves the type to the
+  #   other branch (`cond.then_else(a, UNDEF)` has `a`'s data type).
+  #   When `self` or a branch is a lazy view, the result is a lazy view
+  #   (read-only) that joins the expression around it; `copy` makes an
+  #   array of it.  An object result is always computed at once.
   #   @param x [CArray, Numeric, Object] true-branch value(s).
   #   @param y [CArray, Numeric, Object] false-branch value(s).
-  #   @return [CArray] new array with the same shape as `self`.
-  #   @raise [ArgumentError] when `self` is not a boolean CArray.
+  #   @return [CArray] new array with the same shape as `self`, or a
+  #     lazy view (CATriOp) when an operand is lazy.
+  #   @raise [ArgumentError] when `self` is not a boolean CArray, or when
+  #     an array branch's shape does not pair with `self`'s as an
+  #     operator's operands would (equal, or size-1 axes of the same ndim;
+  #     an object or fixlen result needs `self`'s shape exactly).
   def then_else (x, y)
     # Guard: self must be boolean.  Integer / float receiver would be
     # silently reinterpreted by the indexer setter (`result[self] = ...`)
@@ -38,10 +48,27 @@ class CArray
       raise ArgumentError,
             "then_else: receiver must be a boolean CArray (data_type == CA_BOOLEAN), got #{self.data_type}"
     end
-    # Promote data_type from both branches via CArray.result_type
-    # (a CScalar contributes its own data_type, so CA_INT32(0) keeps int32
-    # where a bare Ruby Integer would widen to int64).
-    dt = CArray.result_type(x, y)
+    reshaped = [x, y].any? { |v| v.is_a?(CArray) && !v.scalar? && v.shape != shape }
+    face = [x, y].find { |v| v.is_a?(CArray) && !v.scalar? && v.face? }
+    if face
+      refuse_branch_shape(x, y) if reshaped
+      return face_then_else(x, y, face)
+    end
+    # Promote data_type from both branches via CArray.result_type, which
+    # lets a scalar branch take the type of an array branch as an operator
+    # does (cond.then_else(0, int32_array) is int32).  UNDEF is a missing
+    # value, not a value of some type: a branch that is UNDEF leaves the
+    # type to the other.
+    typed = [x, y].reject { |v| v.equal?(UNDEF) }
+    dt = typed.empty? ? CA_OBJECT : CArray.result_type(*typed)
+    return lazy_then_else(x, y, dt) if lazy_then_else?(x, y, dt)
+    # A branch shaped unlike the condition goes through the lazy select,
+    # which pairs the shapes by the operator rule (equal, or size-1 axes
+    # with the same ndim) and refuses the rest.
+    if reshaped
+      refuse_branch_shape(x, y) if scalar? || dt == CA_OBJECT || dt == CA_FIXLEN
+      return lazy_then_else(x, y, dt).copy
+    end
     # A CScalar (scalar? CArray) is treated as a scalar value, not as a
     # self-shaped operand: full CArray -> gather/copy, scalar -> broadcast.
     y_full = y.is_a?(CArray) && !y.scalar?
@@ -60,6 +87,66 @@ class CArray
     result
   end
 
+  # A lazy receiver or branch makes the selection a lazy node (CATriOp
+  # select), so it joins the expression around it rather than computing
+  # it here.  The branches carry the result type: a scalar branch becomes
+  # a CScalar of it, UNDEF a masked one.
+  private def lazy_then_else? (x, y, dt)
+    return false if scalar? || dt == CA_OBJECT || dt == CA_FIXLEN
+    [self, x, y].any? { |v| v.is_a?(CArray) && v.__lazy_view__? }
+  end
+
+  # The types the lazy select cannot carry keep the strict form: a full
+  # branch must have the condition's shape.
+  private def refuse_branch_shape (x, y)
+    bad = [x, y].find { |v| v.is_a?(CArray) && !v.scalar? && v.shape != shape }
+    raise ArgumentError,
+          "then_else: branch shape #{bad.shape.inspect} does not match the condition's #{shape.inspect}"
+  end
+
+  private def lazy_then_else (x, y, dt)
+    branch = lambda do |v|
+      next v if v.is_a?(CArray) && !v.scalar?
+      cell = CScalar.new(dt)
+      cell[0] = v.is_a?(CArray) ? v[0] : v
+      cell
+    end
+    CATriOp.__build__(self, branch.(x), branch.(y), CATriOp::OP_SELECT)
+  end
+
+  private def face_then_else (x, y, face)
+    # A Face branch decides the result: the other branch is written into a
+    # copy of it through the Face's own store, so an instant stays an instant.
+    # Promoting the two branches to a common data type would compare a Face's
+    # surface type with its storage's.  A CAConstString cannot be written, so
+    # it selects among its strings and packs them again.
+    if face.is_a?(CAConstString)
+      sx = x.is_a?(CAConstString) ? x.to_string : x
+      sy = y.is_a?(CAConstString) ? y.to_string : y
+      return CArray.const_string(then_else(sx, sy), encoding: face.encoding)
+    end
+    x_full = x.is_a?(CArray) && !x.scalar?
+    y_full = y.is_a?(CArray) && !y.scalar?
+    base   = y_full ? y : x
+    if base.read_only?
+      # No store to write through (a CACategorical's codes): select among
+      # the values themselves.
+      result = CArray.object(*shape)
+      result[] = y_full ? y.to_type(CA_OBJECT) : y
+      result[self] = x_full ? x.to_type(CA_OBJECT)[self] : x
+      result[self.is_masked] = UNDEF if self.has_mask?
+      return result
+    end
+    result = base.copy
+    if y_full
+      result[self] = x_full ? x[self] : x
+    else
+      result[self.not] = y.is_a?(CArray) ? y[0] : y
+    end
+    result[self.is_masked] = UNDEF if self.has_mask?
+    result
+  end
+
   # @overload replace_where(cond, b)
   #   Returns a copy of `self` with cells where `cond` is true
   #   replaced by `b`. Functional sibling of the destructive

data/lib/carray/const_string.rb

@@ -182,6 +182,13 @@ class CAConstString
     to_string.minmax(*args, **kw).map { |r| const_string_lift(r) }
   end
 
+  # @overload clip(min, max = nil, fill_value = nil, lfill: nil, ufill: nil)
+  #   Clips the strings in byte order (see {CArray#clip}).
+  #   @return [CAConstString]
+  def clip (*args, **kw)
+    const_string_lift(to_string.clip(*args, **kw))
+  end
+
   # @overload min_index(axis: nil)
   #   @return [Integer, CArray] where the byte-smallest string sits.
   def min_index (*args, **kw)
@@ -315,6 +322,32 @@ class CAConstString
     to_string.categorize(labels: labels, sort_labels: sort_labels)
   end
 
+  # ---- comparison operators ---------------------------------------------
+  #
+  # The core operators compare storage, and a storage cell is a byte range:
+  # `x < "dd"` compared where each string sits in the buffer and answered
+  # true everywhere.  They read the strings here instead.  `eq` has a native
+  # byte scan and stays; `<=>` composes from `>` and `<`, so it follows.
+
+  # @!method ne(other)
+  #   @return [CArray] boolean, true where the strings differ.
+  # @!method lt(other)
+  #   @return [CArray] boolean, true where self sorts before `other`.
+  # @!method le(other)
+  #   @return [CArray] boolean, true where self sorts before or equals `other`.
+  # @!method gt(other)
+  #   @return [CArray] boolean, true where self sorts after `other`.
+  # @!method ge(other)
+  #   @return [CArray] boolean, true where self sorts after or equals `other`.
+  [:ne, :lt, :le, :gt, :ge].each do |op|
+    define_method(op) { |other| to_string.public_send(op, string_operand(other)) }
+  end
+
+  alias <  lt
+  alias <= le
+  alias >  gt
+  alias >= ge
+
   # A CAConstString operand has to be decoded too; anything else (a CAString,
   # an object array, an Array) already compares as strings.
   private def string_operand (other)
@@ -330,3 +363,16 @@ class CAConstString
   end
 
 end
+
+class CAConstString
+  # The decimal readers of CAFrame#infer_types / #cast, read from the buffer
+  # without a String per cell.
+  def __infer_number_type_of_text__
+    CArray.__const_string_infer_number_type_of_text__(self)
+  end
+
+  def __read_text_as_number__(type, report = nil)
+    out = CArray.__const_string_read_text_as_number__(self, type, report)
+    out && ndim > 1 ? out.reshape(*shape) : out
+  end
+end

data/lib/carray/frame/concat.rb

@@ -9,6 +9,9 @@
 #
 # Both are class methods (symmetric N-ary; no frame is privileged), like
 # +CArray.meld+ / +CArray.concatenate+, not instance verbs.
+#
+# +CAFrame.stack+ and +CAFrame#unstack+ add and take away a layer axis
+# instead of rows.
 
 class CAFrame
   # Weld frames along the row axis, view-style.  Each output column is
@@ -28,6 +31,13 @@ class CAFrame
   #
   # The index is welded too when every frame has one (their +axis_name+
   # must agree); if none do, the result has no index; a mix raises.
+  #
+  # A frame with no rows adds no values, so it takes no part in the result
+  # past these checks: its column set, its N-D columns' trailing dimensions
+  # and its index still have to agree, but the
+  # data types are the other frames' (a header-only CSV reads as object
+  # columns, which would otherwise refuse or demote the rest).  When every
+  # frame is empty, the first one stands for them.
   # Column-set mismatch raises — a union-with-UNDEF mode is a possible
   # future opt-in, kept out here to stay explicit (memo §4.2).
   #
@@ -37,8 +47,11 @@ class CAFrame
     frames = frames.flatten
     check_concat_inputs(frames, verb: "meld")
     first = frames.first
-    names = first.variable_names
+    names = first.column_names
     check_column_sets(frames, names, verb: "meld")
+    index_pieces(frames, verb: "meld")
+    check_trailing_shapes(frames, names, verb: "meld")
+    frames = frames_with_rows(frames)
     cols = {}
     names.each do |name|
       cols[name] = CArray.meld(frames.map { |f| f[name] }, axis: 0)
@@ -54,8 +67,9 @@ class CAFrame
   # For a view frame that shares storage with the inputs (strict same data type
   # per column, chain composability preserved) use {CAFrame.meld}.
   #
-  # Column matching, index handling, and column-set / index-mix rules
-  # match {CAFrame.meld}.
+  # Column matching, index handling, column-set / index-mix rules and the
+  # treatment of a frame with no rows match {CAFrame.meld}: an empty frame's
+  # data types take no part in the common type.
   #
   #   CAFrame.concatenate(jan, feb, mar)     # eager, independent result
   #   CAFrame.concatenate([jan, feb, mar])   # an Array is accepted too
@@ -63,8 +77,11 @@ class CAFrame
     frames = frames.flatten
     check_concat_inputs(frames, verb: "concatenate")
     first = frames.first
-    names = first.variable_names
+    names = first.column_names
     check_column_sets(frames, names, verb: "concatenate")
+    index_pieces(frames, verb: "concatenate")
+    check_trailing_shapes(frames, names, verb: "concatenate")
+    frames = frames_with_rows(frames)
     cols = {}
     names.each do |name|
       cols[name] = CArray.concatenate(frames.map { |f| f[name] })
@@ -72,6 +89,108 @@ class CAFrame
     new(cols, axis_name: first.axis_name, index: concatenate_index(frames))
   end
 
+  # Stack frames of the same shape as layers.  Each output column is
+  # +CArray.stack+ of that column across the frames along +axis+, counted in
+  # the column's own axes (0 is the row axis, so +axis+ starts at 1): a
+  # scalar column of K frames becomes a column of shape (nrow, K).  The rows
+  # stay as they are, so row verbs work on the stack unchanged and a layer is
+  # a column axis -- +s["temp"][nil, k]+ is frame k's column, and
+  # +s["temp"].mean(axis: 1)+ reduces across the layers.
+  #
+  # The result is a view frame: its columns are CAStack views over the
+  # frames' columns, and writes reach them.  Call +copy+ for an independent
+  # frame.  The layers carry no labels; keep them alongside (the dates of the
+  # files, say) as an array of your own.
+  #
+  # Every frame must have the same columns, the same number of rows and the
+  # same index (by value) under the same axis name, or none; the result
+  # takes the first frame's.  +unstack+ is the inverse:
+  #
+  #   s = CAFrame.stack(jan1, jan2, jan3)    # temp:float64 -> temp:float64[3]
+  #   CAFrame.stack(*s.unstack(axis: 1))       # the same frame again
+  def self.stack(*frames, axis: 1)
+    frames = frames.flatten
+    check_concat_inputs(frames, verb: "stack")
+    check_layer_axis(axis, "stack")
+    first = frames.first
+    names = first.column_names
+    check_column_sets(frames, names, verb: "stack")
+    frames.each_with_index do |f, i|
+      next if i.zero?
+      unless f.nrow == first.nrow
+        raise ArgumentError,
+              "stack: frame #{i} has #{f.nrow} rows, frame 0 has #{first.nrow}"
+      end
+    end
+    indexes, = index_pieces(frames, verb: "stack")
+    if indexes && ! indexes.all? { |x| same_index_values?(x, indexes.first) }
+      raise ArgumentError, "stack: the frames' indexes differ; stacked rows have to be the same rows"
+    end
+    cols = {}
+    names.each do |name|
+      parts = frames.map { |f| f[name] }
+      if axis > parts.first.ndim
+        raise ArgumentError,
+              "stack: axis #{axis} out of range for column #{name.inspect} " \
+              "(a #{parts.first.ndim}-D column takes 1 to #{parts.first.ndim})"
+      end
+      cols[name] = CArray.stack(parts, axis: axis)
+    end
+    new(cols, axis_name: first.axis_name, index: first.index)
+  end
+
+  # Split the frame along a column axis into an Array of frames, one per
+  # position on that axis -- the inverse of {CAFrame.stack}.  +axis+ counts
+  # the columns' own axes (0 is the row axis, so it starts at 1).  Every
+  # column must have that axis, and the same length along it: a column
+  # without it has no layer to give each frame.  Each frame's columns are
+  # views of this frame's, and each has this frame's index.
+  #
+  #   s.unstack(axis: 1)    # => [frame of layer 0, frame of layer 1, ...]
+  def unstack(axis:)
+    self.class.send(:check_layer_axis, axis, "unstack")
+    if @columns.empty?
+      raise ArgumentError, "unstack: the frame has no columns to unstack"
+    end
+    length = nil
+    @columns.each do |name, col|
+      unless axis < col.ndim
+        raise ArgumentError,
+              "unstack: column #{name.inspect} is #{col.ndim}-D and has no axis #{axis}"
+      end
+      length ||= col.shape[axis]
+      unless col.shape[axis] == length
+        raise ArgumentError,
+              "unstack: column #{name.inspect} has #{col.shape[axis]} along axis #{axis}, " \
+              "the others #{length}"
+      end
+    end
+    (0...length).map do |k|
+      cols = @columns.to_h do |name, col|
+        idx = [nil] * col.ndim
+        idx[axis] = k
+        [name, col[*idx]]
+      end
+      CAFrame.new(cols, axis_name: @axis_name, index: @index)
+    end
+  end
+
+  def self.check_layer_axis(axis, verb)
+    unless axis.is_a?(Integer)
+      raise TypeError, "#{verb}: axis must be an Integer (got #{axis.inspect})"
+    end
+    if axis == 0
+      raise ArgumentError,
+            "#{verb}: axis 0 is the row axis; rows are joined by meld or concatenate"
+    end
+    if axis < 0
+      raise ArgumentError,
+            "#{verb}: axis counts a column's axes from 1 (got #{axis}); " \
+            "a negative axis would mean a different axis in columns of different ndim"
+    end
+  end
+  private_class_method :check_layer_axis
+
   # ---- shared validators -------------------------------------------------
 
   def self.check_concat_inputs(frames, verb:)
@@ -86,15 +205,47 @@ class CAFrame
     expected = names.sort
     frames.each_with_index do |f, i|
       next if i.zero?
-      if f.variable_names.sort != expected
+      if f.column_names.sort != expected
         raise ArgumentError,
-              "#{verb}: frame #{i} has columns #{f.variable_names.inspect}, " \
+              "#{verb}: frame #{i} has columns #{f.column_names.inspect}, " \
               "expected the same set as #{names.inspect}"
       end
     end
   end
   private_class_method :check_column_sets
 
+  # A column's trailing dimensions are its shape, not a type the other
+  # frames can settle, so a frame with no rows has to agree on them too.
+  def self.check_trailing_shapes(frames, names, verb:)
+    names.each do |name|
+      trailing = frames.first[name].shape[1..]
+      frames.each_with_index do |f, i|
+        next if i.zero? || f[name].shape[1..] == trailing
+        raise ArgumentError,
+              "#{verb}: column #{name.inspect} has trailing dimensions " \
+              "#{f[name].shape[1..].inspect} in frame #{i}, #{trailing.inspect} in frame 0"
+      end
+    end
+  end
+  private_class_method :check_trailing_shapes
+
+  # Two indexes hold the same rows when their values agree cell by cell; a
+  # NaN label matches a NaN label, and a masked cell a masked cell.
+  def self.same_index_values?(a, b)
+    return false unless a.shape == b.shape
+    a.to_a.zip(b.to_a).all? do |x, y|
+      x.equal?(y) || x == y || (x.is_a?(Float) && y.is_a?(Float) && x.nan? && y.nan?)
+    end
+  end
+  private_class_method :same_index_values?
+
+  # The frames that add rows, or the first frame when none does.
+  def self.frames_with_rows(frames)
+    with_rows = frames.reject { |f| f.nrow.zero? }
+    with_rows.empty? ? [frames.first] : with_rows
+  end
+  private_class_method :frames_with_rows
+
   # ---- index helpers -----------------------------------------------------
 
   # Index welded via CArray.meld (view).  Every frame must agree: all

data/lib/carray/frame/convert.rb

@@ -16,21 +16,25 @@ class CAFrame
   # column name => value; a scalar cell is a Ruby value, an N-D cell is a Ruby
   # Array, and a masked cell (UNDEF) is +nil+. Normalizing UNDEF -> nil and
   # CArray -> Array (unlike +each_row+, which yields the raw view with UNDEF /
-  # CArray slices) is what makes it round-trip: +from_records(df.to_records)+
-  # rebuilds the same typing, and the result is JSON-serializable.
+  # CArray slices) makes the result JSON-serializable and lets
+  # +from_records(df.to_records)+ give back the column names, the values and
+  # the mask. The rest is not in a record: +from_records+ rebuilds each type
+  # from the values (an int32 comes back int64, a time column as an object
+  # column of its elements), the index comes back as an ordinary first
+  # column, and a frame with no rows gives +[]+, which has no column names.
   def to_records
     each_row.map { |row| row.transform_values { |v| record_value(v) } }
   end
 
   # Hand the frame over as a CArray with the minimum work (memo §11.9): a
-  # 2-D **view** of shape (nrow, nvar), one column per variable in column
+  # 2-D **view** of shape (nrow, ncol), one per column in column
   # order (§12-C). Nothing is materialised — the result is a +CAStack+ over
   # the stored columns, so reads gather from them and writes flow back
   # (§3.6). Call +copy+ on it for an independent, owned matrix; +CArray.tabulate+
   # is the eager sibling that builds one directly.
   #
   # Only same-shape scalar (1-D) columns qualify. An N-D column has no single
-  # matrix form and raises — escape it per column with +df["name"]+. A mixed
+  # matrix form and raises — take it per column with +df["name"]+. A mixed
   # data type set is promoted to a common type (+result_type+, §12-F) through
   # lazy cast lanes, so the promotion costs no buffer either.
   #
@@ -48,12 +52,12 @@ class CAFrame
     if nd
       raise ArgumentError,
             "to_ca needs all-scalar (1-D) columns; #{nd.first.inspect} is " \
-            "#{nd.last.ndim}-D — escape per column with df[name]"
+            "#{nd.last.ndim}-D — take it per column with df[name]"
     end
     refuse_unshared_columns(cols) if writable
     begin
       CArray.stack(cols, axis: 1)
-    rescue ArgumentError, RuntimeError => e
+    rescue ArgumentError, TypeError, CArray::DataTypeError => e
       # The columns have no common type (a text or Face column beside a
       # numeric one). Point at the frame-level verb that gives them one.
       raise e.class,
@@ -87,7 +91,9 @@ class CAFrame
     if UNDEF.equal?(v)
       nil
     elsif v.is_a?(CArray)
-      v.to_a.map { |e| UNDEF.equal?(e) ? nil : e }
+      record_value(v.to_a)
+    elsif v.is_a?(Array)
+      v.map { |e| record_value(e) }
     else
       v
     end