carray 3.0.0 → 3.0.2

data/lib/carray/data_type_limits.rb

@@ -0,0 +1,91 @@
+#  What each numeric data type can hold, as constants on the typed
+#  classes.
+#
+#      CArray::Int32::MAX        #  =>  2147483647
+#      CArray::Float32::EPSILON  #  =>  1.1920928955078125e-07
+#
+#  The names follow NumPy's iinfo / finfo rather than Ruby's Float, so
+#  that MIN and MAX bracket the range for every numeric type alike:
+#
+#      CArray::Float64::MIN      #  => -1.7976931348623157e+308
+#      Float::MIN                #  =>  2.2250738585072014e-308
+#
+#  Those are not the same number and not the same question.  Ruby's
+#  Float::MIN is the smallest positive normal, which is TINY here; MIN
+#  is the bottom of the range, which for an integer type is the only
+#  thing MIN could mean.  Code that has to bracket a type without
+#  knowing whether it is integer or float reads MIN and MAX and is
+#  right either way, and that is the reason for the choice.
+#
+#  Not every data type has limits.  boolean, fixlen and object have no
+#  numeric range, so they carry none of these constants -- asking gives
+#  a NameError rather than an answer that would have to be invented.
+
+class CArray
+
+  #  The integer widths come from the extension rather than from a table
+  #  written here, so a platform where a type is not the usual width is
+  #  described correctly instead of confidently mis-described.
+
+  {
+    Int8   => true,  Int16  => true,  Int32  => true,  Int64  => true,
+    UInt8  => false, UInt16 => false, UInt32 => false, UInt64 => false,
+  }.each do |klass, signed|
+    bits = 8 * CArray.sizeof(klass::TypeSymbol)
+    if signed
+      klass.const_set(:MIN, -(2 ** (bits - 1)))
+      klass.const_set(:MAX, 2 ** (bits - 1) - 1)
+    else
+      klass.const_set(:MIN, 0)
+      klass.const_set(:MAX, 2 ** bits - 1)
+    end
+  end
+
+  #  float32 and float64 are IEEE-754 binary32 and binary64, and the four
+  #  values below follow from the format: with `p` significand bits (the
+  #  implicit one included) and `emax` the exponent of the largest finite
+  #  value,
+  #
+  #      EPSILON = 2 ** (1 - p)          the step from 1.0 to the next float
+  #      MAX     = (2 - EPSILON) * 2 ** emax
+  #      MIN     = -MAX                  every format here is symmetric
+  #      TINY    = 2 ** (1 - emax)       the smallest positive normal
+  #
+  #  All four are exactly representable as a Ruby Float, binary32
+  #  included, so nothing is rounded on the way in.  The width is
+  #  checked rather than assumed: on a platform where C float or double
+  #  is not one of these formats the arithmetic below would be wrong,
+  #  and a wrong limit is worse than a missing one.
+
+  {
+    Float32 => [4, 24, 127],
+    Float64 => [8, 53, 1023],
+  }.each do |klass, (bytes, precision, max_exponent)|
+    actual = CArray.sizeof(klass::TypeSymbol)
+    unless actual == bytes
+      raise "#{klass} is #{actual} bytes wide, not the #{bytes * 8}-bit IEEE-754 " \
+            "format its limits are derived from"
+    end
+    epsilon = 2.0 ** (1 - precision)
+    max     = (2.0 - epsilon) * 2.0 ** max_exponent
+    klass.const_set(:EPSILON, epsilon)
+    klass.const_set(:MAX,     max)
+    klass.const_set(:MIN,     -max)
+    klass.const_set(:TINY,    2.0 ** (1 - max_exponent))
+  end
+
+  #  A complex type is a pair of floats, so its limits are its
+  #  component's -- MIN and MAX bound the real and the imaginary part
+  #  separately, not any magnitude of the pair.  This is what
+  #  np.finfo(np.complex64) answers too.
+
+  {
+    Complex64  => Float32,
+    Complex128 => Float64,
+  }.each do |klass, component|
+    [:MIN, :MAX, :TINY, :EPSILON].each do |name|
+      klass.const_set(name, component.const_get(name))
+    end
+  end
+
+end

data/lib/carray/fixlen_string.rb

@@ -3,7 +3,7 @@
 #  CAFixlenString high-level construction surface.
 #
 #  CAFixlenString itself (the string interpretation of CA_FIXLEN storage) lives
-#  in ext/ca_obj_string.c.  This file provides the ergonomic builder that packs
+#  in ext/ca_obj_fixlen_string.c.  This file provides the ergonomic builder that packs
 #  Ruby Strings into a fixed-width CA_FIXLEN entity and wraps it via
 #  CAFixlenString.wrap.
 #

data/lib/carray/frame/concat.rb

@@ -2,7 +2,7 @@
 #
 # Two entry points, mirroring the CArray-level taxonomy:
 #
-#   +CAFrame.meld+         view frame,  strict same-dtype per column,
+#   +CAFrame.meld+         view frame,  strict same data type per column,
 #                          each column is a CAMeld view over the inputs
 #   +CAFrame.concatenate+  eager frame, auto-casts per column,
 #                          each column is a materialised entity
@@ -48,10 +48,10 @@ class CAFrame
 
   # Concatenate frames along the row axis, eagerly.  Each output column is
   # +CArray.concatenate+ of that column across the input frames, so per-column
-  # dtypes auto-promote to a common type.  The result is a fresh, independent
+  # data types auto-promote to a common type.  The result is a fresh, independent
   # frame — writes to it do not propagate back to the input frames.
   #
-  # For a view frame that shares storage with the inputs (strict same dtype
+  # For a view frame that shares storage with the inputs (strict same data type
   # per column, chain composability preserved) use {CAFrame.meld}.
   #
   # Column matching, index handling, and column-set / index-mix rules

data/lib/carray/frame/convert.rb

@@ -31,7 +31,7 @@ class CAFrame
   #
   # Only same-shape scalar (1-D) columns qualify. An N-D column has no single
   # matrix form and raises — escape it per column with +df["name"]+. A mixed
-  # dtype set is promoted to a common type (+result_type+, §12-F) through
+  # data type set is promoted to a common type (+result_type+, §12-F) through
   # lazy cast lanes, so the promotion costs no buffer either.
   #
   # +writable: true+ demands a result whose writes reach this frame's own

data/lib/carray/frame/csv_parser.rb

@@ -91,13 +91,19 @@ class CAFrame
         @recno    = 0
       end
 
-      # Fields of the next non-blank record, or nil at EOF.
-      def read(io)
+      # Fields of the next record, or nil at EOF.  A blank line carries no
+      # separator, so it cannot be a row of a file with more than one column
+      # and is skipped as noise between records.  In a single-column file it
+      # is the only spelling a missing single field has -- which is what
+      # to_csv writes for a masked cell -- so the caller passes
+      # +blank_is_row: true+ once the column count is known to be one, and
+      # the empty record becomes a row of no fields for build_frame to pad.
+      def read(io, blank_is_row: false)
         loop do
           rec = CSVParser.read_record(io, @quote)
           return nil if rec.nil?
           @recno += 1
-          next if rec.empty?
+          next if rec.empty? && !blank_is_row
           return rec.count(@quote).zero? ? simple(rec) : scan(rec)
         end
       end
@@ -195,7 +201,8 @@ class CAFrame
     # Consume the remaining records as data rows.
     def body
       rows = []
-      while (fields = @tok.read(@io))
+      blank_is_row = @names && @names.size == 1
+      while (fields = @tok.read(@io, blank_is_row: blank_is_row))
         rows << fields
       end
       @rows = rows

data/lib/carray/frame/frame.rb

@@ -45,14 +45,21 @@ class CAFrame
     @columns   = {}
     @axis_name = axis_name || DEFAULT_AXIS_NAME
     @index     = nil
+    # The row axis name this frame had before set_index promoted a column over
+    # it, so reset_index can put it back.  A frame built with an index never
+    # had one, and neither does a frame derived from an indexed one: both are
+    # born indexed, so there is nothing to restore and the default stands.
+    @axis_name_before_index = nil
 
     n = nil
+    n_from = nil
     columns.each do |name, col|
       key = name.to_s
       ca  = coerce_column(col)
       len = ca.shape[0]
       if n.nil?
-        n = len
+        n      = len
+        n_from = key
       elsif len != n
         raise ArgumentError,
               "column #{key.inspect} has axis-0 length #{len}, expected #{n}"
@@ -67,8 +74,12 @@ class CAFrame
         raise ArgumentError, "index must be a 1-D column (got ndim #{idx.ndim})"
       end
       if n && idx.shape[0] != n
+        # Name the column as well as the index: when the index is the frame's
+        # existing one and a column is the new arrival, blaming the index alone
+        # points at the side the caller cannot change.
         raise ArgumentError,
-              "index length #{idx.shape[0]} does not match nrow #{n}"
+              "column #{n_from.inspect} has axis-0 length #{n}, " \
+              "but the index has length #{idx.shape[0]}"
       end
       @index = idx
       @nrow  = idx.shape[0] if n.nil?
@@ -234,9 +245,10 @@ class CAFrame
     end
   end
 
-  # Bind +key+ to +column+, adding the name when it is new.  This is the one
-  # place a column enters an existing frame, so it is where the axis-0 length
-  # invariant is enforced (§12-A).  It is a replacement rather than an edit,
+  # Bind +key+ to +column+, adding the name when it is new.  This is where a
+  # column arriving from outside the frame has its axis-0 length checked
+  # (§12-A); the conversions in verbs.rb rebind +@columns+ directly and rely on
+  # being length-preserving instead.  It is a replacement rather than an edit,
   # which is what sets it apart from the rest: +fill+ / +mask_eq+ /
   # +df[rows] = UNDEF+ write to the shared column and are therefore visible
   # wherever it is held, while this binds the name to a different column and
@@ -307,6 +319,12 @@ class CAFrame
   # value kept, so the row stays identifiable), leaving a later, better-informed
   # pass to re-judge them. Definitely-true rows carry their values through
   # unchanged in both modes.
+  #
+  # The default is a view-frame. +keep_masked: true+ returns a **materialized**
+  # frame instead -- carrying the UNDEF forward means writing it into the
+  # result, which a view cannot do without masking the parent's rows. That holds
+  # whether or not the selector actually carries a masked cell, so the same call
+  # site does not switch between sharing and copying with the data.
   def filter(keep_masked: false)
     mask = yield(self)
     unless mask.is_a?(CArray) && mask.data_type == :boolean
@@ -330,7 +348,18 @@ class CAFrame
     unless idx.ndim == 1
       raise ArgumentError, "index must be a 1-D column (got ndim #{idx.ndim})"
     end
+    # Only the name held before the frame had an index: a second set_index must
+    # not overwrite it with the first index's name.
+    @axis_name_before_index = @axis_name if @index.nil?
     @columns.delete(key)
+    if @index
+      # Re-indexing replaces the index, so the one being replaced goes back to
+      # being a column -- the same demotion reset_index performs, in the same
+      # position, which makes set_index over an existing index equivalent to
+      # reset_index followed by set_index.  Dropping it would lose the column it
+      # was made from.
+      @columns = { @axis_name => @index }.merge(@columns)
+    end
     @index     = idx
     @axis_name = key
     @nrow      = idx.shape[0]
@@ -344,7 +373,8 @@ class CAFrame
     return self unless @index
     @columns   = { @axis_name => @index }.merge(@columns)
     @index     = nil
-    @axis_name = DEFAULT_AXIS_NAME
+    @axis_name = @axis_name_before_index || DEFAULT_AXIS_NAME
+    @axis_name_before_index = nil
     self
   end
 
@@ -370,6 +400,17 @@ class CAFrame
   # multi-row, frame-returning path). Positional access is +df[i]+.
   def at(label)
     raise ArgumentError, "at requires an index (set one with set_index)" unless @index
+    # An index may hold a masked cell -- an outer join and align both build one
+    # -- but an undefined label identifies no row, and two undefined labels are
+    # not the same label. The addressing primitives already say this (a masked
+    # key matches nothing, not another masked key), so at says it too rather
+    # than answering through eq(UNDEF), which asks about the mask and not about
+    # the label at all.
+    if UNDEF.equal?(label)
+      raise ArgumentError,
+            "at(UNDEF): an undefined label identifies no row; " \
+            "use filter { |f| f.index.is_masked } for the rows with no label"
+    end
     pos = @index.eq(label).where
     case pos.elements
     when 0
@@ -436,13 +477,6 @@ class CAFrame
     h
   end
 
-  # @return [String]
-  def inspect
-    parts = @columns.map { |k, v| "#{k}:#{v.data_type}#{v.ndim > 1 ? v.shape[1..].inspect : ''}" }
-    idx = @index ? " index=#{@axis_name.inspect}" : ""
-    "#<CAFrame nrow=#{@nrow} vars=[#{parts.join(', ')}]#{idx}>"
-  end
-
   private def rebuild(cols)
     CAFrame.new(cols, axis_name: @axis_name, index: @index)
   end
@@ -490,6 +524,19 @@ class CAFrame
   # and index are untouched; the selected cells of every column go to UNDEF.
   # The selector is forwarded to the column indexer, which classifies it.
   private def mask_rows(selector)
+    # Decide before changing anything. A read-only column (a categorical, whose
+    # codes are read-only) refuses the write, and refusing half way through
+    # would leave the frame masked in whichever columns happened to come first.
+    refusing = @columns.find { |_, col| col.read_only? }
+    if refusing
+      raise "can not modify read-only array: column #{refusing[0].inspect} " \
+            "refuses the write, so no column was masked"
+    end
+    # The selector is classified by the column indexer, which is also what
+    # bound-checks it -- so with no column to forward it to, nothing would.
+    # Build the row mask purely to have the frame's own row axis refuse an
+    # out-of-range row, as every other row form does.
+    selected_row_mask(selector) if @columns.empty?
     @columns.each_value do |col|
       col[selector, *([nil] * (col.ndim - 1))] = UNDEF
     end
@@ -511,7 +558,7 @@ class CAFrame
   end
 
   # df[sel] = other : replace the selected contiguous span with other's rows
-  # (any length). Columns are concatenated head + other + tail, so dtypes
+  # (any length). Columns are concatenated head + other + tail, so data types
   # promote and the row count shifts by other.nrow - span.
   private def splice_rows(selector, other)
     lo, hi = contiguous_span(selector)
@@ -536,9 +583,9 @@ class CAFrame
         case pieces.size
         when 0 then col[CArray.int32(0), *tail]   # replaced every row with none
         when 1 then pieces.first
-        else CArray.meld(pieces, axis: 0)         # CAMeld view; dtype mismatch
+        else CArray.meld(pieces, axis: 0)         # CAMeld view; data type mismatch
         end                                        # across pieces raises.
-                                                   # For dtype conversion, cast
+                                                   # For data type conversion, cast
                                                    # the incoming +other+'s
                                                    # column beforehand — silent
                                                    # promotion in a splice would
@@ -547,7 +594,16 @@ class CAFrame
     new_index = splice_index(other, lo, hi)
     @columns  = new_cols
     @index    = new_index
-    @nrow     = @nrow - (hi - lo) + other.nrow
+    # Read the new count off something the frame now holds, rather than
+    # computing it. For every column the pieces welded above are head (lo) +
+    # other's rows + tail (nrow - hi), so the arithmetic and the objects agree
+    # -- except when there is nothing to weld: a frame with no columns and no
+    # index would otherwise come away claiming rows that nothing backs.
+    @nrow = if (witness = new_cols.each_value.first || new_index)
+              witness.shape[0]
+            else
+              0
+            end
     self
   end
 
@@ -566,7 +622,7 @@ class CAFrame
     case pieces.size
     when 0 then @index[CArray.int32(0)]
     when 1 then pieces.first
-    else CArray.meld(pieces, axis: 0)   # index dtype must match across frames
+    else CArray.meld(pieces, axis: 0)   # index data type must match across frames
     end
   end
 
@@ -616,8 +672,11 @@ class CAFrame
   end
 
   private def select_rows(selector, keep_masked: false)
-    if keep_masked && selector.is_a?(CArray) &&
-       selector.data_type == :boolean && selector.has_mask?
+    if keep_masked && selector.is_a?(CArray) && selector.data_type == :boolean
+      # Not gated on selector.has_mask?: gating there would make the same call
+      # site return a view-frame or a materialized one depending on whether that
+      # run's data happened to produce an undetermined cell (the reason splice
+      # always snapshots -- see CAFrame.md section 3).
       return select_rows_keep_masked(selector)
     end
     cols = {}
@@ -638,7 +697,12 @@ class CAFrame
       g[undet_kept, *tail] = UNDEF
       cols[name] = g
     end
-    new_index = @index ? @index[keep] : nil
+    # The index is copied along with the columns. Its values are carried over
+    # unmasked (an undetermined row keeps its label, as mask_rows leaves the
+    # index alone), but a materialized frame has to be materialized whole: an
+    # aliased index would write through to the parent from a frame whose columns
+    # do not.
+    new_index = @index ? @index[keep].copy : nil
     CAFrame.new(cols, axis_name: @axis_name, index: new_index)
   end
 

data/lib/carray/frame/group.rb

@@ -13,7 +13,7 @@ class CAFrame
     raise ArgumentError, "group_by needs at least one key" if keys.empty?
     cat  = grouping_categorical(keys)
     axis = keys.size == 1 && keys.first.is_a?(String) ? keys.first : "group"
-    GroupedFrame.new(self, cat, axis)
+    GroupedFrame.new(self, cat, axis, keys.grep(String))
   end
 
   # Number of rows currently selected — used by group per-group view-frames
@@ -28,7 +28,14 @@ class CAFrame
       # values, categorized by content (the codes are composed from the
       # per-column keys).
       n = nrow
-      CArray.object(n) { |i| cols.map { |c| c[i] } }.categorize
+      key = CArray.object(n) { |i| cols.map { |c| c[i] } }
+      # One undetermined component makes the whole tuple undetermined, the same
+      # answer a single masked key cell gets. Left as a value, the UNDEF inside
+      # the tuple would intern as an ordinary distinct key and the row would
+      # form a group of its own.
+      undetermined = CArray.boolean(n) { |i| key[i].any? { |v| UNDEF.equal?(v) } }
+      key[undetermined] = UNDEF if undetermined.any
+      key.categorize
     end
   end
 
@@ -56,10 +63,14 @@ end
 #   aggregate    -> declarative per-column reductions into a new frame
 #   table { |g| }-> cross-column Ruby escape, g is a per-group view-frame
 class GroupedFrame
-  def initialize(frame, cat, axis_name)
+  # +key_names+ are the frame columns the grouping was keyed on. They become
+  # the result's index, so the reduction shortcuts must not also return them as
+  # reduced columns; an external CArray key contributes no name.
+  def initialize(frame, cat, axis_name, key_names = [])
     @frame     = frame
     @cat       = cat
     @axis_name = axis_name
+    @key_names = key_names
     @labels    = cat.labels          # group values, in code order
   end
 
@@ -124,6 +135,23 @@ class GroupedFrame
 
   # Convenience reductions over every numeric scalar column (memo §6-4
   # "grp.mean"). Non-numeric / N-D columns are skipped.
+  #
+  # @!method sum
+  #   Returns a frame of the per-group sum of every numeric one-dimensional
+  #   column. Non-numeric and multi-dimensional columns are left out.
+  #   @return [CAFrame] one row per group, indexed by the group labels.
+  # @!method mean
+  #   Returns a frame of the per-group arithmetic mean of every numeric
+  #   one-dimensional column, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method min
+  #   Returns a frame of the per-group minimum of every numeric
+  #   one-dimensional column, as {#sum} does.
+  #   @return [CAFrame] one row per group.
+  # @!method max
+  #   Returns a frame of the per-group maximum of every numeric
+  #   one-dimensional column, as {#sum} does.
+  #   @return [CAFrame] one row per group.
   [:sum, :mean, :min, :max].each do |red|
     define_method(red) { reduce_numeric(red) }
   end
@@ -143,6 +171,11 @@ class GroupedFrame
   private def reduce_numeric(reduction)
     cols = {}
     @frame.variable_names.each do |name|
+      # A key column is the index here, not a result column. Filtering it out
+      # by name rather than by data type is what makes a numeric key behave
+      # like a string one: NON_NUMERIC is about which columns a reduction can
+      # apply to, which happened to cover string keys and nothing else.
+      next if @key_names.include?(name)
       col = @frame[name]
       next unless col.ndim == 1 && !NON_NUMERIC.include?(col.data_type)
       cols[name] = col.group_by_category(@cat).public_send(reduction)