carray 3.0.4 → 3.0.6

data/lib/carray/data_type_extension.rb

@@ -164,8 +164,15 @@ class CArray
       #  midpoints go the same way as numpy's integer linspace).
       float_out = CArray.new(CA_FLOAT64, [n]).span(x1.to_f..x2.to_f)
       return float_out if data_type == CA_FLOAT64
+      # to_type masks a cell the type cannot hold; linspace refuses it
+      # instead, as arange does.
       # coerce: the type the caller asked for
-      float_out.floor.to_type(data_type)
+      out = float_out.floor.to_type(data_type)
+      if out.any_masked?
+        raise RangeError,
+              "linspace: #{x1.inspect}..#{x2.inspect} is out of range for #{out.data_type_name}"
+      end
+      out
     end
 
     # @overload arange(stop)

data/lib/carray/fixlen_string.rb

@@ -32,9 +32,11 @@ class CArray
   # zero-copy, other string-bearing arrays materialise, numeric is rejected.
   # @overload fixlen_string(values, bytes: nil, truncate: :error)
   #   @param values [Array<String, nil>] source values.
-  #   @param bytes [Integer, nil] slot width; defaults to the max bytesize.
+  #   @param bytes [Integer, nil] slot width, 1 or more; defaults to the
+  #     max bytesize (1 when every string is empty).
   #   @param truncate [Symbol] `:error` or `:silent`.
   #   @return [CAFixlenString]
+  #   @raise [ArgumentError] when `bytes` is given below 1.
   # @overload fixlen_string(ca, bytes: nil, truncate: :error)
   #   @param ca [CArray] a String Face, CA_OBJECT, or raw CA_FIXLEN array.
   #   @return [CAFixlenString]
@@ -63,8 +65,11 @@ class CArray
       values = arg.to_a
     end
 
+    unless bytes.nil? || (bytes.is_a?(Integer) && bytes >= 1)
+      raise ArgumentError, "CArray.fixlen_string: invalid bytes: #{bytes.inspect} (a cell needs bytes: of 1 or more)"
+    end
     width = bytes || values.compact.map { |s| s.to_s.bytesize }.max || 1
-    width = 1 if width < 1
+    width = 1 if width < 1   # every string empty: the narrowest cell
 
     if truncate == :error
       values.each_with_index do |s, i|
@@ -98,4 +103,38 @@ class CAFixlenString
 
   include CArray::StringOperationMixin
   include CArray::StringOperationMixin::Mutable
+
+  # ---- order family beyond the fixlen kernels --------------------------
+  #
+  # min / max / sort run on the cells' bytes.  The members those kernels do
+  # not cover answer here: minmax as the pair of them, and the running
+  # extremes through the strings ({#to_string}), as CAString answers them,
+  # written back at this width.
+
+  # @return [Array(String, String), Array(CAFixlenString, CAFixlenString)]
+  #   `[min, max]`, with the keywords of {CArray#min} / {CArray#max}.
+  def minmax (*args, **kw)
+    [min(*args, **kw), max(*args, **kw)]
+  end
+
+  # @return [CAFixlenString] the running maximum, as {CArray#cummax}.
+  def cummax (*args, **kw)
+    to_string.cummax(*args, **kw).to_fixlen_string(bytes: bytes)
+  end
+
+  # @return [CAFixlenString] the running minimum, as {CArray#cummin}.
+  def cummin (*args, **kw)
+    to_string.cummin(*args, **kw).to_fixlen_string(bytes: bytes)
+  end
+
+  # A String has no distance, so there is no nearest one; refused in the
+  # words CAString uses.
+  def search_nearest (*args, **kw)
+    to_string.search_nearest(*args, **kw)
+  end
+
+  # @see #search_nearest
+  def search_nearest_addr (*args, **kw)
+    to_string.search_nearest_addr(*args, **kw)
+  end
 end

data/lib/carray/frame/column_verbs.rb

@@ -3,22 +3,32 @@
 class CAFrame
 
   # The one rule of a frame-wide verb over the frame's columns -- the group
-  # reductions and transforms, the windowed reductions -- kept in one place
-  # so that every verb answers in the same shape:
+  # reductions and transforms, the windowed reductions, and the frame's own
+  # reductions and running values -- kept in one place: which columns a verb
+  # computes, and what a named column without the operation does.
   #
-  # - given names, the CArray (one) or the Array of them (several), as
-  #   +df[...]+ returns columns; a named column whose type has no such
-  #   operation is refused with CArray::DataTypeError, as the core refuses
-  #   it, since it was asked for;
+  # - given names, those columns, as the CArray (one) or a Hash of name to
+  #   result (several); a named column whose type has no such operation is
+  #   refused with CArray::DataTypeError, as the core refuses it, since it
+  #   was asked for;
   # - given none, every column whose type has it, the key columns left out,
-  #   as a Hash of name to result that the caller makes a frame of.
+  #   as a Hash of name to result.
+  #
+  # The caller decides the shape of the answer from these: a verb whose
+  # answers lie on the frame's rows (a transform, a window at the rows,
+  # +keep_rows:+) hands one named column back as it is, as +df[name]+
+  # returns it, and several as a frame on the frame's rows, while a verb
+  # with one row per group, per point or for the whole frame makes a frame
+  # of them, labelled by its rows.
   #
   # Whether a type has the operation is asked of the core on two copies of
   # the column's first row (two blank cells of its kind when it has no
   # rows), so the answer is about the type and not the data: a refusal
   # there means the operation does not apply, while an error in the verb
-  # itself is left to raise. An N-D column (each row a profile) is asked,
-  # and computed, along the rows.
+  # itself is left to raise. An object column is the exception: its type
+  # says nothing of what it holds, so one with no present value is left out
+  # of a verb run over every column. An N-D column (each row a profile) is
+  # asked, and computed, along the rows.
   module ColumnVerbs
 
     private
@@ -29,21 +39,36 @@ class CAFrame
         @frame.column_names.each do |name|
           next if key_names.include?(name)
           col = @frame[name]
+          next if valueless_object_column?(col, verb, args)
           next unless column_has?(col, verb, args, keywords)
-          cols[name] = yield(col)
+          cols[name] = yield(col, name)
         end
         return cols
       end
-      results = names.map do |name|
-        col = @frame[@frame.__send__(:column_key, name, verb)]
+      # A column named twice would be one result column twice over.
+      keys = CAFrame.__send__(:refuse_repeated_names,
+                              names.map { |name| @frame.__send__(:column_key, name, verb) }, verb)
+      results = keys.map do |key|
+        name = key
+        col = @frame[key]
         unless column_has?(col, verb, args, keywords)
           raise CArray::DataTypeError,
                 "#{verb}: column #{name.inspect} " \
                 "(#{CArray::Inspector.type_label(col)}) has no #{verb}"
         end
-        yield(col)
+        yield(col, key)
       end
-      results.size == 1 ? results.first : results
+      results.size == 1 ? results.first : keys.zip(results).to_h
+    end
+
+    # An object column with no present cell says nothing of what it holds,
+    # so a verb run over every column leaves it out, as select leaves it out
+    # of every kind; counting the present cells applies to any column.
+    def valueless_object_column? (col, verb, args)
+      return false if col.face? || col.data_type != CA_OBJECT
+      return false if %i[count_not_masked count_masked].include?(verb)
+      return false if verb == :count && args.empty?
+      col.count_not_masked == 0
     end
 
     # count with no value counts the present cells, which every type has;
@@ -53,10 +78,17 @@ class CAFrame
       verb = :count_not_masked if verb == :count && args.empty?
       # Two cells: one cell does not reach what combines two (a String's
       # median interpolates, its sum adds), and an empty column answers
-      # with the identity without asking the type at all.
+      # with the identity without asking the type at all. The row is the
+      # first that holds a present cell, so a masked first row does not
+      # answer with the identity for a column whose values have none.
       tail = [nil] * (col.ndim - 1)
+      row = 0
+      if col.has_mask? && col.elements > 0
+        present = col.is_not_masked.flatten.where
+        row = present[0] / (col.elements / col.shape[0]) if present.elements > 0
+      end
       probe = if col.shape[0] > 0
-                col[CA_INT64([0, 0]), *tail]
+                col[CA_INT64([row, row]), *tail]
               else
                 col.resize(2, *col.shape[1..])
               end

data/lib/carray/frame/concat.rb

@@ -20,9 +20,9 @@ class CAFrame
   # preserved: writes to the result flow back to whichever input frame owns
   # the target segment, and vice versa).
   #
-  # Per-column +data_type+ must match across frames (CArray.meld is a view
-  # constructor that refuses to auto-cast — silent promotion here would
-  # hide schema drift).  For an eager, auto-casting alternative use
+  # Per-column +data_type+ must match across frames: CArray.meld would bring
+  # the pieces to a common type, and in a frame that promotion would hide
+  # schema drift.  For an eager, auto-casting alternative use
   # {CAFrame.concatenate}.
   #
   # Column matching is by name (output order follows the first frame);
@@ -54,6 +54,7 @@ class CAFrame
     frames = frames_with_rows(frames)
     cols = {}
     names.each do |name|
+      check_same_column_type(frames.map { |f| f[name] }, name, verb: "meld")
       cols[name] = CArray.meld(frames.map { |f| f[name] }, axis: 0)
     end
     new(cols, axis_name: first.axis_name, index: meld_index(frames))
@@ -229,6 +230,19 @@ class CAFrame
   end
   private_class_method :check_trailing_shapes
 
+  # The pieces of one column have one data type (and one class, for a
+  # Face): a frame does not promote a column to make its pieces agree.
+  def self.check_same_column_type(pieces, name, verb:)
+    first = pieces.first
+    pieces.each_with_index do |c, i|
+      next if CArray::Inspector.type_label(c) == CArray::Inspector.type_label(first)
+      raise ArgumentError,
+            "#{verb}: column #{name.inspect} is #{CArray::Inspector.type_label(c)} in piece #{i}, " \
+            "#{CArray::Inspector.type_label(first)} in piece 0 (cast it first, or use concatenate)"
+    end
+  end
+  private_class_method :check_same_column_type
+
   # Two indexes hold the same rows when their values agree cell by cell; a
   # NaN label matches a NaN label, and a masked cell a masked cell.
   def self.same_index_values?(a, b)

data/lib/carray/frame/convert.rb

@@ -26,6 +26,25 @@ class CAFrame
     each_row.map { |row| row.transform_values { |v| record_value(v) } }
   end
 
+  # The rows as an Array of Arrays, one value per column in column order --
+  # the table as plain Ruby data, for a library that takes rows (anything
+  # that calls +to_a+ on what it is given). A cell is what the column's own
+  # +to_a+ gives: a masked cell is +UNDEF+, a time column's cell its element,
+  # and an N-D column's cell an Array. The index is a row label, not a value,
+  # and is left out; +reset_index+ first to have it as the first column.
+  #
+  # Masked cells are not turned into anything: decide what they should be
+  # before handing the rows over, with +strip_mask+ or +drop_masked+.
+  #
+  #   df.drop_masked.to_a
+  #   df.strip_mask("temp" => Float::NAN).to_a
+  #
+  # @return [Array<Array>]
+  def to_a
+    return Array.new(@nrow) { [] } if @columns.empty?
+    return @columns.values.map(&:to_a).transpose
+  end
+
   # Hand the frame over as a CArray with the minimum work (memo §11.9): a
   # 2-D **view** of shape (nrow, ncol), one per column in column
   # order (§12-C). Nothing is materialised — the result is a +CAStack+ over

data/lib/carray/frame/csv_parser.rb

@@ -1,4 +1,4 @@
-# A small, self-contained CSV tokenizer for CAFrame.from_csv.
+# A small, self-contained CSV tokenizer for CAFrame.read_csv.
 #
 # It produces raw String cells and does no type inference — casting is a
 # separate step. The tokenizer is tuned for the
@@ -22,14 +22,14 @@
 # Two entry points share the Tokenizer:
 #   * CSVParser.parse / parse_file -- whole-file parse into [headers, rows].
 #   * CSVReader                    -- the block reading-control DSL used by
-#                                     CAFrame.from_csv (header / skip / body /
+#                                     CAFrame.read_csv (header / skip / body /
 #                                     column_names).
 
 require "strscan"
 require "stringio"
 
 class CAFrame
-  # CSV tokenizer behind `CAFrame.from_csv`.  It produces raw String cells
+  # CSV tokenizer behind `CAFrame.read_csv`.  It produces raw String cells
   # and does no type inference — casting is a separate step.
   #
   # Records with no quote character take a `String#split` fast path; only
@@ -79,7 +79,7 @@ class CAFrame
 
     # Parse an IO (or anything answering +gets+). The first record supplies the
     # headers; the rest are data rows. Blank lines are skipped, including in a
-    # one-column file (from_csv reads them there as missing cells).
+    # one-column file (read_csv reads them there as missing cells).
     def parse(io, sep: ",", quote: '"', strip: false)
       tok = Tokenizer.new(sep, quote, strip)
       headers = nil
@@ -315,24 +315,24 @@ class CAFrame
     end
   end
 
-  # The reading-control block for CAFrame.from_csv. A file often has
+  # The reading-control block for CAFrame.read_csv. A file often has
   # preamble lines, a units row, or no header at all; the block is given the
   # reader and says, in order, how to consume the stream:
   #
-  #   CAFrame.from_csv(path) do |r|
+  #   CAFrame.read_csv(path) do |r|
   #     r.skip 3          # drop 3 preamble lines
   #     r.header          # next record supplies the column names
   #     r.skip 1          # drop a units row
   #     r.data            # the rest are data rows
   #   end
   #
-  #   CAFrame.from_csv(path) { it.column_names "date", "temp", "rh"; it.data }
+  #   CAFrame.read_csv(path) { it.column_names "date", "temp", "rh"; it.data }
   #
   # The verbs are +skip+ / +header+ / +column_names+ / +columns+ / +data+;
   # each returns a value useful inline (header returns its fields) and the ordering is the
   # script. The reader is a parameter, not self, so a local variable named
   # +data+ or +header+ cannot stand in for the verb. Without a block,
-  # from_csv reads by its header: / data: / column_names: (see +layout+).
+  # read_csv reads by its header: / data: / column_names: (see +layout+).
   class CSVReader
     def initialize(io, sep: ",", quote: '"', strip: false, read_text_as_const_string: false)
       @read_text_as_const_string = read_text_as_const_string
@@ -388,7 +388,7 @@ class CAFrame
         key = name.to_s
         @named_headers ||= {}
         if @named_headers.key?(key)
-          raise ArgumentError, "from_csv: header(#{name.inspect}) is read twice"
+          raise ArgumentError, "read_csv: header(#{name.inspect}) is read twice"
         end
         @named_headers[key] = [fields, @tok.record_line, name]
       end
@@ -403,7 +403,7 @@ class CAFrame
     def column_names(*names)
       names = names.flatten.map(&:to_s)
       if names.include?("")
-        raise ArgumentError, "from_csv: column_names: a column name can not be empty"
+        raise ArgumentError, "read_csv: column_names: a column name can not be empty"
       end
       if @header_line
         check_column_names(names, @names.size, @header_line)
@@ -431,15 +431,15 @@ class CAFrame
       end
       selection = selection.flatten
       if selection.empty?
-        raise ArgumentError, "from_csv: columns: names no column"
+        raise ArgumentError, "read_csv: columns: names no column"
       end
       selection.each do |c|
         next if c.is_a?(String)
-        CAFrame.__send__(:column_key, c, "from_csv", "columns: a column name") if c.is_a?(Symbol)
+        CAFrame.__send__(:column_key, c, "read_csv", "columns: a column name") if c.is_a?(Symbol)
         next if c.is_a?(Integer) && c >= 0
         next if c.is_a?(Range) && (c.begin.nil? || (c.begin.is_a?(Integer) && c.begin >= 0)) &&
                 (c.end.nil? || c.end.is_a?(Integer))
-        raise ArgumentError, "from_csv: columns: takes column names, or indexes from 0 " \
+        raise ArgumentError, "read_csv: columns: takes column names, or indexes from 0 " \
                              "and Ranges of them (got #{c.inspect})"
       end
       @selection = selection
@@ -450,17 +450,17 @@ class CAFrame
     private def column_renames(map)
       map.each do |key, name|
         if key.is_a?(Range)
-          raise ArgumentError, "from_csv: columns: a Hash selects one column per key, " \
+          raise ArgumentError, "read_csv: columns: a Hash selects one column per key, " \
                                "not a Range (got #{key.inspect})"
         end
-        CAFrame.__send__(:column_key, name, "from_csv", "columns: a column name",
+        CAFrame.__send__(:column_key, name, "read_csv", "columns: a column name",
                          hint: "given for #{key.inspect}")
         if name.empty?
-          raise ArgumentError, "from_csv: columns: a column name can not be empty " \
+          raise ArgumentError, "read_csv: columns: a column name can not be empty " \
                                "(given for #{key.inspect})"
         end
       end
-      CAFrame.__send__(:refuse_repeated_names, map.values, "from_csv", "columns: names")
+      CAFrame.__send__(:refuse_repeated_names, map.values, "read_csv", "columns: names")
       map.values
     end
 
@@ -474,7 +474,7 @@ class CAFrame
           # Without a header the columns are named c0, c1, ... by position.
           j = names ? names.index(c) : c[/\Ac(0|[1-9]\d*)\z/, 1]&.to_i
           if j.nil?
-            raise KeyError, "from_csv: columns: names no column #{c.inspect}" +
+            raise KeyError, "read_csv: columns: names no column #{c.inspect}" +
                             (names ? "" : " (with no header the columns are c0, c1, ...)")
           end
           j
@@ -488,9 +488,9 @@ class CAFrame
           (first..last).to_a
         end
       end
-      CAFrame.__send__(:refuse_repeated_names, indexes, "from_csv", "columns: selects column")
+      CAFrame.__send__(:refuse_repeated_names, indexes, "read_csv", "columns: selects column")
       if ncol && (bad = indexes.find { |j| j >= ncol })
-        raise ArgumentError, "from_csv: columns: #{bad} is past the #{ncol} columns of the file"
+        raise ArgumentError, "read_csv: columns: #{bad} is past the #{ncol} columns of the file"
       end
       indexes
     end
@@ -503,7 +503,7 @@ class CAFrame
 
     private def check_column_names(names, ncol, line)
       return if names.size == ncol
-      raise ArgumentError, "from_csv: column_names: gives #{names.size} " \
+      raise ArgumentError, "read_csv: column_names: gives #{names.size} " \
                            "name#{names.size == 1 ? '' : 's'} for the #{ncol} columns of line #{line}"
     end
 
@@ -525,7 +525,7 @@ class CAFrame
       read_data(nil)
     end
 
-    # The header: / data: / column_names: of from_csv, as the verbs above.
+    # The header: / data: / column_names: of read_csv, as the verbs above.
     # They index the lines of the file from 0, as File.readlines does (an
     # error names a line from 1, as an editor does). header: defaults to 0,
     # or to none (false) when column_names: is given; data: is the first line
@@ -534,12 +534,12 @@ class CAFrame
     def layout(header: nil, data: nil, column_names: nil, columns: nil, attrs: nil)
       header = column_names ? false : 0 if header.nil?
       unless header == false || (header.is_a?(Integer) && header >= 0)
-        raise ArgumentError, "from_csv: header: takes the index of a line, from 0, " \
+        raise ArgumentError, "read_csv: header: takes the index of a line, from 0, " \
                              "or false (got #{header.inspect})"
       end
-      attr_keys = attrs.nil? ? [] : CAFrame.send(:csv_attr_keys, attrs, "from_csv")
+      attr_keys = attrs.nil? ? [] : CAFrame.send(:csv_attr_keys, attrs, "read_csv")
       if !header && !attr_keys.empty?
-        raise ArgumentError, "from_csv: attrs: reads the lines below the header, " \
+        raise ArgumentError, "read_csv: attrs: reads the lines below the header, " \
                              "so it needs a header line"
       end
       first, last = CSVReader.data_lines(data)
@@ -552,7 +552,7 @@ class CAFrame
       self.columns(*(columns.is_a?(Array) ? columns : [columns])) if columns
       first ||= @tok.lineno
       if first < @tok.lineno
-        raise ArgumentError, "from_csv: data: #{first} does not come after header: #{header}" +
+        raise ArgumentError, "read_csv: data: #{first} does not come after header: #{header}" +
                              (attr_keys.empty? ? "" : " and the #{attr_keys.size} attrs: lines")
       end
       skip(first - @tok.lineno)
@@ -570,15 +570,15 @@ class CAFrame
           e -= 1 if e.is_a?(Integer) && spec.exclude_end?
           [spec.begin, e]
         else
-          raise ArgumentError, "from_csv: data: takes the index of a line or a Range of " \
+          raise ArgumentError, "read_csv: data: takes the index of a line or a Range of " \
                                "them, from 0 (got #{spec.inspect})"
         end
       [first, last].each do |n|
         next if n.nil? || (n.is_a?(Integer) && n >= 0)
-        raise ArgumentError, "from_csv: data: lines are indexed from 0 (got #{spec.inspect})"
+        raise ArgumentError, "read_csv: data: lines are indexed from 0 (got #{spec.inspect})"
       end
       if first && last && last < first
-        raise ArgumentError, "from_csv: data: #{spec.inspect} ends before it starts"
+        raise ArgumentError, "read_csv: data: #{spec.inspect} ends before it starts"
       end
       [first, last]
     end
@@ -648,7 +648,7 @@ class CAFrame
     end
 
     # The columns read_body_into_const_string_columns made, as +result+
-    # hands them to CAFrame.from_csv.
+    # hands them to CAFrame.read_csv.
     ConstStringColumns = Struct.new(:columns)
 
     private def body_in_c(last = nil, chunk_bytes = CHUNK_BYTES)
@@ -767,7 +767,7 @@ class CAFrame
       attrs = indexes.map { {} }
       @named_headers.each do |key, (fields, line, name)|
         if fields.size > ncol
-          raise ArgumentError, "from_csv: header(#{name.inspect}) on line #{line} " \
+          raise ArgumentError, "read_csv: header(#{name.inspect}) on line #{line} " \
                                "has #{fields.size} fields for the #{ncol} columns"
         end
         indexes.each_with_index do |j, k|
@@ -784,7 +784,7 @@ class CAFrame
       @names ? @names.size : ((@rows || []).map(&:size).max || 0)
     end
 
-    # [names_or_nil, rows] for CAFrame.from_csv to build from. names is nil when
+    # [names_or_nil, rows] for CAFrame.read_csv to build from. names is nil when
     # neither header nor column_names ran (positional names are generated).
     def result
       return table_result if @table

data/lib/carray/frame/describe.rb

@@ -30,10 +30,12 @@ class CAFrame
   # column with no present cell has UNDEF statistics. The index is not a
   # column and is not summarized.
   #
-  # @param names [Array<String>] the columns to summarize; default all.
+  # @param names [Array<String, Range, Regexp, Symbol>] the columns to
+  #   summarize, picked as +select+ picks them (a Symbol is a kind of column,
+  #   such as :numeric); default all.
   # @return [CAFrame] a new frame with one row per column.
   def describe(*names)
-    names = names.empty? ? column_names : column_keys(names, "describe")
+    names = names.empty? ? column_names : names_selected(names, "describe")
     rows = names.map { |name| describe_column(self[name]) }
     stats = %w[type count masked unique min max mean stddev]
     cols = stats.to_h do |stat|