carray 3.0.0 → 3.0.2

data/lib/carray/frame/io.rb

@@ -11,12 +11,27 @@ class CAFrame
   # later. Broken cells fail to_type and become UNDEF automatically
   # (parse-mask, §6-2).
   #
+  # +source+ is a path, or an open IO -- anything answering +gets+, which a
+  # StringIO is. So CSV that is already in memory does not have to go to a
+  # temporary file first:
+  #
+  #   CAFrame.from_csv("obs.csv")                    # a path
+  #   CAFrame.from_csv(StringIO.new(body))           # text already in hand
+  #   File.open("obs.csv") { |io| CAFrame.from_csv(io) }
+  #
+  # A String is always read as a path, never as CSV text: guessing between the
+  # two by looking for a newline is the kind of guess that is right until it is
+  # not, and StringIO says which one you meant in six characters. An IO is read
+  # from where it is and left open -- the caller opened it and closes it.
+  #
   # Parsing uses the built-in fast tokenizer (CSVParser). Options:
   #   sep:      field separator (default ",")
   #   quote:    quote character (default '"')
   #   strip:    trim spaces from unquoted fields (default false, RFC spacing)
-  #   +encoding+: IO open-mode encoding (default "bom|utf-8", strips a BOM)
-  #   parser:   a callable path -> [headers, rows] to inject another parser
+  #   +encoding+: open-mode encoding for a path (default "bom|utf-8", strips a
+  #             BOM). It has nothing to open when +source+ is an IO, so there
+  #             the IO's own encoding governs and a BOM is the caller's.
+  #   parser:   a callable source -> [headers, rows] to inject another parser
   #             (e.g. the stdlib +csv+, or a typed-table source); when given,
   #             sep/quote/strip/encoding and any block are that parser's concern.
   #
@@ -28,25 +43,25 @@ class CAFrame
   #     skip 2; header; skip 1; body
   #   end
   #
+  # A missing field -- an unquoted empty one, or a cell a short row never
+  # reached -- is UNDEF in the frame, whether or not the column is cast by
+  # +types:+. A quoted empty field ("") is the empty string, which is a value.
+  # So the mask +to_csv+ writes comes back as a mask.
+  #
   # Columns are handed to the frame as CABlock views over one backing object
   # array (§3.6 view-by-default); casting a column materializes it, and +copy+
   # gives an independent frame.
-  def self.from_csv(path, types: nil,
+  def self.from_csv(source, types: nil,
                     sep: ",", quote: '"', strip: false,
                     encoding: "bom|utf-8", parser: nil, &block)
     names, rows =
       if parser
-        parser.call(path)
+        parser.call(source)
+      elsif source.respond_to?(:gets)
+        read_csv(source, sep: sep, quote: quote, strip: strip, &block)
       else
-        File.open(path, "r:#{encoding}") do |io|
-          reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip)
-          if block
-            block.arity == 1 ? block.call(reader) : reader.instance_exec(&block)
-          else
-            reader.header
-            reader.body
-          end
-          reader.result
+        File.open(source, "r:#{encoding}") do |io|
+          read_csv(io, sep: sep, quote: quote, strip: strip, &block)
         end
       end
 
@@ -55,6 +70,22 @@ class CAFrame
     frame
   end
 
+  # Drive the reading-control DSL over one open IO and hand back
+  # [names, rows]. Shared by the path and the IO source, so the two cannot
+  # come to read a file differently.
+  def self.read_csv (io, sep:, quote:, strip:, &block)
+    reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip)
+    if block
+      block.arity == 1 ? block.call(reader) : reader.instance_exec(&block)
+    else
+      reader.header
+      reader.body
+    end
+    reader.result
+  end
+
+  private_class_method :read_csv
+
   # Build a frame from parsed [names, rows]. When names is nil (headerless and
   # no column_names) positional names "c0".."cN" are generated from the widest
   # row. Rows are squared off to the column count (short rows padded with nil,
@@ -62,8 +93,8 @@ class CAFrame
   # is a view into it (§3.6).
   # Write the frame as CSV. CSV is a flat table of scalar cells, so this is the
   # text form of the same all-scalar subset +to_ca+ requires (§11.9): every
-  # column must be 1-D. Unlike +to_ca+ it does not promote to a common dtype --
-  # each column is formatted to text independently, so mixed dtypes (numbers,
+  # column must be 1-D. Unlike +to_ca+ it does not promote to a common data type --
+  # each column is formatted to text independently, so mixed data types (numbers,
   # strings, datetime / categorical Faces) sit side by side. An N-D column has
   # no flat CSV cell and raises; export it per column, or use +to_records+ +
   # JSON for the structured shape (memo §11.9, the N-D escape).
@@ -115,6 +146,177 @@ class CAFrame
     end
   end
 
+  # Render the frame as an aligned text table for reading:
+  #
+  #   puts df.to_table
+  #
+  #   time        temp  station
+  #   ----------  ----  -------
+  #   2026-01-01   1.5  Tokyo
+  #   2026-01-02     _  Osaka
+  #
+  # This is the display counterpart of +to_csv+ and shares none of its
+  # constraints: it is text meant to be looked at, not read back. Numeric
+  # columns are right-aligned, everything else left-aligned; a masked cell
+  # shows as +_+, the same marker CArray's own inspect uses. An N-D column
+  # (which +to_csv+ rejects, having no flat cell) shows each row's slice as
+  # an Array literal.
+  #
+  # Float cells are rounded to +precision+ decimal places for display only
+  # (default 6); +precision: nil+ prints them at full precision, which is
+  # faithful but lets one long value set the column width.
+  #
+  # Long frames are truncated in the middle: +rows+ caps how many rows are
+  # printed (default 20, split evenly around an ellipsis row), and
+  # +rows: nil+ prints every row. +index: false+ drops the index column.
+  def to_table(rows: 20, index: true, precision: 6)
+    head = rows && (rows + 1) / 2
+    render_table(head: head, tail: rows && rows - head,
+                 index: index, precision: precision, footer: true)
+  end
+
+  # +to_s+ is the whole frame, +inspect+ the middle-elided one -- so +puts df+
+  # dumps everything and +p df+ stays a screenful. +inspect+ leads with the
+  # same summary line it always had (nrow, variable data types, index), so the
+  # table under it needs no row-count footer.
+  def to_s
+    to_table(rows: nil)
+  end
+
+  # The summary line (nrow, variable data types, index) followed by the
+  # middle-elided table.
+  # @return [String]
+  def inspect
+    parts = @columns.map { |k, v| "#{k}:#{v.data_type}#{v.ndim > 1 ? v.shape[1..].inspect : ''}" }
+    idx = @index ? " index=#{@axis_name.inspect}" : ""
+    head = "#<CAFrame nrow=#{@nrow} vars=[#{parts.join(', ')}]#{idx}>"
+    # The table counts the index as a column, so a frame whose only data is its
+    # index has one to show.  Gate on the same thing render_table does.
+    return head if @columns.empty? && @index.nil?
+    head + "\n" + render_table(head: 8, tail: 2, index: true, precision: 6,
+                               footer: false)
+  end
+
+  private def render_table(head:, tail:, index:, precision:, footer:)
+    names   = []
+    columns = []
+    aligns  = []
+
+    if index && @index
+      names   << @axis_name
+      columns << @index
+      aligns  << (@index.numeric? ? :right : :left)
+    end
+    @columns.each do |name, col|
+      names   << name
+      columns << col
+      aligns  << (col.ndim == 1 && col.numeric? ? :right : :left)
+    end
+    return "" if names.empty?
+
+    positions = table_row_positions(head, tail)
+    body = positions.map do |i|
+      if i.nil?
+        Array.new(columns.size, ":") # vertical ellipsis for the elided middle
+      else
+        columns.map { |col| format_table_cell(col, i, precision) }
+      end
+    end
+
+    widths = names.each_with_index.map do |name, j|
+      [display_width(name), *body.map { |cells| display_width(cells[j]) }].max
+    end
+
+    out = +""
+    out << table_row(names, widths, aligns) << "\n"
+    out << table_row(widths.map { |w| "-" * w }, widths, aligns) << "\n"
+    body.each { |cells| out << table_row(cells, widths, aligns) << "\n" }
+    if footer && positions.size - positions.count(nil) < @nrow
+      out << "(#{plural(@nrow, 'row')}, #{plural(@columns.size, 'variable')})\n"
+    end
+    out
+  end
+
+  # Row indices to print, with nil marking the elided middle. A nil head means
+  # no cap; a frame that already fits in head + tail is listed whole.
+  private def table_row_positions(head, tail)
+    return (0...@nrow).to_a if head.nil? || @nrow <= head + tail
+    (0...head).to_a + [nil] + ((@nrow - tail)...@nrow).to_a
+  end
+
+  private def table_row(cells, widths, aligns)
+    line = cells.each_with_index.map do |text, j|
+      pad = " " * (widths[j] - display_width(text))
+      aligns[j] == :right ? pad + text : text + pad
+    end.join("  ")
+    line.rstrip
+  end
+
+  # Column widths are counted in terminal cells, not characters: a CJK
+  # ideograph, kana, or full-width form occupies two cells, so counting
+  # characters would leave every column holding such a name ragged. The
+  # ranges below are the East Asian Wide / Fullwidth blocks; a combining
+  # mark takes no cell of its own.
+  WIDE_CHAR_RANGES = [
+    0x1100..0x115F, 0x2E80..0x303E, 0x3041..0x33FF, 0x3400..0x4DBF,
+    0x4E00..0x9FFF, 0xA000..0xA4CF, 0xA960..0xA97F, 0xAC00..0xD7A3,
+    0xF900..0xFAFF, 0xFE10..0xFE19, 0xFE30..0xFE6F, 0xFF00..0xFF60,
+    0xFFE0..0xFFE6, 0x1F300..0x1F64F, 0x1F900..0x1F9FF, 0x20000..0x3FFFD,
+  ].freeze
+  private_constant :WIDE_CHAR_RANGES
+
+  COMBINING_RANGES = [0x0300..0x036F, 0x1AB0..0x1AFF, 0x20D0..0x20F0].freeze
+  private_constant :COMBINING_RANGES
+
+  private def display_width(text)
+    text.each_char.sum do |ch|
+      cp = ch.ord
+      if COMBINING_RANGES.any? { |r| r.cover?(cp) }
+        0
+      elsif WIDE_CHAR_RANGES.any? { |r| r.cover?(cp) }
+        2
+      else
+        1
+      end
+    end
+  end
+
+  private def plural(n, noun)
+    "#{n} #{noun}#{n == 1 ? '' : 's'}"
+  end
+
+  # Render one N-D cell the way Array#inspect would, except that a masked
+  # element prints as the table's missing marker rather than as UNDEF.
+  private def format_nested_cell(v)
+    case v
+    when Array then "[" + v.map { |x| format_nested_cell(x) }.join(", ") + "]"
+    else UNDEF.equal?(v) || v.nil? ? "_" : v.inspect
+    end
+  end
+
+  private def format_table_cell(col, i, precision)
+    e = elem_at(col, i)
+    if UNDEF.equal?(e) || e.nil?
+      "_"
+    elsif e.is_a?(Float) && precision
+      # Display rounding only: a full-precision float (141.67833333333334)
+      # sets the column width for every other row and makes the table hard
+      # to read. precision: nil prints the value as Ruby renders it.
+      e.round(precision).to_s
+    elsif e.is_a?(CArray)
+      # An N-D cell renders its elements, and a masked element among them
+      # takes the same marker a masked scalar does -- UNDEF's own inspect
+      # would put a second spelling of "missing" in the same table.
+      format_nested_cell(e.to_a)
+    elsif e.is_a?(String)
+      e
+    elsif e.respond_to?(:iso8601)
+      e.iso8601
+    else
+      e.to_s
+    end
+  end
+
   private def format_csv_column(col)
     col.to_a.map do |e|
       if UNDEF.equal?(e) || e.nil?
@@ -157,6 +359,13 @@ class CAFrame
       end
     end
     table = CArray.object(rows.size, ncol) { rows }
+    # A missing field is UNDEF, not a Ruby nil sitting in a cell.  The
+    # tokenizer says missing with nil (an unquoted empty field; a quoted
+    # one is the empty string and stays a value), and an object array will
+    # hold that nil quite happily -- so a column read without `types:` used
+    # to carry nil where the same column read with one carried UNDEF, and
+    # the mask a to_csv had written did not survive the trip back.
+    table[:eq, nil] = UNDEF
     names.each_with_index { |name, j| cols[name] = table[nil, j] }
     new(cols)
   end

data/lib/carray/frame/records.rb

@@ -59,30 +59,44 @@ class CAFrame
   # native scalar column (nil -> UNDEF); anything else -> object column.
   def self.build_record_column(values, n)
     present = values.reject(&:nil?)
-    return CArray.object(n) { values } if present.empty?
+    return mask_missing(CArray.object(n) { values }) if present.empty?
 
     if present.all? { |v| v.is_a?(Array) || v.is_a?(CArray) }
       build_nd_column(values, present, n)
     else
       type = numeric_leaf_type(present)
       col = CArray.object(n) { values }
-      type ? col.to_type(type) : col
+      type ? col.to_type(type) : mask_missing(col)
     end
   end
   private_class_method :build_record_column
 
+  # A missing cell is UNDEF, not a Ruby nil sitting in a cell.  to_records
+  # writes a masked cell as nil, so nil on the way back in is the only
+  # spelling a missing cell has; to_type does this conversion for a numeric
+  # column, and an object column would otherwise keep the nil as a value --
+  # a row with no label coming back as a row labelled nil.  The CSV reader
+  # takes the same position (see build_frame in io.rb).
+  def self.mask_missing(col)
+    col[:eq, nil] = UNDEF
+    col
+  end
+  private_class_method :mask_missing
+
   # Stack equal-length array cells into an (N, L) column via an object 2-D fill
   # + to_type (nil rows -> UNDEF, int/float by leaf). Ragged lengths or
   # non-numeric leaves fall back to a 1-D object column of the raw cells.
   def self.build_nd_column(values, present, n)
     lengths = present.map { |v| v.is_a?(CArray) ? v.shape[0] : v.size }
     len = lengths.first
-    return CArray.object(n) { values } unless lengths.all? { |x| x == len }
+    unless lengths.all? { |x| x == len }
+      return mask_missing(CArray.object(n) { values })
+    end
 
     nested = values.map { |v| v.nil? ? Array.new(len) : (v.is_a?(CArray) ? v.to_a : v) }
     type = numeric_leaf_type(nested.flatten.compact)
     table = CArray.object(n, len) { nested }
-    type ? table.to_type(type) : table
+    type ? table.to_type(type) : mask_missing(table)
   end
   private_class_method :build_nd_column
 

data/lib/carray/frame/sort.rb

@@ -13,7 +13,7 @@
 #
 # sort_addr is ascending, so a descending key is expressed by replacing it with
 # its dense descending rank (CArray#order(descending: true, method: :dense)):
-# this works for every dtype (unlike negation, which cannot reverse a string and
+# this works for every data type (unlike negation, which cannot reverse a string and
 # silently wraps an unsigned integer), and the *dense* rank keeps equal values
 # on one rank so ties fall through to later keys in a multi-key sort.
 # masked_position (CArray.sort_addr's kwarg) places masked key rows first or

data/lib/carray/frame/verbs.rb

@@ -143,11 +143,28 @@ class CAFrame
   # carry, so it raises rather than silently truncate. Make it the index with
   # +set_index+ afterward.
   #
+  # A +CATime::Grid+ carries the same (unit, epoch) pair as one value, so a
+  # netCDF +units+ attribute goes straight in. It also carries a phase the
+  # keyword form cannot: the keyword +epoch+ is read on the +unit+ grid, so
+  # an epoch off that grid ("days since 1980-01-01 12:00") loses its
+  # time-of-day, while a grid resolves the finer storage that holds it.
+  #
   #   df.to_time("time", unit: :h, epoch: "1990-01-01").set_index("time")
-  def to_time(name, unit: :s, epoch: nil)
+  #   df.to_time("time", CATime::Grid.parse("hours since 1990-01-01"))
+  #   df.to_time("time", CATime::Grid.parse("days since 1980-01-01 12:00"))
+  def to_time(name, grid = nil, unit: :s, epoch: nil)
     key = name.to_s
     col = @columns.fetch(key) { raise KeyError, "no column #{key.inspect}" }
     raw = integer_serial_column(col, key)
+    grid = unit if unit.is_a?(CATime::Grid)
+    if grid.is_a?(CATime::Grid)
+      @columns[key] = grid.at(raw)
+      return self
+    end
+    unless grid.nil?
+      raise ArgumentError,
+            "the positional argument must be a CATime::Grid (got #{grid.class})"
+    end
     if epoch
       raw = raw + CArray.time(epoch, unit: unit).ticks[0]
     end
@@ -213,7 +230,9 @@ class CAFrame
       return
     end
     yvalid = col.value.float64[present]
-    col[] = yvalid.linear_fetch(addr).to_type(col.data_type).mask_invalid   # write-through
+    # mask_invalid before the cast: linear_fetch marks out of range with NaN,
+    # and casting back to an integer column would turn it into a plausible 0.
+    col[] = yvalid.linear_fetch(addr).mask_invalid.to_type(col.data_type)  # write-through
   end
 
   private def cast_one(name, type)
@@ -250,16 +269,17 @@ class CAFrame
             "promote takes a data type Symbol (got #{type.class}); " \
             "class-shaped targets are not promotion destinations"
     end
-    @columns.each_key do |key|
-      col = @columns[key]
-      # A Face column answers for itself: :object is its surface values, a
-      # numeric target is whatever it declares in #to_numeric (and a TypeError
-      # naming that method when it declares nothing). result_type has nothing
-      # to say about a surface it cannot read, so the widening check -- which
-      # is about primitive promotion -- applies to plain columns only.
-      refuse_narrowing(key, col, type) unless col.face?
-      @columns[key] = col.to_type(type)
-    end
+    # Check every column before rebinding any. A column that would narrow
+    # rejects the whole promote, and rejecting part way through would leave the
+    # frame promoted in whichever columns happened to come first.
+    #
+    # A Face column answers for itself: :object is its surface values, a
+    # numeric target is whatever it declares in #to_numeric (and a TypeError
+    # naming that method when it declares nothing). result_type has nothing
+    # to say about a surface it cannot read, so the widening check -- which
+    # is about primitive promotion -- applies to plain columns only.
+    @columns.each { |key, col| refuse_narrowing(key, col, type) unless col.face? }
+    @columns.each_key { |key| @columns[key] = @columns[key].to_type(type) }
   end
 
   private def refuse_narrowing(key, col, type)

data/lib/carray/fuse_source.rb

@@ -0,0 +1,123 @@
+# ---------------------------------------------------------------------------
+#  Reading `CArray.fuse { a + b * c }`.
+#
+#  The block is not called.  Its `a` is the array itself, so calling it would
+#  evaluate the expression eagerly -- which is the thing fuse exists to avoid.
+#  The source is read instead, every name that holds a CArray is given `.lazy`,
+#  and the result is evaluated back in the block's own binding, so `self`,
+#  instance variables, methods and constants are what they were.
+#
+#  Ruby has no macro, so the alternative was to pass the arrays in and take
+#  shadows back -- `fuse(a, b) { |x, y| ... }` -- which names each of them
+#  twice.  Julia writes `@.` for the same reason and does the same thing to
+#  the expression underneath.
+# ---------------------------------------------------------------------------
+
+require "prism"
+
+class CArray
+
+  module FuseSource
+
+    # Runtime coercion, so the rewrite never has to work out what a name
+    # holds: anything that is not an array passes through untouched.
+    def self.shadow (value)
+      value.is_a?(CArray) ? value.lazy : value
+    end
+
+    def self.evaluate (block)
+      result = eval(rewrite(body_source(block)), block.binding,
+                    *block.source_location)
+      # An expression that is just an array is that array; the shadow put
+      # around it has nothing to fuse.
+      result.is_a?(CALazyMarker) ? result.parent : result
+    end
+
+    # -- the block's own text ---------------------------------------------
+
+    def self.body_source (block)
+      text = extract(block)
+      wrapped = "proc " + text
+      node = Prism.parse(wrapped).value
+                  .breadth_first_search { |n| n.is_a?(Prism::BlockNode) }
+      inner = node && node.body
+      unless inner
+        raise ArgumentError,
+              "CArray.fuse could not read an expression out of this block"
+      end
+      wrapped.byteslice(inner.location.start_offset...inner.location.end_offset)
+    end
+
+    def self.extract (block)
+      sequence = RubyVM::InstructionSequence.of(block) rescue nil
+      location = sequence && sequence.to_a[4][:code_location]
+      path     = sequence && (sequence.absolute_path || sequence.path)
+      unless location && path && File.readable?(path)
+        raise ArgumentError,
+              "CArray.fuse cannot read this block's source (defined in irb, " \
+              "eval, or a file that is no longer there).  Write `.lazy` on " \
+              "the operands instead: `a.lazy + b.lazy`."
+      end
+      lines = File.readlines(path)
+      first_line, first_column, last_line, last_column = location
+      # The columns count bytes, not characters, so a line with anything
+      # multi-byte on it slices in the wrong place unless this does too.
+      if first_line == last_line
+        lines[first_line - 1].byteslice(first_column...last_column)
+      else
+        [lines[first_line - 1].byteslice(first_column..),
+         *lines[first_line...(last_line - 1)],
+         lines[last_line - 1].byteslice(0...last_column)].join
+      end
+    end
+
+    # -- the rewrite -------------------------------------------------------
+
+    # The leaves are the names being read.  Everything else keeps its shape:
+    # calls are inserted around leaves and the expression they sit in is
+    # left alone.
+    class Leaves < Prism::Visitor
+      attr_reader :spots
+
+      def initialize
+        @spots = []
+      end
+
+      def visit_local_variable_read_node (node)    = mark(node)
+      def visit_instance_variable_read_node (node) = mark(node)
+      def visit_constant_read_node (node)          = mark(node)
+
+      # `Math::PI` is one name, not `Math` with something after it.
+      def visit_constant_path_node (node)
+        mark(node)
+      end
+
+      def visit_call_node (node)
+        if node.name == :[] || node.name == :[]=
+          # An index is a position, not a value to fuse: `a[i]` shadows `a`
+          # and leaves `i` alone.
+          visit(node.receiver)
+          return
+        end
+        mark(node) if node.receiver.nil? && node.arguments.nil? && node.block.nil?
+        super
+      end
+
+      private
+
+      def mark (node)
+        @spots << [node.location.start_offset, node.location.end_offset]
+      end
+    end
+
+    def self.rewrite (source)
+      visitor = Leaves.new
+      Prism.parse(source).value.accept(visitor)
+      out = source.dup
+      visitor.spots.sort_by { |start, _| -start }.each do |start, stop|
+        out[start...stop] = "::CArray::FuseSource.shadow(#{source[start...stop]})"
+      end
+      out
+    end
+  end
+end