carray 2.0.1 → 3.0.1

data/lib/carray/frame/concat.rb

@@ -0,0 +1,132 @@
+# CAFrame row concatenation (memo §5, spine = addressing + gather).
+#
+# Two entry points, mirroring the CArray-level taxonomy:
+#
+#   +CAFrame.meld+         view frame,  strict same data type per column,
+#                          each column is a CAMeld view over the inputs
+#   +CAFrame.concatenate+  eager frame, auto-casts per column,
+#                          each column is a materialised entity
+#
+# Both are class methods (symmetric N-ary; no frame is privileged), like
+# +CArray.meld+ / +CArray.concatenate+, not instance verbs.
+
+class CAFrame
+  # Weld frames along the row axis, view-style.  Each output column is
+  # +CArray.meld+ of that column across the input frames, so the result
+  # is a view frame that shares storage with the inputs (chain composability
+  # preserved: writes to the result flow back to whichever input frame owns
+  # the target segment, and vice versa).
+  #
+  # Per-column +data_type+ must match across frames (CArray.meld is a view
+  # constructor that refuses to auto-cast — silent promotion here would
+  # hide schema drift).  For an eager, auto-casting alternative use
+  # {CAFrame.concatenate}.
+  #
+  # Column matching is by name (output order follows the first frame);
+  # every frame must carry the same column-name set.  N-D columns carry
+  # their trailing dimensions along; masks are preserved.
+  #
+  # The index is welded too when every frame has one (their +axis_name+
+  # must agree); if none do, the result has no index; a mix raises.
+  # Column-set mismatch raises — a union-with-UNDEF mode is a possible
+  # future opt-in, kept out here to stay explicit (memo §4.2).
+  #
+  #   CAFrame.meld(jan, feb, mar)      # view over three months
+  #   CAFrame.meld([jan, feb, mar])    # an Array is accepted too
+  def self.meld(*frames)
+    frames = frames.flatten
+    check_concat_inputs(frames, verb: "meld")
+    first = frames.first
+    names = first.variable_names
+    check_column_sets(frames, names, verb: "meld")
+    cols = {}
+    names.each do |name|
+      cols[name] = CArray.meld(frames.map { |f| f[name] }, axis: 0)
+    end
+    new(cols, axis_name: first.axis_name, index: meld_index(frames))
+  end
+
+  # Concatenate frames along the row axis, eagerly.  Each output column is
+  # +CArray.concatenate+ of that column across the input frames, so per-column
+  # data types auto-promote to a common type.  The result is a fresh, independent
+  # frame — writes to it do not propagate back to the input frames.
+  #
+  # For a view frame that shares storage with the inputs (strict same data type
+  # per column, chain composability preserved) use {CAFrame.meld}.
+  #
+  # Column matching, index handling, and column-set / index-mix rules
+  # match {CAFrame.meld}.
+  #
+  #   CAFrame.concatenate(jan, feb, mar)     # eager, independent result
+  #   CAFrame.concatenate([jan, feb, mar])   # an Array is accepted too
+  def self.concatenate(*frames)
+    frames = frames.flatten
+    check_concat_inputs(frames, verb: "concatenate")
+    first = frames.first
+    names = first.variable_names
+    check_column_sets(frames, names, verb: "concatenate")
+    cols = {}
+    names.each do |name|
+      cols[name] = CArray.concatenate(frames.map { |f| f[name] })
+    end
+    new(cols, axis_name: first.axis_name, index: concatenate_index(frames))
+  end
+
+  # ---- shared validators -------------------------------------------------
+
+  def self.check_concat_inputs(frames, verb:)
+    raise ArgumentError, "#{verb} requires at least one frame" if frames.empty?
+    unless frames.all? { |f| f.is_a?(CAFrame) }
+      raise ArgumentError, "#{verb} expects CAFrame arguments"
+    end
+  end
+  private_class_method :check_concat_inputs
+
+  def self.check_column_sets(frames, names, verb:)
+    expected = names.sort
+    frames.each_with_index do |f, i|
+      next if i.zero?
+      if f.variable_names.sort != expected
+        raise ArgumentError,
+              "#{verb}: frame #{i} has columns #{f.variable_names.inspect}, " \
+              "expected the same set as #{names.inspect}"
+      end
+    end
+  end
+  private_class_method :check_column_sets
+
+  # ---- index helpers -----------------------------------------------------
+
+  # Index welded via CArray.meld (view).  Every frame must agree: all
+  # indexed (axis names must match) or none.
+  def self.meld_index(frames)
+    indexes, axis = index_pieces(frames, verb: "meld")
+    return nil unless indexes
+    axis   # unused; kept for symmetry
+    CArray.meld(indexes, axis: 0)
+  end
+  private_class_method :meld_index
+
+  # Index concatenated eagerly via CArray.concatenate.
+  def self.concatenate_index(frames)
+    indexes, _axis = index_pieces(frames, verb: "concatenate")
+    return nil unless indexes
+    CArray.concatenate(indexes)
+  end
+  private_class_method :concatenate_index
+
+  def self.index_pieces(frames, verb:)
+    indexes = frames.map(&:index)
+    return nil if indexes.none?
+    unless indexes.all?
+      raise ArgumentError, "#{verb}: some frames have an index and others do not"
+    end
+    axis_names = frames.map(&:axis_name).uniq
+    unless axis_names.size == 1
+      raise ArgumentError,
+            "#{verb}: indexed frames have different axis names #{axis_names.inspect}"
+    end
+    [indexes, axis_names.first]
+  end
+  private_class_method :index_pieces
+end

data/lib/carray/frame/convert.rb

@@ -0,0 +1,95 @@
+# CAFrame row iteration and matrix conversion (memo §11.9, §11.11).
+
+class CAFrame
+  # Iterate rows as Ruby Hashes (memo §11.11). This is an escape path — the
+  # primary idiom is column-vectorized work (§4.3); each_row is for touching
+  # heterogeneous / Face-carrying rows now and then, not hot loops. Without a
+  # block, returns an Enumerator.
+  def each_row
+    return enum_for(:each_row) unless block_given?
+    nrow.times { |i| yield row(i) }
+    self
+  end
+
+  # Export rows as an Array of plain Ruby Hashes (memo §11.11) — the inverse of
+  # +from_records+ and the shape +JSON.generate+ wants. Each row is one Hash of
+  # column name => value; a scalar cell is a Ruby value, an N-D cell is a Ruby
+  # Array, and a masked cell (UNDEF) is +nil+. Normalizing UNDEF -> nil and
+  # CArray -> Array (unlike +each_row+, which yields the raw view with UNDEF /
+  # CArray slices) is what makes it round-trip: +from_records(df.to_records)+
+  # rebuilds the same typing, and the result is JSON-serializable.
+  def to_records
+    each_row.map { |row| row.transform_values { |v| record_value(v) } }
+  end
+
+  # Hand the frame over as a CArray with the minimum work (memo §11.9): a
+  # 2-D **view** of shape (nrow, nvar), one column per variable in column
+  # order (§12-C). Nothing is materialised — the result is a +CAStack+ over
+  # the stored columns, so reads gather from them and writes flow back
+  # (§3.6). Call +copy+ on it for an independent, owned matrix; +CArray.tabulate+
+  # is the eager sibling that builds one directly.
+  #
+  # Only same-shape scalar (1-D) columns qualify. An N-D column has no single
+  # matrix form and raises — escape it per column with +df["name"]+. A mixed
+  # data type set is promoted to a common type (+result_type+, §12-F) through
+  # lazy cast lanes, so the promotion costs no buffer either.
+  #
+  # +writable: true+ demands a result whose writes reach this frame's own
+  # columns (the 3.0 +to_ca+ contract). That holds only when every column
+  # enters the stack unchanged, so it is refused when a column is read-only
+  # or when the common type promotes it: the promoted column is stacked
+  # through a cast lane, and a write there is no longer the value the caller
+  # handed over. Cast the frame first (+df.cast+) when the promoted matrix is
+  # what should be written to.
+  def to_ca(writable: false)
+    cols = @columns.values
+    raise ArgumentError, "frame has no columns to stack into a matrix" if cols.empty?
+    nd = @columns.find { |_, c| c.ndim != 1 }
+    if nd
+      raise ArgumentError,
+            "to_ca needs all-scalar (1-D) columns; #{nd.first.inspect} is " \
+            "#{nd.last.ndim}-D — escape per column with df[name]"
+    end
+    refuse_unshared_columns(cols) if writable
+    begin
+      CArray.stack(cols, axis: 1)
+    rescue ArgumentError, RuntimeError => e
+      # The columns have no common type (a text or Face column beside a
+      # numeric one). Point at the frame-level verb that gives them one.
+      raise e.class,
+            "#{e.message} -- df.promote(:object) brings every column to its " \
+            "surface values, and that frame stacks"
+    end
+  end
+
+  # The +writable: true+ half of +to_ca+: raise unless the stack's parents are
+  # the stored columns themselves. +promote_list+ hands back each column
+  # untouched when no coercion is needed, so identity is the exact test for
+  # "this column is shared, not re-expressed".
+  private def refuse_unshared_columns(cols)
+    ro = @columns.find { |_, c| c.read_only? }
+    if ro
+      raise "CAFrame#to_ca cannot satisfy `writable: true': column " \
+            "#{ro.first.inspect} is read-only"
+    end
+    promoted = CArray.promote_list(cols)
+    recast = @columns.keys.zip(cols, promoted).find { |_, col, p| !p.equal?(col) }
+    if recast
+      raise "CAFrame#to_ca cannot satisfy `writable: true': column " \
+            "#{recast[0].inspect} (#{recast[1].data_type}) is promoted to " \
+            "#{recast[2].data_type} for the common type, so it is stacked " \
+            "through a cast lane; cast the frame first (df.cast) or take the " \
+            "column with df[name]"
+    end
+  end
+
+  private def record_value(v)
+    if UNDEF.equal?(v)
+      nil
+    elsif v.is_a?(CArray)
+      v.to_a.map { |e| UNDEF.equal?(e) ? nil : e }
+    else
+      v
+    end
+  end
+end

data/lib/carray/frame/csv_parser.rb

@@ -0,0 +1,211 @@
+# A small, self-contained CSV tokenizer for CAFrame.from_csv.
+#
+# It produces raw String cells and does no type inference — casting is a
+# separate step. The tokenizer is tuned for the
+# common shape of scientific / ETL CSV: records with no quote character take a
+# single String#split fast path, and only quote-bearing records fall back to
+# the field scanner. Quoted fields carry embedded separators, embedded newlines
+# (multi-line records) and doubled-quote ("") escapes.
+#
+# Correctness choices:
+#   * RFC 4180 spacing by default -- spaces are significant and preserved.
+#     Pass +strip: true+ to trim unquoted fields (handy for numeric columns).
+#   * An empty unquoted field is nil (missing); an empty quoted field ("") is
+#     the empty String. Both cast to UNDEF for numeric columns (parse-mask).
+#   * A UTF-8 BOM is stripped via the "bom|utf-8" read mode by default.
+#   * A parse error names the record number.
+#
+# Two entry points share the Tokenizer:
+#   * CSVParser.parse / parse_file -- whole-file parse into [headers, rows].
+#   * CSVReader                    -- the block reading-control DSL used by
+#                                     CAFrame.from_csv (header / skip / body /
+#                                     column_names).
+
+require "strscan"
+
+class CAFrame
+  # CSV tokenizer behind `CAFrame.from_csv`.  It produces raw String cells
+  # and does no type inference — casting is a separate step.
+  #
+  # Records with no quote character take a `String#split` fast path; only
+  # quote-bearing records fall back to the field scanner, which handles
+  # embedded separators, embedded newlines and doubled-quote escapes.
+  # Spacing follows RFC 4180 (significant and preserved) unless `strip:` is
+  # given.  An empty unquoted field is `nil` (missing); an empty quoted field
+  # is the empty String.
+  module CSVParser
+    module_function
+
+    # Raised when the input cannot be tokenized, e.g. a quoted field that
+    # never closes before end of input.
+    class MalformedCSV < StandardError; end
+
+    # Parse a file into [headers, rows]. +encoding+ is an IO open-mode encoding
+    # string; the default strips a leading BOM and reads UTF-8.
+    def parse_file(path, encoding: "bom|utf-8", **opts)
+      File.open(path, "r:#{encoding}") { |io| parse(io, **opts) }
+    end
+
+    # Parse an IO (or anything answering +gets+). The first record supplies the
+    # headers; the rest are data rows. Fully blank lines are skipped.
+    def parse(io, sep: ",", quote: '"', strip: false)
+      tok = Tokenizer.new(sep, quote, strip)
+      headers = nil
+      rows    = []
+      while (fields = tok.read(io))
+        if headers.nil?
+          headers = fields.map(&:to_s)
+        else
+          rows << fields
+        end
+      end
+      [headers, rows]
+    end
+
+    # Read one logical record, joining continuation lines while a quoted field
+    # is still open (an odd number of quote characters means unbalanced).
+    def read_record(io, quote)
+      line = io.gets
+      return nil if line.nil?
+      rec = line.dup
+      while rec.count(quote).odd?
+        more = io.gets
+        raise MalformedCSV, "unterminated quoted field at end of input" if more.nil?
+        rec << more
+      end
+      rec.chomp!
+      rec
+    end
+
+    # Splits records into fields. Regexps are compiled once and reused across
+    # every record, so per-row cost stays low.
+    class Tokenizer
+      def initialize(sep, quote, strip)
+        @sep      = sep
+        @quote    = quote
+        @strip    = strip
+        @sep_re   = /#{Regexp.escape(sep)}/
+        @quote_re = /#{Regexp.escape(quote)}/
+        @unquoted = /[^#{Regexp.escape(sep)}]*/
+        @inner    = /[^#{Regexp.escape(quote)}]*/
+        @recno    = 0
+      end
+
+      # Fields of the next non-blank record, or nil at EOF.
+      def read(io)
+        loop do
+          rec = CSVParser.read_record(io, @quote)
+          return nil if rec.nil?
+          @recno += 1
+          next if rec.empty?
+          return rec.count(@quote).zero? ? simple(rec) : scan(rec)
+        end
+      end
+
+      private def simple(rec)
+        rec.split(@sep, -1).map do |cell|
+          cell = cell.strip if @strip
+          cell.empty? ? nil : cell
+        end
+      end
+
+      private def scan(rec)
+        sc = StringScanner.new(rec)
+        fields = []
+        loop do
+          if sc.scan(@quote_re)
+            buf = +""
+            loop do
+              buf << sc.scan(@inner)
+              unless sc.scan(@quote_re)
+                raise MalformedCSV, "unterminated quoted field in record #{@recno}"
+              end
+              if sc.scan(@quote_re) # doubled quote -> literal quote, field continues
+                buf << @quote
+              else
+                break
+              end
+            end
+            fields << buf
+          else
+            cell = sc.scan(@unquoted) || ""
+            cell = cell.strip if @strip
+            fields << (cell.empty? ? nil : cell)
+          end
+          break unless sc.scan(@sep_re)
+        end
+        fields
+      end
+    end
+  end
+
+  # The block reading-control DSL for CAFrame.from_csv. A file
+  # often has preamble lines, a units row, or no header at all; the block says,
+  # in order, how to consume the stream:
+  #
+  #   CAFrame.from_csv(path) do
+  #     skip 3          # drop 3 preamble lines
+  #     header          # next record supplies the column names
+  #     skip 1          # drop a units row
+  #     body            # the rest are data rows
+  #   end
+  #
+  #   CAFrame.from_csv(path) do   # headerless file
+  #     column_names "date", "temp", "rh"
+  #     body
+  #   end
+  #
+  # The verbs are +skip+ / +header+ / +column_names+ / +body+; each returns a
+  # value useful inline (header returns its fields) and the ordering is the
+  # script. Without a block, from_csv runs the default +header+ then +body+.
+  class CSVReader
+    def initialize(io, sep: ",", quote: '"', strip: false)
+      @io    = io
+      @tok   = CSVParser::Tokenizer.new(sep, quote, strip)
+      @names = nil
+      @rows  = nil
+    end
+
+    # Drop +n+ raw lines (preamble, units, notes).
+    def skip(n = 1)
+      n.times { @io.gets }
+      self
+    end
+
+    # Read one record. With no argument it becomes the column names; with a
+    # name it is a secondary header (e.g. units) -- read, returned, not used as
+    # names. Returns the record's fields either way.
+    def header(name = nil)
+      fields = @tok.read(@io)
+      raise CSVParser::MalformedCSV, "header expected but input ended" if fields.nil?
+      if name.nil?
+        @names = fields.map(&:to_s)
+      else
+        (@named_headers ||= {})[name.to_s] = fields
+      end
+      fields
+    end
+
+    # Set the column names explicitly (headerless files).
+    def column_names(*names)
+      @names = names.flatten.map(&:to_s)
+      self
+    end
+
+    # Consume the remaining records as data rows.
+    def body
+      rows = []
+      while (fields = @tok.read(@io))
+        rows << fields
+      end
+      @rows = rows
+      self
+    end
+
+    # [names_or_nil, rows] for CAFrame.from_csv to build from. names is nil when
+    # neither header nor column_names ran (positional names are generated).
+    def result
+      [@names, @rows || []]
+    end
+  end
+end