carray 2.0.0 → 3.0.0

data/lib/carray/frame/convert.rb

@@ -0,0 +1,95 @@
+# CAFrame row iteration and matrix conversion (memo §11.9, §11.11).
+
+class CAFrame
+  # Iterate rows as Ruby Hashes (memo §11.11). This is an escape path — the
+  # primary idiom is column-vectorized work (§4.3); each_row is for touching
+  # heterogeneous / Face-carrying rows now and then, not hot loops. Without a
+  # block, returns an Enumerator.
+  def each_row
+    return enum_for(:each_row) unless block_given?
+    nrow.times { |i| yield row(i) }
+    self
+  end
+
+  # Export rows as an Array of plain Ruby Hashes (memo §11.11) — the inverse of
+  # +from_records+ and the shape +JSON.generate+ wants. Each row is one Hash of
+  # column name => value; a scalar cell is a Ruby value, an N-D cell is a Ruby
+  # Array, and a masked cell (UNDEF) is +nil+. Normalizing UNDEF -> nil and
+  # CArray -> Array (unlike +each_row+, which yields the raw view with UNDEF /
+  # CArray slices) is what makes it round-trip: +from_records(df.to_records)+
+  # rebuilds the same typing, and the result is JSON-serializable.
+  def to_records
+    each_row.map { |row| row.transform_values { |v| record_value(v) } }
+  end
+
+  # Hand the frame over as a CArray with the minimum work (memo §11.9): a
+  # 2-D **view** of shape (nrow, nvar), one column per variable in column
+  # order (§12-C). Nothing is materialised — the result is a +CAStack+ over
+  # the stored columns, so reads gather from them and writes flow back
+  # (§3.6). Call +copy+ on it for an independent, owned matrix; +CArray.tabulate+
+  # is the eager sibling that builds one directly.
+  #
+  # Only same-shape scalar (1-D) columns qualify. An N-D column has no single
+  # matrix form and raises — escape it per column with +df["name"]+. A mixed
+  # dtype set is promoted to a common type (+result_type+, §12-F) through
+  # lazy cast lanes, so the promotion costs no buffer either.
+  #
+  # +writable: true+ demands a result whose writes reach this frame's own
+  # columns (the 3.0 +to_ca+ contract). That holds only when every column
+  # enters the stack unchanged, so it is refused when a column is read-only
+  # or when the common type promotes it: the promoted column is stacked
+  # through a cast lane, and a write there is no longer the value the caller
+  # handed over. Cast the frame first (+df.cast+) when the promoted matrix is
+  # what should be written to.
+  def to_ca(writable: false)
+    cols = @columns.values
+    raise ArgumentError, "frame has no columns to stack into a matrix" if cols.empty?
+    nd = @columns.find { |_, c| c.ndim != 1 }
+    if nd
+      raise ArgumentError,
+            "to_ca needs all-scalar (1-D) columns; #{nd.first.inspect} is " \
+            "#{nd.last.ndim}-D — escape per column with df[name]"
+    end
+    refuse_unshared_columns(cols) if writable
+    begin
+      CArray.stack(cols, axis: 1)
+    rescue ArgumentError, RuntimeError => e
+      # The columns have no common type (a text or Face column beside a
+      # numeric one). Point at the frame-level verb that gives them one.
+      raise e.class,
+            "#{e.message} -- df.promote(:object) brings every column to its " \
+            "surface values, and that frame stacks"
+    end
+  end
+
+  # The +writable: true+ half of +to_ca+: raise unless the stack's parents are
+  # the stored columns themselves. +promote_list+ hands back each column
+  # untouched when no coercion is needed, so identity is the exact test for
+  # "this column is shared, not re-expressed".
+  private def refuse_unshared_columns(cols)
+    ro = @columns.find { |_, c| c.read_only? }
+    if ro
+      raise "CAFrame#to_ca cannot satisfy `writable: true': column " \
+            "#{ro.first.inspect} is read-only"
+    end
+    promoted = CArray.promote_list(cols)
+    recast = @columns.keys.zip(cols, promoted).find { |_, col, p| !p.equal?(col) }
+    if recast
+      raise "CAFrame#to_ca cannot satisfy `writable: true': column " \
+            "#{recast[0].inspect} (#{recast[1].data_type}) is promoted to " \
+            "#{recast[2].data_type} for the common type, so it is stacked " \
+            "through a cast lane; cast the frame first (df.cast) or take the " \
+            "column with df[name]"
+    end
+  end
+
+  private def record_value(v)
+    if UNDEF.equal?(v)
+      nil
+    elsif v.is_a?(CArray)
+      v.to_a.map { |e| UNDEF.equal?(e) ? nil : e }
+    else
+      v
+    end
+  end
+end

data/lib/carray/frame/csv_parser.rb

@@ -0,0 +1,211 @@
+# A small, self-contained CSV tokenizer for CAFrame.from_csv.
+#
+# It produces raw String cells and does no type inference — casting is a
+# separate step. The tokenizer is tuned for the
+# common shape of scientific / ETL CSV: records with no quote character take a
+# single String#split fast path, and only quote-bearing records fall back to
+# the field scanner. Quoted fields carry embedded separators, embedded newlines
+# (multi-line records) and doubled-quote ("") escapes.
+#
+# Correctness choices:
+#   * RFC 4180 spacing by default -- spaces are significant and preserved.
+#     Pass +strip: true+ to trim unquoted fields (handy for numeric columns).
+#   * An empty unquoted field is nil (missing); an empty quoted field ("") is
+#     the empty String. Both cast to UNDEF for numeric columns (parse-mask).
+#   * A UTF-8 BOM is stripped via the "bom|utf-8" read mode by default.
+#   * A parse error names the record number.
+#
+# Two entry points share the Tokenizer:
+#   * CSVParser.parse / parse_file -- whole-file parse into [headers, rows].
+#   * CSVReader                    -- the block reading-control DSL used by
+#                                     CAFrame.from_csv (header / skip / body /
+#                                     column_names).
+
+require "strscan"
+
+class CAFrame
+  # CSV tokenizer behind `CAFrame.from_csv`.  It produces raw String cells
+  # and does no type inference — casting is a separate step.
+  #
+  # Records with no quote character take a `String#split` fast path; only
+  # quote-bearing records fall back to the field scanner, which handles
+  # embedded separators, embedded newlines and doubled-quote escapes.
+  # Spacing follows RFC 4180 (significant and preserved) unless `strip:` is
+  # given.  An empty unquoted field is `nil` (missing); an empty quoted field
+  # is the empty String.
+  module CSVParser
+    module_function
+
+    # Raised when the input cannot be tokenized, e.g. a quoted field that
+    # never closes before end of input.
+    class MalformedCSV < StandardError; end
+
+    # Parse a file into [headers, rows]. +encoding+ is an IO open-mode encoding
+    # string; the default strips a leading BOM and reads UTF-8.
+    def parse_file(path, encoding: "bom|utf-8", **opts)
+      File.open(path, "r:#{encoding}") { |io| parse(io, **opts) }
+    end
+
+    # Parse an IO (or anything answering +gets+). The first record supplies the
+    # headers; the rest are data rows. Fully blank lines are skipped.
+    def parse(io, sep: ",", quote: '"', strip: false)
+      tok = Tokenizer.new(sep, quote, strip)
+      headers = nil
+      rows    = []
+      while (fields = tok.read(io))
+        if headers.nil?
+          headers = fields.map(&:to_s)
+        else
+          rows << fields
+        end
+      end
+      [headers, rows]
+    end
+
+    # Read one logical record, joining continuation lines while a quoted field
+    # is still open (an odd number of quote characters means unbalanced).
+    def read_record(io, quote)
+      line = io.gets
+      return nil if line.nil?
+      rec = line.dup
+      while rec.count(quote).odd?
+        more = io.gets
+        raise MalformedCSV, "unterminated quoted field at end of input" if more.nil?
+        rec << more
+      end
+      rec.chomp!
+      rec
+    end
+
+    # Splits records into fields. Regexps are compiled once and reused across
+    # every record, so per-row cost stays low.
+    class Tokenizer
+      def initialize(sep, quote, strip)
+        @sep      = sep
+        @quote    = quote
+        @strip    = strip
+        @sep_re   = /#{Regexp.escape(sep)}/
+        @quote_re = /#{Regexp.escape(quote)}/
+        @unquoted = /[^#{Regexp.escape(sep)}]*/
+        @inner    = /[^#{Regexp.escape(quote)}]*/
+        @recno    = 0
+      end
+
+      # Fields of the next non-blank record, or nil at EOF.
+      def read(io)
+        loop do
+          rec = CSVParser.read_record(io, @quote)
+          return nil if rec.nil?
+          @recno += 1
+          next if rec.empty?
+          return rec.count(@quote).zero? ? simple(rec) : scan(rec)
+        end
+      end
+
+      private def simple(rec)
+        rec.split(@sep, -1).map do |cell|
+          cell = cell.strip if @strip
+          cell.empty? ? nil : cell
+        end
+      end
+
+      private def scan(rec)
+        sc = StringScanner.new(rec)
+        fields = []
+        loop do
+          if sc.scan(@quote_re)
+            buf = +""
+            loop do
+              buf << sc.scan(@inner)
+              unless sc.scan(@quote_re)
+                raise MalformedCSV, "unterminated quoted field in record #{@recno}"
+              end
+              if sc.scan(@quote_re) # doubled quote -> literal quote, field continues
+                buf << @quote
+              else
+                break
+              end
+            end
+            fields << buf
+          else
+            cell = sc.scan(@unquoted) || ""
+            cell = cell.strip if @strip
+            fields << (cell.empty? ? nil : cell)
+          end
+          break unless sc.scan(@sep_re)
+        end
+        fields
+      end
+    end
+  end
+
+  # The block reading-control DSL for CAFrame.from_csv. A file
+  # often has preamble lines, a units row, or no header at all; the block says,
+  # in order, how to consume the stream:
+  #
+  #   CAFrame.from_csv(path) do
+  #     skip 3          # drop 3 preamble lines
+  #     header          # next record supplies the column names
+  #     skip 1          # drop a units row
+  #     body            # the rest are data rows
+  #   end
+  #
+  #   CAFrame.from_csv(path) do   # headerless file
+  #     column_names "date", "temp", "rh"
+  #     body
+  #   end
+  #
+  # The verbs are +skip+ / +header+ / +column_names+ / +body+; each returns a
+  # value useful inline (header returns its fields) and the ordering is the
+  # script. Without a block, from_csv runs the default +header+ then +body+.
+  class CSVReader
+    def initialize(io, sep: ",", quote: '"', strip: false)
+      @io    = io
+      @tok   = CSVParser::Tokenizer.new(sep, quote, strip)
+      @names = nil
+      @rows  = nil
+    end
+
+    # Drop +n+ raw lines (preamble, units, notes).
+    def skip(n = 1)
+      n.times { @io.gets }
+      self
+    end
+
+    # Read one record. With no argument it becomes the column names; with a
+    # name it is a secondary header (e.g. units) -- read, returned, not used as
+    # names. Returns the record's fields either way.
+    def header(name = nil)
+      fields = @tok.read(@io)
+      raise CSVParser::MalformedCSV, "header expected but input ended" if fields.nil?
+      if name.nil?
+        @names = fields.map(&:to_s)
+      else
+        (@named_headers ||= {})[name.to_s] = fields
+      end
+      fields
+    end
+
+    # Set the column names explicitly (headerless files).
+    def column_names(*names)
+      @names = names.flatten.map(&:to_s)
+      self
+    end
+
+    # Consume the remaining records as data rows.
+    def body
+      rows = []
+      while (fields = @tok.read(@io))
+        rows << fields
+      end
+      @rows = rows
+      self
+    end
+
+    # [names_or_nil, rows] for CAFrame.from_csv to build from. names is nil when
+    # neither header nor column_names ran (positional names are generated).
+    def result
+      [@names, @rows || []]
+    end
+  end
+end