carray 3.0.2 → 3.0.4

data/lib/carray/frame/io.rb

@@ -1,15 +1,21 @@
 # CAFrame CSV input/output (memo §11.2, §6-2).
 
 require "carray/frame/csv_parser"
+require "carray/inspect"   # CArray::Inspector.face_label, for the column list
 
 class CAFrame
   # Read a CSV into a frame. The header row supplies column names (Strings,
-  # §3.7); every column is built raw as an object CArray of the cell strings
-  # (§4.2 -- read and arrange only, no type-inference engine). Casting is a
-  # separate step: pass +types:+ ({ "temp" => :float64 }, or the array-key /
-  # reverse forms of +cast+) to cast named columns on load, or call +cast+
-  # later. Broken cells fail to_type and become UNDEF automatically
-  # (parse-mask, §6-2).
+  # §3.7); every column is built raw as a CAString of the cell strings, all
+  # over one object array (§4.2). Casting is a separate step: +types:+ takes
+  # anything +cast+ takes as a map -- { "temp" => :float64 }, :infer, or a
+  # map with +default:+ -- and casts the frame with it on load, which is the
+  # same as calling +cast+ afterwards. Broken cells become UNDEF
+  # automatically (parse-mask, §6-2).
+  #
+  #   CAFrame.from_csv("obs.csv", types: :infer)
+  #   CAFrame.from_csv("obs.csv", types: { default: :infer, "code" => :int32, "id" => nil })
+  #   CAFrame.from_csv("obs.csv", types: { default: :float64, "station" => nil })
+  #   CAFrame.from_csv("obs.csv", types: { "station" => :categorical })
   #
   # +source+ is a path, or an open IO -- anything answering +gets+, which a
   # StringIO is. So CSV that is already in memory does not have to go to a
@@ -30,67 +36,253 @@ class CAFrame
   #   strip:    trim spaces from unquoted fields (default false, RFC spacing)
   #   +encoding+: open-mode encoding for a path (default "bom|utf-8", strips a
   #             BOM). It has nothing to open when +source+ is an IO, so there
-  #             the IO's own encoding governs and a BOM is the caller's.
+  #             the IO's own encoding governs and a BOM is the caller's. To
+  #             read a file in another encoding, name it and the one to
+  #             transcode to: "CP932:UTF-8" for a CSV written by Excel in
+  #             Japanese.
   #   parser:   a callable source -> [headers, rows] to inject another parser
   #             (e.g. the stdlib +csv+, or a typed-table source); when given,
-  #             sep/quote/strip/encoding and any block are that parser's concern.
+  #             sep/quote/strip/encoding and any block are that parser's concern,
+  #             and header:/data:/column_names:/columns:/attrs: are refused.
+  #   on_error: what a +types:+ cast does with a cell that does not read
+  #             (:mask default, :warn, :raise); see +cast+.
+  #
+  # Where the header and the data are is given by the index of the line,
+  # from 0, as in File.readlines (an error names the line from 1, as an
+  # editor numbers it):
+  #   header:   the line of the column names (default 0), or false for none
+  #   data:     the first line of the data (default the line after the
+  #             header), or a Range of lines; a record that starts in it is
+  #             read whole
+  #   column_names: the names, for a file with none or to replace them; the
+  #             header then defaults to none
+  #   columns:  the columns to read, by name or by index from 0 (Integers
+  #             and Ranges), in the order the frame has them; the others are
+  #             read past without a String made for their fields. Without a
+  #             header the columns are named c0, c1, ..., and those names
+  #             select too. A Hash selects by its keys and names the columns
+  #             with its values: { 2 => "code", "c7" => "area" }; types: and
+  #             missing: then use the new names
+  #   attrs:    attribute keys for the lines right below the header, one
+  #             key per line: field j of a line becomes that attribute of
+  #             column j (an empty field gives none). The data then starts
+  #             below them. What to_csv(attrs:) writes, it reads back
   #
-  # A block gives reading control for files with preamble lines, a units row,
-  # or no header (memo §11.2), using +skip+ / +header+ / +column_names+ /
-  # +body+ (see CSVReader). Without a block the default is +header+ then +body+.
+  #   CAFrame.from_csv("obs.csv", header: 2)            # a title above
+  #   CAFrame.from_csv("obs.csv", header: 0, data: 3)   # units on lines 1-2
+  #   CAFrame.from_csv("big.csv", data: 1..100)         # the first 100 lines
+  #   CAFrame.from_csv("raw.csv", column_names: %w[date temp rh])
+  #   CAFrame.from_csv("wide.csv", columns: %w[date temp])
+  #   CAFrame.from_csv("raw.csv", header: false, columns: { 0 => "date", 3 => "temp" })
+  #   CAFrame.from_csv("obs.csv", attrs: [:units])      # units on line 1
   #
-  #   CAFrame.from_csv("obs.csv") do
-  #     skip 2; header; skip 1; body
-  #   end
+  # A block, given the reader, reads in any other order with +skip+ /
+  # +header+ / +header(name)+ / +column_names+ / +columns+ / +data+ (see
+  # CSVReader):
+  #
+  #   CAFrame.from_csv("obs.csv") { |r| r.skip 2; r.header; r.skip 1; r.data }
   #
   # A missing field -- an unquoted empty one, or a cell a short row never
   # reached -- is UNDEF in the frame, whether or not the column is cast by
   # +types:+. A quoted empty field ("") is the empty string, which is a value.
   # So the mask +to_csv+ writes comes back as a mask.
   #
-  # Columns are handed to the frame as CABlock views over one backing object
-  # array (§3.6 view-by-default); casting a column materializes it, and +copy+
-  # gives an independent frame.
-  def self.from_csv(source, types: nil,
-                    sep: ",", quote: '"', strip: false,
+  # Files that spell "missing" some other way say so with +missing:+. A field
+  # whose text is one of the given tokens is UNDEF too, before +types:+
+  # casts, so a -999 sentinel never reaches the column as a number:
+  #
+  #   CAFrame.from_csv("obs.csv", missing: ["-999", "///"])        # every column
+  #   CAFrame.from_csv("obs.csv", missing: { "temp" => "-999" })   # one column
+  #   CAFrame.from_csv("obs.csv", missing: { default: "-999", "id" => [] })
+  #
+  # Tokens are Strings and are compared with the field's text as read (after
+  # +strip:+, quoted or not), so "-999" does not match "-999.0". In a Hash,
+  # +:default+ gives the tokens for every column and a column name gives that
+  # column's own, which replace the default ([] or "" for none); without
+  # +:default+ the columns it does not name keep only the empty field. As in
+  # +to_csv+, "" means the empty field, so a quoted "" is still a value.
+  #
+  # Without +types:+, the columns are handed to the frame as CABlock views
+  # over one backing object array (§3.6 view-by-default). With +types:+, the
+  # body is read without making a String per cell: a column cast to a number
+  # is read from the text directly, and a column left as text becomes its own
+  # CAString. Casting a column materializes it, and +copy+ gives an
+  # independent frame.
+  def self.from_csv(source, types: nil, on_error: :mask, missing: nil,
+                    header: nil, data: nil, column_names: nil, columns: nil,
+                    attrs: nil, sep: ",", quote: '"', strip: false,
                     encoding: "bom|utf-8", parser: nil, &block)
-    names, rows =
+    layout = { header: header, data: data, column_names: column_names,
+               columns: columns, attrs: attrs }.compact
+    if block && !layout.empty?
+      raise ArgumentError, "from_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
+                           "or a reading block, not both"
+    end
+    # The parser reads the header, the data and the columns itself.
+    if parser && !layout.empty?
+      raise ArgumentError, "from_csv: give #{layout.keys.map { |k| "#{k}:" }.join(', ')} " \
+                           "or parser:, not both"
+    end
+    if block && block.arity != 1
+      raise ArgumentError, "from_csv: the reading block takes the reader as its " \
+                           "parameter, as in from_csv(path) { |r| r.skip 2; r.header; r.data }"
+    end
+    missing = missing_tokens(missing) if missing
+    # With types:, the body is read into CAConstString columns, so a column
+    # cast to a number never becomes Strings; the text columns left over
+    # become CAString below. Without types: every column stays text, so the
+    # body goes straight to Strings. missing: masks text in place, which
+    # wants the Strings first, so it takes that path too.
+    read_text_as_const_string = !types.nil? && missing.nil?
+    names, rows, column_attrs =
       if parser
         parser.call(source)
       elsif source.respond_to?(:gets)
-        read_csv(source, sep: sep, quote: quote, strip: strip, &block)
+        with_encoding_hint("open the IO in the file's encoding, as in " \
+                           "File.open(path, \"r:<file encoding>:UTF-8\")") do
+          read_csv(source, sep: sep, quote: quote, strip: strip, layout: layout,
+                   read_text_as_const_string: read_text_as_const_string, &block)
+        end
       else
-        File.open(source, "r:#{encoding}") do |io|
-          read_csv(io, sep: sep, quote: quote, strip: strip, &block)
+        with_encoding_hint("it was opened as #{encoding.inspect}; pass the " \
+                           "file's encoding, as in " \
+                           "encoding: \"<file encoding>:UTF-8\"") do
+          File.open(source, "r:#{encoding}") do |io|
+            read_csv(io, sep: sep, quote: quote, strip: strip, layout: layout,
+                     read_text_as_const_string: read_text_as_const_string, &block)
+          end
         end
       end
 
     frame = build_frame(names, rows)
-    frame.cast(types) if types
+    mask_missing_tokens(frame, missing) if missing
+    frame.cast(types, on_error: on_error, option_name: :types) unless types.nil?
+    convert_const_string_columns_to_castring(frame)
+    column_attrs&.each_with_index do |attrs, j|
+      frame[frame.column_names[j]].set_attrs(attrs) unless attrs.empty?
+    end
     frame
   end
 
+  # A text column from_csv reads is a CAString, whichever way the body was
+  # read: the string operations (strip!, gsub!, ...) write through to it.
+  def self.convert_const_string_columns_to_castring(frame)
+    frame.column_names.each do |name|
+      col = frame[name]
+      next unless col.is_a?(CAConstString)
+      frame[name] = col.to_string
+    end
+  end
+  private_class_method :convert_const_string_columns_to_castring
+
+  # Split a +missing:+ argument, as from_csv and to_csv both take it, into the
+  # setting for every column and a Hash of per-column settings that replace
+  # it. A Hash says :default for every column and a column name for one;
+  # column names are Strings, so the Symbol cannot collide with one. A
+  # setting that is not a Hash is the default.
+  def self.split_missing(missing, verb)
+    return [missing, {}] unless missing.is_a?(Hash)
+    default = nil
+    per_column = {}
+    missing.each do |key, value|
+      case key
+      when :default then default = value
+      else
+        per_column[CAFrame.__send__(:column_key, key, verb, "missing: a column name",
+                                    hint: ":default is the only Symbol key")] = value
+      end
+    end
+    [default, per_column]
+  end
+
+  # The read side of missing:, checked before the file is read: each setting
+  # becomes an Array of token Strings.
+  def self.missing_tokens(missing)
+    default, per_column = split_missing(missing, "from_csv")
+    tokens = lambda do |value|
+      list = Array(value)
+      bad = list.reject { |t| t.is_a?(String) }
+      unless bad.empty?
+        raise ArgumentError,
+              "missing: tokens are matched against a field's text, so give them " \
+              "as Strings (#{bad.map { |t| t.to_s.inspect }.join(', ')}, not " \
+              "#{bad.map(&:inspect).join(', ')})"
+      end
+      # "" is the empty field, which is missing already; as on the write side
+      # it adds nothing, so a quoted "" stays the empty string.
+      list.reject(&:empty?)
+    end
+    [tokens.(default), per_column.transform_values(&tokens)]
+  end
+
+  def self.mask_missing_tokens(frame, missing)
+    default, per_column = missing
+    per_column.each_key { |name| frame[name] }    # KeyError for a name the file lacks
+    frame.column_names.each do |name|
+      col = frame[name]
+      per_column.fetch(name, default).each { |t| col[:eq, t] = UNDEF }
+    end
+  end
+
+  # The attribute keys an +attrs:+ argument names, as Strings: an Array of
+  # Symbols or Strings, none twice.
+  def self.csv_attr_keys(attrs, verb)
+    unless attrs.is_a?(Array)
+      raise TypeError, "#{verb}: attrs: takes an Array of attribute keys " \
+                       "(got #{attrs.inspect})"
+    end
+    keys = attrs.map do |key|
+      unless key.is_a?(Symbol) || key.is_a?(String)
+        raise TypeError, "#{verb}: attrs: an attribute key is a Symbol or String " \
+                         "(got #{key.inspect})"
+      end
+      key.to_s
+    end
+    # names-check: attribute keys, not column names
+    dup = keys.tally.find { |_, n| n > 1 }
+    raise ArgumentError, "#{verb}: attrs: names #{dup.first.inspect} twice" if dup
+    keys
+  end
+
+  private_class_method :split_missing, :missing_tokens, :mask_missing_tokens,
+                       :csv_attr_keys
+
   # Drive the reading-control DSL over one open IO and hand back
-  # [names, rows]. Shared by the path and the IO source, so the two cannot
-  # come to read a file differently.
-  def self.read_csv (io, sep:, quote:, strip:, &block)
-    reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip)
-    if block
-      block.arity == 1 ? block.call(reader) : reader.instance_exec(&block)
-    else
-      reader.header
-      reader.body
+  # [names, rows, column_attrs]. Shared by the path and the IO source, so the
+  # two cannot come to read a file differently.
+  def self.read_csv (io, sep:, quote:, strip:, layout:, read_text_as_const_string: false, &block)
+    reader = CSVReader.new(io, sep: sep, quote: quote, strip: strip,
+                           read_text_as_const_string: read_text_as_const_string)
+    reader.reporting do
+      block ? block.call(reader) : reader.layout(**layout)
     end
-    reader.result
+    names, rows = reader.result
+    [names, rows, reader.column_attrs(reader.file_ncol)]
   end
 
   private_class_method :read_csv
 
-  # Build a frame from parsed [names, rows]. When names is nil (headerless and
-  # no column_names) positional names "c0".."cN" are generated from the widest
-  # row. Rows are squared off to the column count (short rows padded with nil,
-  # over-long rows raise), one 2-D object array is bulk-filled, and each column
-  # is a view into it (§3.6).
+  # Re-raise a failure to decode the input with a line saying how to name the
+  # file's encoding. The error and its class are kept; only the message grows.
+  # Which encoding the file is in cannot be told from its bytes, so the hint
+  # names the option, not a value.
+  def self.with_encoding_hint(hint)
+    yield
+  rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError,
+         Encoding::CompatibilityError, ArgumentError => e
+    raise if e.is_a?(ArgumentError) && !e.message.include?("invalid byte sequence")
+    if e.is_a?(Encoding::CompatibilityError)
+      # The text decoded; the separator or quote is in another encoding.
+      raise e.class, "#{e.message} -- sep: and quote: are matched against the " \
+                     "text as read, so read the file transcoded to UTF-8, as in " \
+                     "encoding: \"<file encoding>:UTF-8\"", e.backtrace
+    end
+    raise e.class, "#{e.message} -- the CSV is not in the encoding it was " \
+                   "read as: #{hint}", e.backtrace
+  end
+
+  private_class_method :with_encoding_hint
+
   # Write the frame as CSV. CSV is a flat table of scalar cells, so this is the
   # text form of the same all-scalar subset +to_ca+ requires (§11.9): every
   # column must be 1-D. Unlike +to_ca+ it does not promote to a common data type --
@@ -110,36 +302,125 @@ class CAFrame
   #   csv = df.to_csv               # get a String
   #
   # Options: +sep+ / +quote+ mirror +from_csv+; +header+ writes the name row
-  # (default true); +index+ writes the index column (default true).
-  def to_csv(path = nil, sep: ",", quote: '"', header: true, index: true)
+  # (default true); +index+ writes the index column (default true);
+  # +encoding+ transcodes the text before it is written or returned (default
+  # nil leaves it as built, UTF-8). A character the encoding cannot hold
+  # raises Encoding::UndefinedConversionError rather than being dropped. A
+  # String cell in another encoding is transcoded to UTF-8; one with bytes
+  # that are not valid in it, or with no encoding (ASCII-8BIT), raises. Each
+  # names the row and the column of the cell.
+  #
+  #   df.to_csv("out.csv", encoding: "CP932")   # for Excel in Japanese
+  #
+  # +missing:+ writes a masked cell as that String instead of an empty field,
+  # for a reader that expects a sentinel; +from_csv(missing:)+ with the same
+  # String reads it back as UNDEF. A Hash sets it per column the way
+  # +from_csv+ does: +:default+ for every column, and a column name (or the
+  # index's axis name) for one, replacing the default; "" is the empty field.
+  # A cell whose value would be written as the same text raises, since the
+  # file could not tell the two apart.
+  #
+  #   df.to_csv("out.csv", missing: "-999")
+  #   df.to_csv("out.csv", missing: { default: "-999", "comment" => "" })
+  #
+  # +attrs:+ writes, below the name row, one line per attribute key given,
+  # in that order: field j is +attr(key)+ of column j (the index's for the
+  # index column). A column without the attribute gets an empty field, an
+  # empty String a quoted "", and a number its +to_s+.
+  # +from_csv(attrs:)+ with the same keys reads them back, as Strings.
+  #
+  #   df.to_csv("out.csv", attrs: [:units, :long_name])
+  def to_csv(path = nil, sep: ",", quote: '"', header: true, index: true,
+             encoding: nil, missing: nil, attrs: [])
+    default, per_column = CAFrame.send(:split_missing, missing, "to_csv")
+    [default, *per_column.values].each do |text|
+      next if text.nil? || text.is_a?(String)
+      raise ArgumentError, "to_csv: missing: is the String written for a masked " \
+                           "cell (got #{text.inspect})"
+    end
+    attr_keys = attrs.nil? ? [] : CAFrame.send(:csv_attr_keys, attrs, "to_csv")
+    if !header && !attr_keys.empty?
+      raise ArgumentError, "to_csv: attrs: writes lines below the name row, " \
+                           "so it needs header: true"
+    end
     nd = @columns.find { |_, c| c.ndim != 1 }
     if nd
       raise ArgumentError,
             "to_csv needs all-scalar (1-D) columns; #{nd.first.inspect} is " \
             "#{nd.last.ndim}-D — export it per column or via to_records + JSON"
     end
+    # A record cell has no text form that reads back; its members do.
+    rec = @columns.find { |_, c| c.is_a?(CARecord) }
+    if rec
+      name, col = rec
+      member = col.members.first
+      raise ArgumentError,
+            "to_csv cannot write the record column #{name.inspect}; write its " \
+            "members as columns, e.g. df[#{"#{name}_#{member}".inspect}] = " \
+            "df[#{name.inspect}][#{member.inspect}]"
+    end
 
     names     = []
     formatted = []
+    written   = []
     if index && @index
       names     << @axis_name
       formatted << format_csv_column(@index)
+      written   << @index
     end
     @columns.each do |name, col|
       names     << name
       formatted << format_csv_column(col)
+      written   << col
+    end
+    attr_rows = attr_keys.map do |key|
+      written.each_with_index.map { |col, j| csv_attr_text(col, key, names[j]) }
+    end
+    unknown = per_column.keys - names
+    raise KeyError, "to_csv: missing: names no column #{unknown.first.inspect}" unless unknown.empty?
+    formatted.each_with_index do |fcol, j|
+      token = per_column.fetch(names[j], default)
+      next if token.nil? || token.empty?        # the empty field, as without missing:
+      i = fcol.index(token)
+      if i
+        raise ArgumentError,
+              "to_csv: row #{i} of #{names[j].inspect} is written as " \
+              "#{token.inspect}, the text given for a masked cell"
+      end
+      fcol.map! { |text| text.nil? ? token : text }
     end
 
-    out = +""
-    if header
-      out << names.map { |t| quote_csv_field(t, sep, quote) }.join(sep) << "\n"
+    out = begin
+      csv_text(names, attr_rows, formatted, header, sep, quote)
+    rescue Encoding::CompatibilityError
+      nil
     end
-    @nrow.times do |i|
-      out << formatted.map { |fcol| quote_csv_field(fcol[i], sep, quote) }.join(sep) << "\n"
+    # A cell in another encoding, or with bytes that are not text, is found
+    # and named here, after the text was built in one pass, so a frame of
+    # UTF-8 text pays nothing for the check.
+    unless out && out.encoding == Encoding::UTF_8 && out.valid_encoding?
+      names = names.each_with_index.map { |t, j| csv_text_in_utf8(t, "the name of column #{j}") }
+      attr_rows = attr_rows.each_with_index.map do |row, k|
+        row.each_with_index.map do |t, j|
+          csv_text_in_utf8(t, "attribute #{attr_keys[k].inspect} of #{names[j].inspect}")
+        end
+      end
+      formatted = formatted.each_with_index.map do |fcol, j|
+        fcol.each_with_index.map { |t, i| csv_text_in_utf8(t, "row #{i} of #{names[j].inspect}") }
+      end
+      out = csv_text(names, attr_rows, formatted, header, sep, quote)
+    end
+
+    if encoding
+      out = begin
+        out.encode(encoding)
+      rescue Encoding::UndefinedConversionError => e
+        raise csv_unwritable_cell(names, attr_keys, attr_rows, formatted, encoding) || e
+      end
     end
 
     if path
-      File.write(path, out)
+      File.binwrite(path, out)
       self
     else
       out
@@ -177,19 +458,22 @@ class CAFrame
 
   # +to_s+ is the whole frame, +inspect+ the middle-elided one -- so +puts df+
   # dumps everything and +p df+ stays a screenful. +inspect+ leads with the
-  # same summary line it always had (nrow, variable data types, index), so the
+  # same summary line it always had (nrow, column data types, index), so the
   # table under it needs no row-count footer.
   def to_s
     to_table(rows: nil)
   end
 
-  # The summary line (nrow, variable data types, index) followed by the
+  # The summary line (nrow, column data types, index) followed by the
   # middle-elided table.
   # @return [String]
   def inspect
-    parts = @columns.map { |k, v| "#{k}:#{v.data_type}#{v.ndim > 1 ? v.shape[1..].inspect : ''}" }
+    parts = @columns.map do |k, v|
+      type = CArray::Inspector.type_label(v)
+      "#{k}:#{type}#{v.ndim > 1 ? v.shape[1..].inspect : ''}"
+    end
     idx = @index ? " index=#{@axis_name.inspect}" : ""
-    head = "#<CAFrame nrow=#{@nrow} vars=[#{parts.join(', ')}]#{idx}>"
+    head = "#<CAFrame nrow=#{@nrow} columns=[#{parts.join(', ')}]#{idx}>"
     # The table counts the index as a column, so a frame whose only data is its
     # index has one to show.  Gate on the same thing render_table does.
     return head if @columns.empty? && @index.nil?
@@ -232,7 +516,7 @@ class CAFrame
     out << table_row(widths.map { |w| "-" * w }, widths, aligns) << "\n"
     body.each { |cells| out << table_row(cells, widths, aligns) << "\n" }
     if footer && positions.size - positions.count(nil) < @nrow
-      out << "(#{plural(@nrow, 'row')}, #{plural(@columns.size, 'variable')})\n"
+      out << "(#{plural(@nrow, 'row')}, #{plural(@columns.size, 'column')})\n"
     end
     out
   end
@@ -263,9 +547,11 @@ class CAFrame
     0xF900..0xFAFF, 0xFE10..0xFE19, 0xFE30..0xFE6F, 0xFF00..0xFF60,
     0xFFE0..0xFFE6, 0x1F300..0x1F64F, 0x1F900..0x1F9FF, 0x20000..0x3FFFD,
   ].freeze
+  Ractor.make_shareable(WIDE_CHAR_RANGES)
   private_constant :WIDE_CHAR_RANGES
 
   COMBINING_RANGES = [0x0300..0x036F, 0x1AB0..0x1AFF, 0x20D0..0x20F0].freeze
+  Ractor.make_shareable(COMBINING_RANGES)
   private_constant :COMBINING_RANGES
 
   private def display_width(text)
@@ -285,17 +571,31 @@ class CAFrame
     "#{n} #{noun}#{n == 1 ? '' : 's'}"
   end
 
-  # Render one N-D cell the way Array#inspect would, except that a masked
-  # element prints as the table's missing marker rather than as UNDEF.
-  private def format_nested_cell(v)
+  # Render one N-D cell as a bracketed list.  Each element is shown as a
+  # scalar cell of its kind would be -- floats rounded to +precision+, a
+  # masked element as the table's missing marker -- except that a String
+  # keeps its quotes, so the list stays readable when one holds a comma.
+  private def format_nested_cell(v, precision)
     case v
-    when Array then "[" + v.map { |x| format_nested_cell(x) }.join(", ") + "]"
-    else UNDEF.equal?(v) || v.nil? ? "_" : v.inspect
+    when Array  then "[" + v.map { |x| format_nested_cell(x, precision) }.join(", ") + "]"
+    when String then v.inspect
+    else format_scalar_cell(v, precision)
     end
   end
 
   private def format_table_cell(col, i, precision)
     e = elem_at(col, i)
+    if e.is_a?(CArray)
+      # An N-D cell renders its elements, and a masked element among them
+      # takes the same marker a masked scalar does -- UNDEF's own inspect
+      # would put a second spelling of "missing" in the same table.
+      format_nested_cell(e.to_a, precision)
+    else
+      format_scalar_cell(e, precision)
+    end
+  end
+
+  private def format_scalar_cell(e, precision)
     if UNDEF.equal?(e) || e.nil?
       "_"
     elsif e.is_a?(Float) && precision
@@ -303,11 +603,6 @@ class CAFrame
       # sets the column width for every other row and makes the table hard
       # to read. precision: nil prints the value as Ruby renders it.
       e.round(precision).to_s
-    elsif e.is_a?(CArray)
-      # An N-D cell renders its elements, and a masked element among them
-      # takes the same marker a masked scalar does -- UNDEF's own inspect
-      # would put a second spelling of "missing" in the same table.
-      format_nested_cell(e.to_a)
     elsif e.is_a?(String)
       e
     elsif e.respond_to?(:iso8601)
@@ -318,6 +613,15 @@ class CAFrame
   end
 
   private def format_csv_column(col)
+    # A float32 value goes out as its own shortest decimal ("0.1"), which
+    # from_csv reads back as the same value, not as the double it widens
+    # to ("0.10000000149011612"); cmplx64 has that in each part.
+    if !col.face? && (col.data_type == CA_FLOAT32 || col.data_type == CA_CMPLX64)
+      col = col.__widen_keeping_shortest_decimal__
+    end
+    # A boolean goes out as 0 / 1, as it does in every other serialization;
+    # from_csv reads it back with types: { name => :boolean }.
+    col = col.to_type(:uint8) if !col.face? && col.data_type == CA_BOOLEAN
     col.to_a.map do |e|
       if UNDEF.equal?(e) || e.nil?
         nil
@@ -331,43 +635,166 @@ class CAFrame
     end
   end
 
+  private def csv_text(names, attr_rows, formatted, header, sep, quote)
+    out = +""
+    if header
+      out << names.map { |t| quote_csv_field(t, sep, quote) }.join(sep) << "\n"
+    end
+    attr_rows.each do |row|
+      out << row.map { |t| quote_csv_field(t, sep, quote) }.join(sep) << "\n"
+    end
+    @nrow.times do |i|
+      out << formatted.map { |fcol| quote_csv_field(fcol[i], sep, quote) }.join(sep) << "\n"
+    end
+    out
+  end
+
+  # The field +to_csv(attrs:)+ writes for attribute +key+ of +col+: nil for
+  # none (the empty field), the String itself, or a scalar's to_s.
+  private def csv_attr_text(col, key, name)
+    value = col.attr(key)
+    case value
+    when nil, String then value
+    when Array, Hash
+      raise ArgumentError, "to_csv: attribute #{key.inspect} of #{name.inspect} is " \
+                           "#{value.class}, which a CSV field cannot hold"
+    else value.to_s
+    end
+  end
+
+  # The text of a cell as UTF-8, the encoding the CSV is built in: text in
+  # another encoding is transcoded; bytes that are not valid in their
+  # encoding, or have none (ASCII-8BIT), raise naming the cell.
+  private def csv_text_in_utf8(text, where)
+    return text if text.nil?
+    enc = text.encoding
+    unless text.valid_encoding?
+      raise Encoding::InvalidByteSequenceError,
+            "to_csv: #{where} (#{csv_excerpt(text)}) is not valid #{enc}"
+    end
+    return text if enc == Encoding::UTF_8 || (enc.ascii_compatible? && text.ascii_only?)
+    if enc == Encoding::BINARY
+      raise Encoding::CompatibilityError,
+            "to_csv: #{where} (#{csv_excerpt(text)}) is bytes with no encoding " \
+            "(ASCII-8BIT); give it one with force_encoding"
+    end
+    text.encode(Encoding::UTF_8)
+  rescue Encoding::UndefinedConversionError
+    raise Encoding::UndefinedConversionError,
+          "to_csv: #{where} (#{csv_excerpt(text)}) has a character with no UTF-8 form in #{enc}"
+  end
+
+  # The start of a cell's text, quoted, for an error. Text is shown as it
+  # is, whatever the locale (inspect would escape it where the locale is not
+  # UTF-8); control characters and bytes that are not text are escaped.
+  private def csv_excerpt(text)
+    head = text[0, 20]
+    return head.dump unless head.valid_encoding? && head.encoding != Encoding::BINARY
+    head = head.encode(Encoding::UTF_8)
+    '"' + head.gsub(/[\x00-\x1f\x7f"\\]/) { |c| c.dump[1..-2] } + '"'
+  rescue EncodingError
+    text[0, 20].dump
+  end
+
+  # The error for the first cell (or name) +encoding+ cannot hold, or nil.
+  private def csv_unwritable_cell(names, attr_keys, attr_rows, formatted, encoding)
+    cells = names.each_with_index.map { |t, j| [t, "the name of column #{j}"] }
+    attr_rows.each_with_index do |row, k|
+      row.each_with_index do |t, j|
+        cells << [t, "attribute #{attr_keys[k].inspect} of #{names[j].inspect}"] if t
+      end
+    end
+    formatted.each_with_index do |fcol, j|
+      fcol.each_with_index { |t, i| cells << [t, "row #{i} of #{names[j].inspect}"] if t }
+    end
+    cells.each do |text, where|
+      text.encode(encoding)
+    rescue Encoding::UndefinedConversionError => e
+      return Encoding::UndefinedConversionError.new(
+        "to_csv: #{where} (#{csv_excerpt(text)}): #{csv_excerpt(e.error_char)} (U+#{format("%04X", e.error_char.ord)}) cannot be written in #{encoding}")
+    end
+    nil
+  end
+
   private def quote_csv_field(text, sep, quote)
     return "" if text.nil?
+    # With a separator longer than one character, a value can end in part of
+    # it ("a:" before "::"), and the reader would find the separator inside
+    # the value; such a value is quoted too.  A value of spaces and tabs is
+    # quoted so that it is neither a blank line the reader skips (a header of
+    # one column) nor a field strip: empties.
     if text.empty? || text.include?(sep) || text.include?(quote) ||
-       text.include?("\n") || text.include?("\r")
+       text.include?("\n") || text.include?("\r") || text.each_byte.all? { |b| b == 0x20 || b == 0x09 } ||
+       (sep.size > 1 && (text + sep).index(sep) != text.size)
       quote + text.gsub(quote, quote * 2) + quote
     else
       text
     end
   end
 
+  # Build a frame from parsed [names, rows]. When names is nil (headerless and
+  # no column_names) positional names "c0".."cN" are generated from the widest
+  # row. Rows are squared off to the column count (short rows padded with nil,
+  # over-long rows raise), one 2-D object array is bulk-filled, and each column
+  # is a view into it (§3.6).
   def self.build_frame(names, rows)
-    ncol  = names ? names.size : (rows.map(&:size).max || 0)
+    ncol  = if names then names.size
+            elsif rows.is_a?(CArray) then rows.shape[1]
+            else rows.map(&:size).max || 0
+            end
     names ||= Array.new(ncol) { |j| "c#{j}" }
+    # A frame holds one column per name, so a repeated one would keep only
+    # the last of its columns.
+    CAFrame.__send__(:refuse_repeated_names, names, "from_csv", "the header row names",
+                     hint: "pass header: false for the default column names " \
+                           "(c0, c1, ...), or name the columns with column_names:")
 
     cols = {}
-    if rows.empty?
-      names.each { |name| cols[name] = CArray.object(0) }
+    # Columns the C reader built as CAConstStrings over the text, for
+    # cast to read without a String per cell.
+    if rows.is_a?(CSVReader::ConstStringColumns)
+      names.zip(rows.columns) { |name, col| cols[name] = col }
       return new(cols)
     end
-    rows.each_with_index do |r, i|
-      if r.size < ncol
-        r.concat(Array.new(ncol - r.size))
-      elsif r.size > ncol
-        raise ArgumentError,
-              "row #{i + 1} has #{r.size} fields, expected #{ncol}"
-      end
+    # A table the C reader built: one row per record, missing cells masked.
+    table = rows if rows.is_a?(CArray) && rows.elements > 0
+    if table.nil? && (rows.is_a?(CArray) || rows.empty?)
+      names.each { |name| cols[name] = CArray.string(CArray.object(0)) }
+      return new(cols)
     end
-    table = CArray.object(rows.size, ncol) { rows }
+    table ||= table_of_rows(rows, ncol)
+    # Each column is a CAString over its view of the table: the text, with
+    # the string operations (strip!, gsub, extract, ...) at hand, writing
+    # through to the same cells.
+    names.each_with_index { |name, j| cols[name] = CArray.string(table[nil, j]) }
+    new(cols)
+  end
+  private_class_method :build_frame
+
+  def self.table_of_rows(rows, ncol)
     # A missing field is UNDEF, not a Ruby nil sitting in a cell.  The
     # tokenizer says missing with nil (an unquoted empty field; a quoted
     # one is the empty string and stays a value), and an object array will
     # hold that nil quite happily -- so a column read without `types:` used
     # to carry nil where the same column read with one carried UNDEF, and
-    # the mask a to_csv had written did not survive the trip back.
-    table[:eq, nil] = UNDEF
-    names.each_with_index { |name, j| cols[name] = table[nil, j] }
-    new(cols)
+    # the mask a to_csv had written did not survive the trip back.  A row
+    # holding a nil gets UNDEF in its place before the table is built (an
+    # UNDEF in the rows becomes a masked cell), so a row with nothing
+    # missing costs no pass over its cells.  Short rows are padded, and a
+    # row with a nil is changed, on a copy: the rows may be a parser:
+    # callable's own arrays.
+    rows = rows.each_with_index.map do |r, i|
+      if r.size > ncol
+        raise ArgumentError,
+              "row #{i + 1} has #{r.size} fields, expected #{ncol}"
+      end
+      r = r + Array.new(ncol - r.size) if r.size < ncol
+      # compact tests for nil without calling == on every cell, as
+      # include?(nil) would.
+      r = r.map { |cell| cell.nil? ? UNDEF : cell } if r.compact.size < ncol
+      r
+    end
+    CArray.object(rows.size, ncol) { rows }
   end
-  private_class_method :build_frame
+  private_class_method :table_of_rows
 end