carray 3.0.2 → 3.0.4

data/lib/carray/frame/csv_parser.rb

@@ -13,7 +13,11 @@
 #   * An empty unquoted field is nil (missing); an empty quoted field ("") is
 #     the empty String. Both cast to UNDEF for numeric columns (parse-mask).
 #   * A UTF-8 BOM is stripped via the "bom|utf-8" read mode by default.
-#   * A parse error names the record number.
+#   * Malformed input raises MalformedCSV naming the file and the line: a
+#     quote inside an unquoted field, text after a closing quote, a quoted
+#     field still open at the end of input, or a record with more fields
+#     than there are columns. A record continues onto the next line only
+#     inside a quoted field.
 #
 # Two entry points share the Tokenizer:
 #   * CSVParser.parse / parse_file -- whole-file parse into [headers, rows].
@@ -22,6 +26,7 @@
 #                                     column_names).
 
 require "strscan"
+require "stringio"
 
 class CAFrame
   # CSV tokenizer behind `CAFrame.from_csv`.  It produces raw String cells
@@ -36,9 +41,35 @@ class CAFrame
   module CSVParser
     module_function
 
-    # Raised when the input cannot be tokenized, e.g. a quoted field that
-    # never closes before end of input.
-    class MalformedCSV < StandardError; end
+    # Raised when the input cannot be read as written. The message starts
+    # with "path:line:" (or "line N:" for an IO with no path), the form an
+    # editor or terminal jumps to; the record number follows when it is not
+    # the line's, as after a quoted field of several lines or a +skip+.
+    # +path+, +lineno+ and +record+ give the same for a program.
+    class MalformedCSV < StandardError
+      attr_accessor :path
+      attr_reader :lineno, :record, :detail
+
+      def initialize(detail = nil, lineno: nil, record: nil, path: nil)
+        @detail = detail
+        @lineno = lineno
+        @record = record
+        @path   = path
+        super(detail)
+      end
+
+      def message
+        return super if @detail.nil?
+        where =
+          if @path && @lineno then "#{@path}:#{@lineno}: "
+          elsif @path         then "#{@path}: "
+          elsif @lineno       then "line #{@lineno}: "
+          else ""
+          end
+        tail = @record && @record != @lineno ? " (record #{@record})" : ""
+        "#{where}#{@detail}#{tail}"
+      end
+    end
 
     # Parse a file into [headers, rows]. +encoding+ is an IO open-mode encoding
     # string; the default strips a leading BOM and reads UTF-8.
@@ -47,7 +78,8 @@ class CAFrame
     end
 
     # Parse an IO (or anything answering +gets+). The first record supplies the
-    # headers; the rest are data rows. Fully blank lines are skipped.
+    # headers; the rest are data rows. Blank lines are skipped, including in a
+    # one-column file (from_csv reads them there as missing cells).
     def parse(io, sep: ",", quote: '"', strip: false)
       tok = Tokenizer.new(sep, quote, strip)
       headers = nil
@@ -62,111 +94,250 @@ class CAFrame
       [headers, rows]
     end
 
-    # Read one logical record, joining continuation lines while a quoted field
-    # is still open (an odd number of quote characters means unbalanced).
-    def read_record(io, quote)
-      line = io.gets
-      return nil if line.nil?
-      rec = line.dup
-      while rec.count(quote).odd?
-        more = io.gets
-        raise MalformedCSV, "unterminated quoted field at end of input" if more.nil?
-        rec << more
-      end
-      rec.chomp!
-      rec
-    end
-
     # Splits records into fields. Regexps are compiled once and reused across
     # every record, so per-row cost stays low.
     class Tokenizer
+      attr_reader :sep, :quote, :strip
+
       def initialize(sep, quote, strip)
+        unless sep.is_a?(String) && !sep.empty?
+          raise ArgumentError, "sep: must be a non-empty String (got #{sep.inspect})"
+        end
+        unless quote.is_a?(String) && quote.size == 1
+          raise ArgumentError, "quote: must be one character (got #{quote.inspect})"
+        end
+        if sep.include?(quote)
+          raise ArgumentError, "sep: #{sep.inspect} cannot contain the quote character"
+        end
         @sep      = sep
         @quote    = quote
         @strip    = strip
-        @sep_re   = /#{Regexp.escape(sep)}/
-        @quote_re = /#{Regexp.escape(quote)}/
-        @unquoted = /[^#{Regexp.escape(sep)}]*/
-        @inner    = /[^#{Regexp.escape(quote)}]*/
+        s = Regexp.escape(sep)
+        q = Regexp.escape(quote)
+        @sep_re   = /#{s}/
+        # String#split(" ") splits on runs of whitespace (awk style); a
+        # Regexp splits at each separator as written, as the C reader does.
+        @split_sep = sep == " " ? @sep_re : sep
+        # strip: skips the spaces and tabs around a field, but not one that is
+        # the separator: with sep: "\t" a tab ends the field.
+        pad = [" ", "\t"].reject { |c| sep.include?(c) }
+        @pad_re = pad.empty? ? nil : /[#{Regexp.escape(pad.join)}]+/
+        @quote_re = /#{q}/
+        # An unquoted field runs up to the separator, a quote or the end of
+        # the record; a CR that does not end the record is part of the field.
+        # A character class says "up to the separator" for a one-character
+        # one; a longer one has to be matched as a whole, or "::" would end
+        # the field at a single ":".
+        @unquoted =
+          if sep.size == 1
+            /(?:[^#{s}#{q}\r\n]|\r(?!\n|\z))*/
+          else
+            /(?:(?!#{s})(?:[^#{q}\r\n]|\r(?!\n|\z)))*/
+          end
+        @inner    = /[^#{q}]*/
+        @eol      = /\r?\n|\r\z/
+        @blank_re = /\A[ \t]*\z/
+        @sep2     = sep * 2
+        @qq       = quote * 2
+        # A whole quoted field: its quotes closed, the ones inside doubled.
+        @quoted_re = /\A#{q}(?:[^#{q}]|#{q}#{q})*#{q}\z/m
         @recno    = 0
+        @lineno   = 0
+        @record_line = 0
+      end
+
+      # The lines read so far, and the line the last record started on.
+      attr_reader :lineno, :record_line
+
+      # The next line of +io+, counted.
+      def gets(io)
+        line = io.gets
+        @lineno += 1 if line
+        line
+      end
+
+      # A MalformedCSV at +line+ of the current record.
+      def malformed(detail, line = @record_line)
+        MalformedCSV.new(detail, lineno: line, record: @recno)
       end
 
-      # Fields of the next record, or nil at EOF.  A blank line carries no
-      # separator, so it cannot be a row of a file with more than one column
-      # and is skipped as noise between records.  In a single-column file it
-      # is the only spelling a missing single field has -- which is what
-      # to_csv writes for a masked cell -- so the caller passes
+      # Fields of the next record, or nil at EOF.  A blank line -- empty, or
+      # only spaces and tabs with no separator -- cannot be a row of a file
+      # with more than one column and is skipped as noise between records.
+      # In a single-column file it is a row: an empty line is the only
+      # spelling a missing single field has, which is what to_csv writes for
+      # a masked cell, and spaces are a value. So the caller passes
       # +blank_is_row: true+ once the column count is known to be one, and
-      # the empty record becomes a row of no fields for build_frame to pad.
-      def read(io, blank_is_row: false)
+      # the line becomes a row for build_frame to pad. +blank?+ says whether
+      # the record just returned was such a line, for a caller that learns
+      # the column count only after the body.
+      #
+      # A line with no quote character is a whole record. One with a quote
+      # goes to the scanner, which reads further lines only while a field
+      # that opened with a quote is still open -- so a quote inside an
+      # unquoted field is malformed rather than a reason to swallow the lines
+      # after it, and a field of many lines is read once, not re-counted.
+      def read(io, blank_is_row: false, last_line: nil)
         loop do
-          rec = CSVParser.read_record(io, @quote)
-          return nil if rec.nil?
+          return nil if last_line && @lineno >= last_line
+          line = gets(io)
+          return nil if line.nil?
           @recno += 1
-          next if rec.empty? && !blank_is_row
-          return rec.count(@quote).zero? ? simple(rec) : scan(rec)
+          @record_line = @lineno
+          @blank = false
+          if line.include?(@quote)
+            return (!@strip && quoted_line(line)) || scan(line, io)
+          end
+          rec = line.chomp
+          if rec.match?(@blank_re) && !rec.include?(@sep)
+            next unless blank_is_row
+            @blank = true
+          end
+          return simple(rec)
         end
       end
 
+      def blank?
+        @blank
+      end
+
+      # Count records and lines read elsewhere (the C reader), so the
+      # numbers in an error stay those of the file.
+      def advance(records, lines)
+        @recno += records
+        @lineno += lines
+      end
+
+      # An empty field lies at the start or the end of the record or between
+      # two separators, so a record without one is the split as it stands.
       private def simple(rec)
-        rec.split(@sep, -1).map do |cell|
-          cell = cell.strip if @strip
-          cell.empty? ? nil : cell
+        fields = rec.split(@split_sep, -1)
+        if @strip
+          fields.map! { |cell| cell = cell.strip; cell.empty? ? nil : cell }
+        elsif rec.empty? || rec.start_with?(@sep) || rec.end_with?(@sep) || rec.include?(@sep2)
+          fields.map! { |cell| cell.empty? ? nil : cell }
         end
+        fields
       end
 
-      private def scan(rec)
-        sc = StringScanner.new(rec)
+      # The fields of a one-line record holding quotes, read by splitting at
+      # the separator: a quoted field is the pieces from one that opens with
+      # a quote to the one that closes it (it may hold the separator). nil
+      # when the record is not that simple -- a quoted field open at the end
+      # of the line, a quote inside an unquoted field, text after a closing
+      # quote -- for the scanner to read or to report.
+      private def quoted_line(line)
+        parts = line.chomp.split(@split_sep, -1)
+        fields = []
+        i = 0
+        while i < parts.size
+          part = parts[i]
+          if part.start_with?(@quote)
+            until part.match?(@quoted_re)
+              i += 1
+              return nil if i >= parts.size
+              part = part + @sep + parts[i]
+            end
+            inner = part[1...-1]
+            fields << (inner.include?(@quote) ? inner.gsub(@qq, @quote) : inner)
+          elsif part.include?(@quote)
+            return nil
+          else
+            fields << (part.empty? ? nil : part)
+          end
+          i += 1
+        end
+        fields
+      end
+
+      # Fields of the record that starts on +line+, reading more lines from
+      # +io+ while a quoted field is open. strip: skips spaces before a field,
+      # so a quote after them still opens one.
+      private def scan(line, io)
+        sc = StringScanner.new(line)
         fields = []
         loop do
-          if sc.scan(@quote_re)
-            buf = +""
-            loop do
-              buf << sc.scan(@inner)
-              unless sc.scan(@quote_re)
-                raise MalformedCSV, "unterminated quoted field in record #{@recno}"
-              end
-              if sc.scan(@quote_re) # doubled quote -> literal quote, field continues
-                buf << @quote
-              else
-                break
-              end
+          sc.skip(@pad_re) if @strip && @pad_re
+          if sc.skip(@quote_re)
+            fields << quoted_field(sc, io, fields.size + 1, line_at(sc))
+            # A closed quoted field ends at the separator or the end of the
+            # record. Anything else there is malformed, and read on it would
+            # drop the rest of the record without a word. strip: lets spaces
+            # through, as it does around an unquoted field.
+            sc.skip(@pad_re) if @strip && @pad_re
+            unless sc.eos? || sc.check(@sep_re) || sc.check(@eol)
+              raise malformed("text #{sc.rest.chomp[0, 20].inspect} after the closing quote " \
+                              "of field #{fields.size}", line_at(sc))
             end
-            fields << buf
           else
-            cell = sc.scan(@unquoted) || ""
+            cell = sc.scan(@unquoted)
+            if sc.check(@quote_re)
+              raise malformed("a quote inside unquoted field #{fields.size + 1} " \
+                              "(#{(cell + sc.rest.chomp)[0, 20].inspect}); a field holding " \
+                              "a quote is written quoted, with the quote doubled", line_at(sc))
+            end
             cell = cell.strip if @strip
             fields << (cell.empty? ? nil : cell)
           end
-          break unless sc.scan(@sep_re)
+          break unless sc.skip(@sep_re)
         end
+        sc.skip(@eol)
         fields
       end
+
+      # The line of the file the scanner is at: the record's first line and
+      # the line breaks it has passed in quoted fields.
+      private def line_at(sc)
+        @record_line + sc.string[0, sc.pos].count("\n")
+      end
+
+      # The rest of a field whose opening quote has been read, through its
+      # closing quote. A doubled quote is a literal one; the end of the line
+      # inside the field brings the next line in.
+      private def quoted_field(sc, io, field_no, open_line)
+        buf = +""
+        loop do
+          buf << sc.scan(@inner)
+          if sc.eos?
+            more = gets(io)
+            if more.nil?
+              raise malformed("quoted field #{field_no} opened here is never closed", open_line)
+            end
+            sc << more
+            next
+          end
+          sc.skip(@quote_re)
+          break unless sc.skip(@quote_re)
+          buf << @quote
+        end
+        buf
+      end
     end
   end
 
-  # The block reading-control DSL for CAFrame.from_csv. A file
-  # often has preamble lines, a units row, or no header at all; the block says,
-  # in order, how to consume the stream:
+  # The reading-control block for CAFrame.from_csv. A file often has
+  # preamble lines, a units row, or no header at all; the block is given the
+  # reader and says, in order, how to consume the stream:
   #
-  #   CAFrame.from_csv(path) do
-  #     skip 3          # drop 3 preamble lines
-  #     header          # next record supplies the column names
-  #     skip 1          # drop a units row
-  #     body            # the rest are data rows
+  #   CAFrame.from_csv(path) do |r|
+  #     r.skip 3          # drop 3 preamble lines
+  #     r.header          # next record supplies the column names
+  #     r.skip 1          # drop a units row
+  #     r.data            # the rest are data rows
   #   end
   #
-  #   CAFrame.from_csv(path) do   # headerless file
-  #     column_names "date", "temp", "rh"
-  #     body
-  #   end
+  #   CAFrame.from_csv(path) { it.column_names "date", "temp", "rh"; it.data }
   #
-  # The verbs are +skip+ / +header+ / +column_names+ / +body+; each returns a
-  # value useful inline (header returns its fields) and the ordering is the
-  # script. Without a block, from_csv runs the default +header+ then +body+.
+  # The verbs are +skip+ / +header+ / +column_names+ / +columns+ / +data+;
+  # each returns a value useful inline (header returns its fields) and the ordering is the
+  # script. The reader is a parameter, not self, so a local variable named
+  # +data+ or +header+ cannot stand in for the verb. Without a block,
+  # from_csv reads by its header: / data: / column_names: (see +layout+).
   class CSVReader
-    def initialize(io, sep: ",", quote: '"', strip: false)
+    def initialize(io, sep: ",", quote: '"', strip: false, read_text_as_const_string: false)
+      @read_text_as_const_string = read_text_as_const_string
       @io    = io
+      @path  = io.path if io.respond_to?(:path)
       @tok   = CSVParser::Tokenizer.new(sep, quote, strip)
       @names = nil
       @rows  = nil
@@ -174,45 +345,475 @@ class CAFrame
 
     # Drop +n+ raw lines (preamble, units, notes).
     def skip(n = 1)
-      n.times { @io.gets }
+      unless n.is_a?(Integer) && n >= 0
+        raise ArgumentError, "skip takes a number of lines (got #{n.inspect})"
+      end
+      n.times { @tok.gets(@io) }
+      @skipped = true if n > 0
       self
     end
 
     # Read one record. With no argument it becomes the column names; with a
-    # name it is a secondary header (e.g. units) -- read, returned, not used as
-    # names. Returns the record's fields either way.
+    # name it is a secondary header (e.g. units): its field j becomes the
+    # attribute +name+ of column j (an empty field gives none). Returns the
+    # record's fields either way.
     def header(name = nil)
-      fields = @tok.read(@io)
-      raise CSVParser::MalformedCSV, "header expected but input ended" if fields.nil?
+      # A named header of a file with at most one column may be an empty
+      # line: that is how to_csv writes a column without the attribute, so
+      # the line is the header's, as an empty line is a row in the data.
+      blank_is_row = !name.nil? && !@names.nil? && @names.size <= 1
+      fields = @tok.read(@io, blank_is_row: blank_is_row)
+      if fields.nil?
+        # A file with no columns has nothing to give a named header.
+        return [] if !name.nil? && @names&.empty?
+        # An input with no record at all is a table with no columns: that is
+        # what to_csv writes for a frame without columns or index.  Once lines
+        # have been skipped to reach the header, a missing header is an error.
+        if name.nil? && !@skipped
+          @names = []
+          @header_line = @tok.lineno
+          @names_to_check = nil
+          return []
+        end
+        raise CSVParser::MalformedCSV.new("header expected but input ended",
+                                          lineno: @tok.lineno + 1)
+      end
       if name.nil?
-        @names = fields.map(&:to_s)
+        # An empty field names no column: the column takes the name a file
+        # without a header gives it, from its place in the line.
+        @names = fields.each_with_index.map { |f, j| f.to_s.empty? ? "c#{j}" : f.to_s }
+        @header_line = @tok.record_line
+        @names_to_check = nil
       else
-        (@named_headers ||= {})[name.to_s] = fields
+        key = name.to_s
+        @named_headers ||= {}
+        if @named_headers.key?(key)
+          raise ArgumentError, "from_csv: header(#{name.inspect}) is read twice"
+        end
+        @named_headers[key] = [fields, @tok.record_line, name]
       end
       fields
     end
 
-    # Set the column names explicitly (headerless files).
+    # Set the column names explicitly: for a file with no header, or in
+    # place of the names a header gave. Their number has to be the file's
+    # number of columns -- the header's, or else that of the first record
+    # of the data -- or a name would go to no column, or a column without
+    # one.
     def column_names(*names)
-      @names = names.flatten.map(&:to_s)
+      names = names.flatten.map(&:to_s)
+      if names.include?("")
+        raise ArgumentError, "from_csv: column_names: a column name can not be empty"
+      end
+      if @header_line
+        check_column_names(names, @names.size, @header_line)
+      else
+        @names_to_check = names
+      end
+      @names = names
+      self
+    end
+
+    # Read only these columns: names of the header or of column_names, or
+    # indexes of the file's columns from 0 (Integers or Ranges), in the order
+    # the frame is to have them. The fields of the others are read past, so
+    # a few columns of a wide file cost little more than a narrow file.
+    #
+    # A Hash selects by its keys, as above (a name or an index, not a Range),
+    # and names the columns read with its values:
+    #
+    #   r.columns(2 => "code", "c7" => "area1")
+    def columns(*selection)
+      @renames = nil
+      if selection.size == 1 && selection[0].is_a?(Hash)
+        @renames = column_renames(selection[0])
+        selection = selection[0].keys
+      end
+      selection = selection.flatten
+      if selection.empty?
+        raise ArgumentError, "from_csv: columns: names no column"
+      end
+      selection.each do |c|
+        next if c.is_a?(String)
+        CAFrame.__send__(:column_key, c, "from_csv", "columns: a column name") if c.is_a?(Symbol)
+        next if c.is_a?(Integer) && c >= 0
+        next if c.is_a?(Range) && (c.begin.nil? || (c.begin.is_a?(Integer) && c.begin >= 0)) &&
+                (c.end.nil? || c.end.is_a?(Integer))
+        raise ArgumentError, "from_csv: columns: takes column names, or indexes from 0 " \
+                             "and Ranges of them (got #{c.inspect})"
+      end
+      @selection = selection
       self
     end
 
-    # Consume the remaining records as data rows.
-    def body
-      rows = []
-      blank_is_row = @names && @names.size == 1
-      while (fields = @tok.read(@io, blank_is_row: blank_is_row))
+    # The new names a Hash given to +columns+ holds, checked.
+    private def column_renames(map)
+      map.each do |key, name|
+        if key.is_a?(Range)
+          raise ArgumentError, "from_csv: columns: a Hash selects one column per key, " \
+                               "not a Range (got #{key.inspect})"
+        end
+        CAFrame.__send__(:column_key, name, "from_csv", "columns: a column name",
+                         hint: "given for #{key.inspect}")
+        if name.empty?
+          raise ArgumentError, "from_csv: columns: a column name can not be empty " \
+                               "(given for #{key.inspect})"
+        end
+      end
+      CAFrame.__send__(:refuse_repeated_names, map.values, "from_csv", "columns: names")
+      map.values
+    end
+
+    # The indexes of the selected columns, in the order asked for, among
+    # ncol columns named +names+ (nil: by index only). nil when ncol is not
+    # known yet and the selection needs it.
+    private def selected_indexes(names, ncol)
+      indexes = @selection.flat_map do |c|
+        case c
+        when String
+          # Without a header the columns are named c0, c1, ... by position.
+          j = names ? names.index(c) : c[/\Ac(0|[1-9]\d*)\z/, 1]&.to_i
+          if j.nil?
+            raise KeyError, "from_csv: columns: names no column #{c.inspect}" +
+                            (names ? "" : " (with no header the columns are c0, c1, ...)")
+          end
+          j
+        when Integer then c
+        else
+          first = c.begin || 0
+          last = c.end
+          return nil if last.nil? && ncol.nil?
+          last = ncol - 1 if last.nil?
+          last -= 1 if c.end && c.exclude_end?
+          (first..last).to_a
+        end
+      end
+      CAFrame.__send__(:refuse_repeated_names, indexes, "from_csv", "columns: selects column")
+      if ncol && (bad = indexes.find { |j| j >= ncol })
+        raise ArgumentError, "from_csv: columns: #{bad} is past the #{ncol} columns of the file"
+      end
+      indexes
+    end
+
+    # The indexes C is to read, ascending, when it can tell which they are
+    # before it reads; nil for every column.
+    private def columns_to_read
+      @selection && selected_indexes(@names, @names&.size)&.sort
+    end
+
+    private def check_column_names(names, ncol, line)
+      return if names.size == ncol
+      raise ArgumentError, "from_csv: column_names: gives #{names.size} " \
+                           "name#{names.size == 1 ? '' : 's'} for the #{ncol} columns of line #{line}"
+    end
+
+    # Consume the remaining records as data rows. A second data adds what is
+    # left, which is nothing; it does not discard the rows already read.
+    #
+    # Without names the column count is known only once every row is read,
+    # so blank lines are kept and marked, and +result+ drops them unless the
+    # file turns out to have one column -- the same rows a header would have
+    # given.
+    #
+    # Without strip:, the rest of the input is read in C (CArray.__csv_read_body_as_string_cells__)
+    # straight into the table, with the column count of the names, or of the
+    # first record when there are none. When C declines -- text not in
+    # UTF-8, a record it does not take, or one longer than the first in a
+    # file without names -- the text from there on is read here, which pads
+    # the earlier rows or reports what is wrong.
+    def data
+      read_data(nil)
+    end
+
+    # The header: / data: / column_names: of from_csv, as the verbs above.
+    # They index the lines of the file from 0, as File.readlines does (an
+    # error names a line from 1, as an editor does). header: defaults to 0,
+    # or to none (false) when column_names: is given; data: is the first line
+    # of the data or a Range of lines, by default from the line after the
+    # header. A record that starts within data: is read whole.
+    def layout(header: nil, data: nil, column_names: nil, columns: nil, attrs: nil)
+      header = column_names ? false : 0 if header.nil?
+      unless header == false || (header.is_a?(Integer) && header >= 0)
+        raise ArgumentError, "from_csv: header: takes the index of a line, from 0, " \
+                             "or false (got #{header.inspect})"
+      end
+      attr_keys = attrs.nil? ? [] : CAFrame.send(:csv_attr_keys, attrs, "from_csv")
+      if !header && !attr_keys.empty?
+        raise ArgumentError, "from_csv: attrs: reads the lines below the header, " \
+                             "so it needs a header line"
+      end
+      first, last = CSVReader.data_lines(data)
+      if header
+        skip(header - @tok.lineno) if header > @tok.lineno
+        self.header
+        attr_keys.each { |key| self.header(key.to_sym) }
+      end
+      column_names(*column_names) if column_names
+      self.columns(*(columns.is_a?(Array) ? columns : [columns])) if columns
+      first ||= @tok.lineno
+      if first < @tok.lineno
+        raise ArgumentError, "from_csv: data: #{first} does not come after header: #{header}" +
+                             (attr_keys.empty? ? "" : " and the #{attr_keys.size} attrs: lines")
+      end
+      skip(first - @tok.lineno)
+      read_data(last && last + 1)
+    end
+
+    # [first, last] line indexes of a data: setting, nil for an end left open.
+    def self.data_lines(spec)
+      first, last =
+        case spec
+        when nil     then [nil, nil]
+        when Integer then [spec, nil]
+        when Range
+          e = spec.end
+          e -= 1 if e.is_a?(Integer) && spec.exclude_end?
+          [spec.begin, e]
+        else
+          raise ArgumentError, "from_csv: data: takes the index of a line or a Range of " \
+                               "them, from 0 (got #{spec.inspect})"
+        end
+      [first, last].each do |n|
+        next if n.nil? || (n.is_a?(Integer) && n >= 0)
+        raise ArgumentError, "from_csv: data: lines are indexed from 0 (got #{spec.inspect})"
+      end
+      if first && last && last < first
+        raise ArgumentError, "from_csv: data: #{spec.inspect} ends before it starts"
+      end
+      [first, last]
+    end
+
+    # The records from here through those starting on +last+ (nil: the end).
+    private def read_data(last)
+      body_in_c(last) if @rows.nil? && !@tok.strip && @io.respond_to?(:read)
+      return self if @table
+      rows = (@rows ||= [])
+      @blank_rows ||= []
+      blank_is_row = @names.nil? || @names.size == 1
+      ncol = @names&.size
+      while (fields = @tok.read(@io, blank_is_row: blank_is_row, last_line: last))
+        if @names_to_check && !@tok.blank?
+          check_column_names(@names_to_check, fields.size, @tok.record_line)
+          @names_to_check = nil
+        end
+        if ncol && fields.size > ncol
+          raise @tok.malformed("#{fields.size} fields, but there are #{ncol} columns")
+        end
+        @blank_rows << rows.size if @names.nil? && @tok.blank?
         rows << fields
       end
-      @rows = rows
       self
     end
 
+    # Run the block, naming the file in a MalformedCSV it raises.
+    def reporting
+      yield
+    rescue CSVParser::MalformedCSV => e
+      e.path ||= @path
+      raise
+    end
+
+    # The text C reads at a time when the column count is known; a chunk
+    # ends at a line end, and not inside a quoted field.
+    CHUNK_BYTES = 1 << 22
+
+    # With names, the input goes to C a chunk at a time, so the text is not
+    # held whole beside its cells. Without them, the column count and which
+    # blank lines are rows are known only at the end, so it goes in whole;
+    # so it does when the IO transcodes, since read(length) does not.
+    #
+    # With read_text_as_const_string:, the body goes to C whole, into one
+    # CAConstString per column over the text itself: the text is the buffer
+    # the columns read, so it is held whole either way. When C declines, the
+    # text is read here from the start of the body.
+    private def read_body_into_const_string_columns
+      ncol = @names && !@names_to_check ? @names.size : 0
+      keep = columns_to_read
+      text = read_rest
+      n, buffer, pairs, records = CArray.__csv_read_body_as_const_string_columns__(text, @tok.sep, @tok.quote, ncol, keep)
+      unless buffer
+        @rows = []
+        @io = StringIO.new(text)
+        return
+      end
+      if @names_to_check
+        check_column_names(@names_to_check, n, @tok.lineno + first_record_line(text))
+        @names_to_check = nil
+      end
+      @tok.advance(records, text.count("\n") + (text.empty? || text.end_with?("\n") ? 0 : 1))
+      @table = ConstStringColumns.new(pairs.map { |pr| CAConstString.wrap(pr, buffer: buffer) })
+      @table_ncol = n
+      @table_keep = keep
+      @rows = []
+    end
+
+    # The columns read_body_into_const_string_columns made, as +result+
+    # hands them to CAFrame.from_csv.
+    ConstStringColumns = Struct.new(:columns)
+
+    private def body_in_c(last = nil, chunk_bytes = CHUNK_BYTES)
+      return read_body_into_const_string_columns if last.nil? && @read_text_as_const_string
+      chunked = last.nil? && @names && @io.respond_to?(:external_encoding) &&
+                @io.internal_encoding.nil?
+      # Names that have to agree with the first record leave its column
+      # count to C, to be checked against them.
+      ncol = @names && !@names_to_check ? @names.size : 0
+      keep = columns_to_read
+      cells = []
+      loop do
+        text = if chunked then next_chunk(chunk_bytes)
+               elsif last then lines_through(last)
+               else read_rest
+               end
+        break if text.nil?
+        n, flat, records = CArray.__csv_read_body_as_string_cells__(text, @tok.sep, @tok.quote, ncol, keep)
+        unless flat
+          # This chunk and the rest are read by the Ruby tokenizer, after the
+          # rows read so far.
+          if keep && ncol > 0 && !cells.empty?
+            # The rows read so far hold the selected fields only; spread
+            # them back to their columns for the Ruby reader.
+            @rows = cells.each_slice(keep.size).map do |row|
+              full = Array.new(ncol)
+              keep.each_with_index { |j, k| full[j] = UNDEF.equal?(row[k]) ? nil : row[k] }
+              full
+            end
+          else
+            @rows = ncol > 0 ? cells.each_slice(ncol).to_a : []
+          end
+          @io = StringIO.new(chunked ? text + (@io.read || "") : text)
+          return
+        end
+        if @names_to_check
+          check_column_names(@names_to_check, n, @tok.lineno + first_record_line(text))
+          @names_to_check = nil
+        end
+        ncol = n
+        cells.concat(flat)
+        @tok.advance(records, text.count("\n") + (text.empty? || text.end_with?("\n") ? 0 : 1))
+        break unless chunked
+      end
+      if ncol == 0                    # no record, for the names to be checked against
+        @rows = []
+        return
+      end
+      width = keep ? keep.size : ncol
+      @table = CArray.object(cells.size / width, width) { cells }
+      @table_ncol = ncol
+      @table_keep = keep
+      @rows = []
+    end
+
+    # The line, from 1 within +text+, of its first record: past the blank
+    # lines before it, as C reads them.
+    private def first_record_line(text)
+      line = 1
+      text.each_line do |l|
+        rec = l.chomp
+        break unless rec.match?(/\A[ \t]*\z/) && !rec.include?(@tok.sep)
+        line += 1
+      end
+      line
+    end
+
+    # The lines through +last+, read on past any quoted field still open.
+    private def lines_through(last)
+      buf = +""
+      (last - @tok.lineno).times do
+        line = @io.gets or break
+        buf << line
+      end
+      while buf.count(@tok.quote).odd? && (more = @io.gets)
+        buf << more
+      end
+      buf
+    end
+
+    # The rest of the input, in the IO's encoding. Read in pieces when the IO
+    # does not transcode: IO#read without a length checks the text's encoding
+    # as it reads, and the C reader checks it again faster.
+    private def read_rest
+      unless @io.respond_to?(:internal_encoding) && @io.internal_encoding.nil?
+        return @io.read || ""
+      end
+      buf = +""
+      while (piece = @io.read(CHUNK_BYTES))
+        buf << piece
+      end
+      buf.force_encoding(@io.external_encoding || Encoding.default_external)
+    end
+
+    # The next chunk_bytes of the input, read on to the end of its line and
+    # past any quoted field still open, in the IO's encoding; nil at EOF.
+    private def next_chunk(chunk_bytes)
+      buf = @io.read(chunk_bytes)
+      return nil if buf.nil?
+      buf.force_encoding(@io.external_encoding || Encoding.default_external)
+      if !buf.end_with?("\n") && (rest = @io.gets)
+        buf << rest
+      end
+      while buf.count(@tok.quote).odd? && (more = @io.gets)
+        buf << more
+      end
+      buf
+    end
+
+    # The attributes the named headers give the columns +result+ returns, as
+    # [{key => value}, ...] in the same order. A column of the file the
+    # selection left out takes no part.
+    def column_attrs (ncol)
+      return nil unless @named_headers
+      indexes = @selection ? selected_indexes(@names, ncol) : (0...ncol).to_a
+      attrs = indexes.map { {} }
+      @named_headers.each do |key, (fields, line, name)|
+        if fields.size > ncol
+          raise ArgumentError, "from_csv: header(#{name.inspect}) on line #{line} " \
+                               "has #{fields.size} fields for the #{ncol} columns"
+        end
+        indexes.each_with_index do |j, k|
+          value = fields[j]
+          attrs[k][key] = value unless value.nil?
+        end
+      end
+      attrs
+    end
+
+    # The number of columns of the file, once the data is read.
+    def file_ncol
+      return @table_ncol if @table
+      @names ? @names.size : ((@rows || []).map(&:size).max || 0)
+    end
+
     # [names_or_nil, rows] for CAFrame.from_csv to build from. names is nil when
     # neither header nor column_names ran (positional names are generated).
     def result
-      [@names, @rows || []]
+      return table_result if @table
+      rows = @rows || []
+      if @names.nil? && @blank_rows && !@blank_rows.empty? &&
+         rows.map(&:size).max != 1
+        drop = @blank_rows.to_h { |i| [i, true] }
+        rows = rows.reject.with_index { |_, i| drop[i] }
+      end
+      return [@names, rows] unless @selection
+      ncol = @names ? @names.size : (rows.map(&:size).max || 0)
+      indexes = selected_indexes(@names, ncol)
+      names = @renames || indexes.map { |j| @names ? @names[j] : "c#{j}" }
+      [names, rows.map { |r| r.values_at(*indexes) }]
+    end
+
+    # The table C read, its columns those selected, in the order asked for.
+    private def table_result
+      return [@names, @table] unless @selection
+      indexes = selected_indexes(@names, @table_ncol)
+      names = @renames || indexes.map { |j| @names ? @names[j] : "c#{j}" }
+      kept = @table_keep || (0...@table_ncol).to_a
+      order = indexes.map { |j| kept.index(j) }
+      table =
+        if order == (0...kept.size).to_a then @table
+        elsif @table.is_a?(ConstStringColumns) then ConstStringColumns.new(@table.columns.values_at(*order))
+        else @table[nil, CA_INT64(order)]
+        end
+      [names, table]
     end
   end
 end