csv 3.2.2 → 3.3.5

data/lib/csv/parser.rb

@@ -2,15 +2,10 @@
 
 require "strscan"
 
-require_relative "delete_suffix"
 require_relative "input_record_separator"
-require_relative "match_p"
 require_relative "row"
 require_relative "table"
 
-using CSV::DeleteSuffix if CSV.const_defined?(:DeleteSuffix)
-using CSV::MatchP if CSV.const_defined?(:MatchP)
-
 class CSV
   # Note: Don't use this class directly. This is an internal class.
   class Parser
@@ -23,10 +18,27 @@ class CSV
     # into your Encoding.
     #
 
+    class << self
+      ARGF_OBJECT_ID = ARGF.object_id
+      # Convenient method to check whether the give input reached EOF
+      # or not.
+      def eof?(input)
+        # We can't use input != ARGF in Ractor. Because ARGF isn't a
+        # shareable object.
+        input.object_id != ARGF_OBJECT_ID and
+          input.respond_to?(:eof) and
+          input.eof?
+      end
+    end
+
     # Raised when encoding is invalid.
     class InvalidEncoding < StandardError
     end
 
+    # Raised when unexpected case is happen.
+    class UnexpectedError < StandardError
+    end
+
     #
     # CSV::Scanner receives a CSV output, scans it and return the content.
     # It also controls the life cycle of the object with its methods +keep_start+,
@@ -35,7 +47,7 @@ class CSV
     # Uses StringScanner (the official strscan gem). Strscan provides lexical
     # scanning operations on a String. We inherit its object and take advantage
     # on the methods. For more information, please visit:
-    # https://ruby-doc.org/stdlib-2.6.1/libdoc/strscan/rdoc/StringScanner.html
+    # https://docs.ruby-lang.org/en/master/StringScanner.html
     #
     class Scanner < StringScanner
       alias_method :scan_all, :scan
@@ -78,10 +90,10 @@ class CSV
     # +keep_end+, +keep_back+, +keep_drop+.
     #
     # CSV::InputsScanner.scan() tries to match with pattern at the current position.
-    # If there's a match, the scanner advances the “scan pointer” and returns the matched string.
+    # If there's a match, the scanner advances the "scan pointer" and returns the matched string.
     # Otherwise, the scanner returns nil.
     #
-    # CSV::InputsScanner.rest() returns the “rest” of the string (i.e. everything after the scan pointer).
+    # CSV::InputsScanner.rest() returns the "rest" of the string (i.e. everything after the scan pointer).
     # If there is no more data (eos? = true), it returns "".
     #
     class InputsScanner
@@ -96,11 +108,13 @@ class CSV
       end
 
       def each_line(row_separator)
+        return enum_for(__method__, row_separator) unless block_given?
         buffer = nil
         input = @scanner.rest
         position = @scanner.pos
         offset = 0
         n_row_separator_chars = row_separator.size
+        # trace(__method__, :start, input)
         while true
           input.each_line(row_separator) do |line|
             @scanner.pos += line.bytesize
@@ -140,25 +154,29 @@ class CSV
       end
 
       def scan(pattern)
+        # trace(__method__, pattern, :start)
         value = @scanner.scan(pattern)
+        # trace(__method__, pattern, :done, :last, value) if @last_scanner
         return value if @last_scanner
 
-        if value
-          read_chunk if @scanner.eos?
-          return value
-        else
-          nil
-        end
+        read_chunk if value and @scanner.eos?
+        # trace(__method__, pattern, :done, value)
+        value
       end
 
       def scan_all(pattern)
+        # trace(__method__, pattern, :start)
         value = @scanner.scan(pattern)
+        # trace(__method__, pattern, :done, :last, value) if @last_scanner
         return value if @last_scanner
 
+        # trace(__method__, pattern, :done, :nil) if value.nil?
         return nil if value.nil?
         while @scanner.eos? and read_chunk and (sub_value = @scanner.scan(pattern))
+          # trace(__method__, pattern, :sub, sub_value)
           value << sub_value
         end
+        # trace(__method__, pattern, :done, value)
         value
       end
 
@@ -167,68 +185,136 @@ class CSV
       end
 
       def keep_start
-        @keeps.push([@scanner.pos, nil])
+        # trace(__method__, :start)
+        adjust_last_keep
+        @keeps.push([@scanner, @scanner.pos, nil])
+        # trace(__method__, :done)
       end
 
       def keep_end
-        start, buffer = @keeps.pop
-        keep = @scanner.string.byteslice(start, @scanner.pos - start)
+        # trace(__method__, :start)
+        scanner, start, buffer = @keeps.pop
+        if scanner == @scanner
+          keep = @scanner.string.byteslice(start, @scanner.pos - start)
+        else
+          keep = @scanner.string.byteslice(0, @scanner.pos)
+        end
         if buffer
           buffer << keep
           keep = buffer
         end
+        # trace(__method__, :done, keep)
         keep
       end
 
       def keep_back
-        start, buffer = @keeps.pop
+        # trace(__method__, :start)
+        scanner, start, buffer = @keeps.pop
         if buffer
+          # trace(__method__, :rescan, start, buffer)
           string = @scanner.string
-          keep = string.byteslice(start, string.bytesize - start)
+          if scanner == @scanner
+            keep = string.byteslice(start,
+                                    string.bytesize - @scanner.pos - start)
+          else
+            keep = string
+          end
           if keep and not keep.empty?
             @inputs.unshift(StringIO.new(keep))
             @last_scanner = false
           end
           @scanner = StringScanner.new(buffer)
         else
+          if @scanner != scanner
+            message = "scanners are different but no buffer: "
+            message += "#{@scanner.inspect}(#{@scanner.object_id}): "
+            message += "#{scanner.inspect}(#{scanner.object_id})"
+            raise UnexpectedError, message
+          end
+          # trace(__method__, :repos, start, buffer)
           @scanner.pos = start
+          last_scanner, last_start, last_buffer = @keeps.last
+          # Drop the last buffer when the last buffer is the same data
+          # in the last keep. If we keep it, we have duplicated data
+          # by the next keep_back.
+          if last_scanner == @scanner and
+            last_buffer and
+            last_buffer == last_scanner.string.byteslice(last_start, start)
+            @keeps.last[2] = nil
+          end
         end
         read_chunk if @scanner.eos?
       end
 
       def keep_drop
-        @keeps.pop
+        _, _, buffer = @keeps.pop
+        # trace(__method__, :done, :empty) unless buffer
+        return unless buffer
+
+        last_keep = @keeps.last
+        # trace(__method__, :done, :no_last_keep) unless last_keep
+        return unless last_keep
+
+        if last_keep[2]
+          last_keep[2] << buffer
+        else
+          last_keep[2] = buffer
+        end
+        # trace(__method__, :done)
       end
 
       def rest
         @scanner.rest
       end
 
+      def check(pattern)
+        @scanner.check(pattern)
+      end
+
       private
-      def read_chunk
-        return false if @last_scanner
+      def trace(*args)
+        pp([*args, @scanner, @scanner&.string, @scanner&.pos, @keeps])
+      end
 
-        unless @keeps.empty?
-          keep = @keeps.last
-          keep_start = keep[0]
-          string = @scanner.string
-          keep_data = string.byteslice(keep_start, @scanner.pos - keep_start)
-          if keep_data
-            keep_buffer = keep[1]
-            if keep_buffer
-              keep_buffer << keep_data
-            else
-              keep[1] = keep_data.dup
-            end
+      def adjust_last_keep
+        # trace(__method__, :start)
+
+        keep = @keeps.last
+        # trace(__method__, :done, :empty) if keep.nil?
+        return if keep.nil?
+
+        scanner, start, buffer = keep
+        string = @scanner.string
+        if @scanner != scanner
+          start = 0
+        end
+        if start == 0 and @scanner.eos?
+          keep_data = string
+        else
+          keep_data = string.byteslice(start, @scanner.pos - start)
+        end
+        if keep_data
+          if buffer
+            buffer << keep_data
+          else
+            keep[2] = keep_data.dup
           end
-          keep[0] = 0
         end
 
+        # trace(__method__, :done)
+      end
+
+      def read_chunk
+        return false if @last_scanner
+
+        adjust_last_keep
+
         input = @inputs.first
         case input
         when StringIO
           string = input.read
           raise InvalidEncoding unless string.valid_encoding?
+          # trace(__method__, :stringio, string)
           @scanner = StringScanner.new(string)
           @inputs.shift
           @last_scanner = @inputs.empty?
@@ -237,13 +323,15 @@ class CSV
           chunk = input.gets(@row_separator, @chunk_size)
           if chunk
             raise InvalidEncoding unless chunk.valid_encoding?
+            # trace(__method__, :chunk, chunk)
             @scanner = StringScanner.new(chunk)
-            if input.respond_to?(:eof?) and input.eof?
+            if Parser.eof?(input)
               @inputs.shift
               @last_scanner = @inputs.empty?
             end
             true
           else
+            # trace(__method__, :no_chunk)
             @scanner = StringScanner.new("".encode(@encoding))
             @inputs.shift
             @last_scanner = @inputs.empty?
@@ -278,7 +366,11 @@ class CSV
     end
 
     def field_size_limit
-      @field_size_limit
+      @max_field_size&.succ
+    end
+
+    def max_field_size
+      @max_field_size
     end
 
     def skip_lines
@@ -330,21 +422,24 @@ class CSV
 
       begin
         @scanner ||= build_scanner
-        if quote_character.nil?
-          parse_no_quote(&block)
-        elsif @need_robust_parsing
-          parse_quotable_robust(&block)
+        __send__(@parse_method, &block)
+      rescue InvalidEncoding
+        if @scanner
+          ignore_broken_line
+          lineno = @lineno
         else
-          parse_quotable_loose(&block)
+          lineno = @lineno + 1
         end
-      rescue InvalidEncoding
+        raise InvalidEncodingError.new(@encoding, lineno)
+      rescue UnexpectedError => error
         if @scanner
           ignore_broken_line
           lineno = @lineno
         else
           lineno = @lineno + 1
         end
-        message = "Invalid byte sequence in #{@encoding}"
+        message = "This should not be happen: #{error.message}: "
+        message += "Please report this to https://github.com/ruby/csv/issues"
         raise MalformedCSVError.new(message, lineno)
       end
     end
@@ -371,7 +466,6 @@ class CSV
     end
 
     def prepare_variable
-      @need_robust_parsing = false
       @encoding = @options[:encoding]
       liberal_parsing = @options[:liberal_parsing]
       if liberal_parsing
@@ -384,13 +478,12 @@ class CSV
           @double_quote_outside_quote = false
           @backslash_quote = false
         end
-        @need_robust_parsing = true
       else
         @liberal_parsing = false
         @backslash_quote = false
       end
       @unconverted_fields = @options[:unconverted_fields]
-      @field_size_limit = @options[:field_size_limit]
+      @max_field_size = @options[:max_field_size]
       @skip_blanks = @options[:skip_blanks]
       @fields_converter = @options[:fields_converter]
       @header_fields_converter = @options[:header_fields_converter]
@@ -407,7 +500,6 @@ class CSV
           message = ":quote_char has to be nil or a single character String"
           raise ArgumentError, message
         end
-        @double_quote_character = @quote_character * 2
         @escaped_quote_character = Regexp.escape(@quote_character)
         @escaped_quote = Regexp.new(@escaped_quote_character)
       end
@@ -467,7 +559,6 @@ class CSV
           @rstrip_value = Regexp.new(@escaped_strip +
                                      "+\\z".encode(@encoding))
         end
-        @need_robust_parsing = true
       elsif @strip
         strip_values = " \t\f\v"
         @escaped_strip = strip_values.encode(@encoding)
@@ -475,7 +566,6 @@ class CSV
           @strip_value = Regexp.new("[#{strip_values}]+".encode(@encoding))
           @rstrip_value = Regexp.new("[#{strip_values}]+\\z".encode(@encoding))
         end
-        @need_robust_parsing = true
       end
     end
 
@@ -680,9 +770,10 @@ class CSV
       case headers
       when Array
         @raw_headers = headers
+        quoted_fields = FieldsConverter::NO_QUOTED_FIELDS
         @use_headers = true
       when String
-        @raw_headers = parse_headers(headers)
+        @raw_headers, quoted_fields = parse_headers(headers)
         @use_headers = true
       when nil, false
         @raw_headers = nil
@@ -692,27 +783,41 @@ class CSV
         @use_headers = true
       end
       if @raw_headers
-        @headers = adjust_headers(@raw_headers)
+        @headers = adjust_headers(@raw_headers, quoted_fields)
       else
         @headers = nil
       end
     end
 
     def parse_headers(row)
-      CSV.parse_line(row,
-                     col_sep:    @column_separator,
-                     row_sep:    @row_separator,
-                     quote_char: @quote_character)
+      quoted_fields = []
+      converter = lambda do |field, info|
+        quoted_fields << info.quoted?
+        field
+      end
+      headers = CSV.parse_line(row,
+                               col_sep:    @column_separator,
+                               row_sep:    @row_separator,
+                               quote_char: @quote_character,
+                               converters: [converter])
+      [headers, quoted_fields]
     end
 
-    def adjust_headers(headers)
-      adjusted_headers = @header_fields_converter.convert(headers, nil, @lineno)
+    def adjust_headers(headers, quoted_fields)
+      adjusted_headers = @header_fields_converter.convert(headers, nil, @lineno, quoted_fields)
       adjusted_headers.each {|h| h.freeze if h.is_a? String}
       adjusted_headers
     end
 
     def prepare_parser
       @may_quoted = may_quoted?
+      if @quote_character.nil?
+        @parse_method = :parse_no_quote
+      elsif @liberal_parsing or @strip
+        @parse_method = :parse_quotable_robust
+      else
+        @parse_method = :parse_quotable_loose
+      end
     end
 
     def may_quoted?
@@ -729,28 +834,28 @@ class CSV
       sample[0, 128].index(@quote_character)
     end
 
-    SCANNER_TEST = (ENV["CSV_PARSER_SCANNER_TEST"] == "yes")
-    if SCANNER_TEST
-      class UnoptimizedStringIO
-        def initialize(string)
-          @io = StringIO.new(string, "rb:#{string.encoding}")
-        end
+    class UnoptimizedStringIO # :nodoc:
+      def initialize(string)
+        @io = StringIO.new(string, "rb:#{string.encoding}")
+      end
 
-        def gets(*args)
-          @io.gets(*args)
-        end
+      def gets(*args)
+        @io.gets(*args)
+      end
 
-        def each_line(*args, &block)
-          @io.each_line(*args, &block)
-        end
+      def each_line(*args, &block)
+        @io.each_line(*args, &block)
+      end
 
-        def eof?
-          @io.eof?
-        end
+      def eof?
+        @io.eof?
       end
+    end
 
-      SCANNER_TEST_CHUNK_SIZE =
-        Integer((ENV["CSV_PARSER_SCANNER_TEST_CHUNK_SIZE"] || "1"), 10)
+    SCANNER_TEST = (ENV["CSV_PARSER_SCANNER_TEST"] == "yes")
+    if SCANNER_TEST
+      SCANNER_TEST_CHUNK_SIZE_NAME = "CSV_PARSER_SCANNER_TEST_CHUNK_SIZE"
+      SCANNER_TEST_CHUNK_SIZE_VALUE = ENV[SCANNER_TEST_CHUNK_SIZE_NAME]
       def build_scanner
         inputs = @samples.collect do |sample|
           UnoptimizedStringIO.new(sample)
@@ -760,20 +865,24 @@ class CSV
         else
           inputs << @input
         end
+        begin
+          chunk_size_value = ENV[SCANNER_TEST_CHUNK_SIZE_NAME]
+        rescue # Ractor::IsolationError
+          # Ractor on Ruby 3.0 can't read ENV value.
+          chunk_size_value = SCANNER_TEST_CHUNK_SIZE_VALUE
+        end
+        chunk_size = Integer((chunk_size_value || "1"), 10)
         InputsScanner.new(inputs,
                           @encoding,
                           @row_separator,
-                          chunk_size: SCANNER_TEST_CHUNK_SIZE)
+                          chunk_size: chunk_size)
       end
     else
       def build_scanner
         string = nil
         if @samples.empty? and @input.is_a?(StringIO)
           string = @input.read
-        elsif @samples.size == 1 and
-              @input != ARGF and
-              @input.respond_to?(:eof?) and
-              @input.eof?
+        elsif @samples.size == 1 and Parser.eof?(@input)
           string = @samples[0]
         end
         if string
@@ -782,8 +891,7 @@ class CSV
               !line.valid_encoding?
             end
             if index
-              message = "Invalid byte sequence in #{@encoding}"
-              raise MalformedCSVError.new(message, @lineno + index + 1)
+              raise InvalidEncodingError.new(@encoding, @lineno + index + 1)
             end
           end
           Scanner.new(string)
@@ -826,6 +934,14 @@ class CSV
       end
     end
 
+    def validate_field_size(field)
+      return unless @max_field_size
+      return if field.size <= @max_field_size
+      ignore_broken_line
+      message = "Field size exceeded: #{field.size} > #{@max_field_size}"
+      raise MalformedCSVError.new(message, @lineno)
+    end
+
     def parse_no_quote(&block)
       @scanner.each_line(@row_separator) do |line|
         next if @skip_lines and skip_line?(line)
@@ -838,6 +954,11 @@ class CSV
         else
           line = strip_value(line)
           row = line.split(@split_column_separator, -1)
+          if @max_field_size
+            row.each do |column|
+              validate_field_size(column)
+            end
+          end
           n_columns = row.size
           i = 0
           while i < n_columns
@@ -868,31 +989,37 @@ class CSV
             next
           end
           row = []
+          quoted_fields = FieldsConverter::NO_QUOTED_FIELDS
         elsif line.include?(@cr) or line.include?(@lf)
           @scanner.keep_back
-          @need_robust_parsing = true
+          @parse_method = :parse_quotable_robust
           return parse_quotable_robust(&block)
         else
           row = line.split(@split_column_separator, -1)
+          quoted_fields = []
           n_columns = row.size
           i = 0
           while i < n_columns
             column = row[i]
             if column.empty?
+              quoted_fields << false
               row[i] = nil
             else
               n_quotes = column.count(@quote_character)
               if n_quotes.zero?
+                quoted_fields << false
                 # no quote
               elsif n_quotes == 2 and
                    column.start_with?(@quote_character) and
                    column.end_with?(@quote_character)
+                quoted_fields << true
                 row[i] = column[1..-2]
               else
                 @scanner.keep_back
-                @need_robust_parsing = true
+                @parse_method = :parse_quotable_robust
                 return parse_quotable_robust(&block)
               end
+              validate_field_size(row[i])
             end
             i += 1
           end
@@ -900,13 +1027,14 @@ class CSV
         @scanner.keep_drop
         @scanner.keep_start
         @last_line = original_line
-        emit_row(row, &block)
+        emit_row(row, quoted_fields, &block)
       end
       @scanner.keep_drop
     end
 
     def parse_quotable_robust(&block)
       row = []
+      quoted_fields = []
       skip_needless_lines
       start_row
       while true
@@ -916,32 +1044,39 @@ class CSV
         value = parse_column_value
         if value
           @scanner.scan_all(@strip_value) if @strip_value
-          if @field_size_limit and value.size >= @field_size_limit
-            ignore_broken_line
-            raise MalformedCSVError.new("Field size exceeded", @lineno)
-          end
+          validate_field_size(value)
         end
         if parse_column_end
           row << value
+          quoted_fields << @quoted_column_value
         elsif parse_row_end
           if row.empty? and value.nil?
             emit_row([], &block) unless @skip_blanks
           else
             row << value
-            emit_row(row, &block)
+            quoted_fields << @quoted_column_value
+            emit_row(row, quoted_fields, &block)
             row = []
+            quoted_fields.clear
           end
           skip_needless_lines
           start_row
         elsif @scanner.eos?
           break if row.empty? and value.nil?
           row << value
-          emit_row(row, &block)
+          quoted_fields << @quoted_column_value
+          emit_row(row, quoted_fields, &block)
           break
         else
           if @quoted_column_value
+            if liberal_parsing? and (new_line = @scanner.check(@line_end))
+              message =
+                "Illegal end-of-line sequence outside of a quoted field " +
+                "<#{new_line.inspect}>"
+            else
+              message = "Any value after quoted field isn't allowed"
+            end
             ignore_broken_line
-            message = "Any value after quoted field isn't allowed"
             raise MalformedCSVError.new(message, @lineno)
           elsif @unquoted_column_value and
                 (new_line = @scanner.scan(@line_end))
@@ -1034,7 +1169,7 @@ class CSV
       if (n_quotes % 2).zero?
         quotes[0, (n_quotes - 2) / 2]
       else
-        value = quotes[0, (n_quotes - 1) / 2]
+        value = quotes[0, n_quotes / 2]
         while true
           quoted_value = @scanner.scan_all(@quoted_value)
           value << quoted_value if quoted_value
@@ -1058,11 +1193,9 @@ class CSV
           n_quotes = quotes.size
           if n_quotes == 1
             break
-          elsif (n_quotes % 2) == 1
-            value << quotes[0, (n_quotes - 1) / 2]
-            break
           else
             value << quotes[0, n_quotes / 2]
+            break if (n_quotes % 2) == 1
           end
         end
         value
@@ -1098,18 +1231,15 @@ class CSV
 
     def strip_value(value)
       return value unless @strip
-      return nil if value.nil?
+      return value if value.nil?
 
       case @strip
       when String
-        size = value.size
-        while value.start_with?(@strip)
-          size -= 1
-          value = value[1, size]
+        while value.delete_prefix!(@strip)
+          # do nothing
         end
-        while value.end_with?(@strip)
-          size -= 1
-          value = value[0, size]
+        while value.delete_suffix!(@strip)
+          # do nothing
         end
       else
         value.strip!
@@ -1132,22 +1262,22 @@ class CSV
       @scanner.keep_start
     end
 
-    def emit_row(row, &block)
+    def emit_row(row, quoted_fields=FieldsConverter::NO_QUOTED_FIELDS, &block)
       @lineno += 1
 
       raw_row = row
       if @use_headers
         if @headers.nil?
-          @headers = adjust_headers(row)
+          @headers = adjust_headers(row, quoted_fields)
           return unless @return_headers
           row = Row.new(@headers, row, true)
         else
           row = Row.new(@headers,
-                        @fields_converter.convert(raw_row, @headers, @lineno))
+                        @fields_converter.convert(raw_row, @headers, @lineno, quoted_fields))
         end
       else
         # convert fields, if needed...
-        row = @fields_converter.convert(raw_row, nil, @lineno)
+        row = @fields_converter.convert(raw_row, nil, @lineno, quoted_fields)
       end
 
       # inject unconverted fields and accessor, if requested...