csv 3.0.4 → 3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0c308f4cca10d6b48a2268e992fa545918c6cbe99837b83ec048b55eb76b702
-  data.tar.gz: 8661eb5e276abdeb5e939e3c2359d6ee991199550cfaa8e623496946fad17b8f
+  metadata.gz: 277fe614e5fc3f2f8ad100a70c6021aea0b8c9f989a46aad6618f9c3e81e5baf
+  data.tar.gz: cb8aafb272a93788371dcc6992f61ea26a7a19844f3aabacb0e16fc9a3e54058
 SHA512:
-  metadata.gz: 5875a4b446bd22fbdb0ab1a829576e219b64aa80a66f27e73ae487ed7fa6fb09429f8f3185ab8b08c07ee30a49204b98acd1d7a72a97c9ec89234d79dc597553
-  data.tar.gz: bda1a607b00312b0e4bc0e2d444670d5f9661251ad73cb2061baeb695cb264af0a06f3bf44fc4d00d6406a1b61f0d4bbdfb8e6fd036d276f51fd30fec4f179e3
+  metadata.gz: 063bf8125079ad1c9f42f26990ab9a6a7a07100fecc2db71a792e5b89aa757decc571e1c39a70c8c8a2eb50475749bbc103985a0fa66a1e35762f7568d7a7f85
+  data.tar.gz: 7daef31a3902cba8dbc139a1d828d15493def81c95011f4da6ee69369dbdfdd3fb97a20a7d9a9f0cd1a8129669e27403f6426644e3b2f48e7fef592c22967dc0

data/NEWS.md

@@ -1,5 +1,67 @@
 # News
 
+## 3.0.5 - 2019-03-24
+
+### Improvements
+
+  * Added `:liberal_parsing => {backslash_quote: true}` option.
+    [GitHub#74][Patch by 284km]
+
+  * Added `:write_converters` option.
+    [GitHub#73][Patch by Danillo Souza]
+
+  * Added `:write_nil_value` option.
+
+  * Added `:write_empty_value` option.
+
+  * Improved invalid byte line number detection.
+    [GitHub#78][Patch by Alyssa Ross]
+
+  * Added `quote_char: nil` optimization.
+    [GitHub#79][Patch by 284km]
+
+  * Improved error message.
+    [GitHub#81][Patch by Andrés Torres]
+
+  * Improved IO-like implementation for `StringIO` data.
+    [GitHub#80][Patch by Genadi Samokovarov]
+
+  * Added `:strip` option.
+    [GitHub#58]
+
+### Fixes
+
+  * Fixed a compatibility bug that `CSV#each` doesn't care `CSV#shift`.
+    [GitHub#76][Patch by Alyssa Ross]
+
+  * Fixed a compatibility bug that `CSV#eof?` doesn't care `CSV#each`
+    and `CSV#shift`.
+    [GitHub#77][Reported by Chi Leung]
+
+  * Fixed a compatibility bug that invalid line isn't ignored.
+    [GitHub#82][Reported by krororo]
+
+  * Fixed a bug that `:skip_lines` doesn't work with multibyte characters data.
+    [GitHub#83][Reported by ff2248]
+
+### Thanks
+
+  * Alyssa Ross
+
+  * 284km
+
+  * Chi Leung
+
+  * Danillo Souza
+
+  * Andrés Torres
+
+  * Genadi Samokovarov
+
+  * krororo
+
+  * ff2248
+
 ## 3.0.4 - 2019-01-25
 
 ### Improvements

data/lib/csv.rb

@@ -885,6 +885,10 @@ class CSV
   #                                       blank string field is replaced by
   #                                       the set object.
   # <b><tt>:quote_empty</tt></b>::        TODO
+  # <b><tt>:write_converters</tt></b>::   TODO
+  # <b><tt>:write_nil_value</tt></b>::    TODO
+  # <b><tt>:write_empty_value</tt></b>::  TODO
+  # <b><tt>:strip</tt></b>::              TODO
   #
   # See CSV::DEFAULT_OPTIONS for the default settings.
   #
@@ -911,7 +915,11 @@ class CSV
                  encoding: nil,
                  nil_value: nil,
                  empty_value: "",
-                 quote_empty: true)
+                 quote_empty: true,
+                 write_converters: nil,
+                 write_nil_value: nil,
+                 write_empty_value: "",
+                 strip: false)
     raise ArgumentError.new("Cannot parse nil as CSV") if data.nil?
 
     # create the IO object we will read from
@@ -922,8 +930,13 @@ class CSV
       nil_value: nil_value,
       empty_value: empty_value,
     }
+    @write_fields_converter_options = {
+      nil_value: write_nil_value,
+      empty_value: write_empty_value,
+    }
     @initial_converters = converters
     @initial_header_converters = header_converters
+    @initial_write_converters = write_converters
 
     @parser_options = {
       column_separator: col_sep,
@@ -939,6 +952,7 @@ class CSV
       encoding: @encoding,
       nil_value: nil_value,
       empty_value: empty_value,
+      strip: strip,
     }
     @parser = nil
 
@@ -998,7 +1012,7 @@ class CSV
   # as is.
   #
   def converters
-    fields_converter.map do |converter|
+    parser_fields_converter.map do |converter|
       name = Converters.rassoc(converter)
       name ? name.first : converter
     end
@@ -1098,12 +1112,58 @@ class CSV
   ### IO and StringIO Delegation ###
 
   extend Forwardable
-  def_delegators :@io, :binmode, :binmode?, :close, :close_read, :close_write,
-                       :closed?, :eof, :eof?, :external_encoding, :fcntl,
-                       :fileno, :flock, :flush, :fsync, :internal_encoding,
-                       :ioctl, :isatty, :path, :pid, :pos, :pos=, :reopen,
-                       :seek, :stat, :string, :sync, :sync=, :tell, :to_i,
-                       :to_io, :truncate, :tty?
+  def_delegators :@io, :binmode, :close, :close_read, :close_write,
+                       :closed?, :external_encoding, :fcntl,
+                       :fileno, :flush, :fsync, :internal_encoding,
+                       :isatty, :pid, :pos, :pos=, :reopen,
+                       :seek, :string, :sync, :sync=, :tell,
+                       :truncate, :tty?
+
+  def binmode?
+    if @io.respond_to?(:binmode?)
+      @io.binmode?
+    else
+      false
+    end
+  end
+
+  def flock(*args)
+    raise NotImplementedError unless @io.respond_to?(:flock)
+    @io.flock(*args)
+  end
+
+  def ioctl(*args)
+    raise NotImplementedError unless @io.respond_to?(:ioctl)
+    @io.ioctl(*args)
+  end
+
+  def path
+    @io.path if @io.respond_to?(:path)
+  end
+
+  def stat(*args)
+    raise NotImplementedError unless @io.respond_to?(:stat)
+    @io.stat(*args)
+  end
+
+  def to_i
+    raise NotImplementedError unless @io.respond_to?(:to_i)
+    @io.to_i
+  end
+
+  def to_io
+    @io.respond_to?(:to_io) ? @io.to_io : @io
+  end
+
+  def eof?
+    begin
+      parser_enumerator.peek
+      false
+    rescue StopIteration
+      true
+    end
+  end
+  alias_method :eof, :eof?
 
   # Rewinds the underlying IO object and resets CSV's lineno() counter.
   def rewind
@@ -1145,7 +1205,7 @@ class CSV
   # converted field or the field itself.
   #
   def convert(name = nil, &converter)
-    fields_converter.add_converter(name, &converter)
+    parser_fields_converter.add_converter(name, &converter)
   end
 
   #
@@ -1172,8 +1232,16 @@ class CSV
   #
   # The data source must be open for reading.
   #
-  def each(&block)
-    parser.parse(&block)
+  def each
+    return to_enum(__method__) unless block_given?
+    enumerator = parser_enumerator
+    begin
+      while true
+        yield enumerator.next
+      end
+    rescue StopIteration
+    end
+    self
   end
 
   #
@@ -1204,9 +1272,8 @@ class CSV
   # The data source must be open for reading.
   #
   def shift
-    @parser_enumerator ||= parser.parse
     begin
-      @parser_enumerator.next
+      parser_enumerator.next
     rescue StopIteration
       nil
     end
@@ -1299,7 +1366,7 @@ class CSV
     if headers
       header_fields_converter.convert(fields, nil, 0)
     else
-      fields_converter.convert(fields, @headers, lineno)
+      parser_fields_converter.convert(fields, @headers, lineno)
     end
   end
 
@@ -1316,20 +1383,16 @@ class CSV
     end
   end
 
-  def fields_converter
-    @fields_converter ||= build_fields_converter
+  def parser_fields_converter
+    @parser_fields_converter ||= build_parser_fields_converter
   end
 
-  def build_fields_converter
+  def build_parser_fields_converter
     specific_options = {
       builtin_converters: Converters,
     }
     options = @base_fields_converter_options.merge(specific_options)
-    fields_converter = FieldsConverter.new(options)
-    normalize_converters(@initial_converters).each do |name, converter|
-      fields_converter.add_converter(name, &converter)
-    end
-    fields_converter
+    build_fields_converter(@initial_converters, options)
   end
 
   def header_fields_converter
@@ -1342,8 +1405,21 @@ class CSV
       accept_nil: true,
     }
     options = @base_fields_converter_options.merge(specific_options)
+    build_fields_converter(@initial_header_converters, options)
+  end
+
+  def writer_fields_converter
+    @writer_fields_converter ||= build_writer_fields_converter
+  end
+
+  def build_writer_fields_converter
+    build_fields_converter(@initial_write_converters,
+                           @write_fields_converter_options)
+  end
+
+  def build_fields_converter(initial_converters, options)
     fields_converter = FieldsConverter.new(options)
-    normalize_converters(@initial_header_converters).each do |name, converter|
+    normalize_converters(initial_converters).each do |name, converter|
       fields_converter.add_converter(name, &converter)
     end
     fields_converter
@@ -1354,8 +1430,12 @@ class CSV
   end
 
   def parser_options
-    @parser_options.merge(fields_converter: fields_converter,
-                          header_fields_converter: header_fields_converter)
+    @parser_options.merge(header_fields_converter: header_fields_converter,
+                          fields_converter: parser_fields_converter)
+  end
+
+  def parser_enumerator
+    @parser_enumerator ||= parser.parse
   end
 
   def writer
@@ -1363,7 +1443,8 @@ class CSV
   end
 
   def writer_options
-    @writer_options.merge(header_fields_converter: header_fields_converter)
+    @writer_options.merge(header_fields_converter: header_fields_converter,
+                          fields_converter: writer_fields_converter)
   end
 end
 

data/lib/csv/parser.rb

@@ -49,6 +49,28 @@ class CSV
         read_chunk
       end
 
+      def each_line(row_separator)
+        buffer = nil
+        input = @scanner.rest
+        @scanner.terminate
+        while input
+          input.each_line(row_separator) do |line|
+            if buffer
+              buffer << line
+              line = buffer
+              buffer = nil
+            end
+            if line.end_with?(row_separator)
+              yield(line)
+            else
+              buffer = line
+            end
+          end
+          input = @inputs.shift
+        end
+        yield(buffer) if buffer
+      end
+
       def scan(pattern)
         value = @scanner.scan(pattern)
         return value if @last_scanner
@@ -94,7 +116,7 @@ class CSV
         start, buffer = @keeps.pop
         if buffer
           string = @scanner.string
-          keep = string[start, string.size - start]
+          keep = string.byteslice(start, string.bytesize - start)
           if keep and not keep.empty?
             @inputs.unshift(StringIO.new(keep))
             @last_scanner = false
@@ -121,7 +143,7 @@ class CSV
           keep = @keeps.last
           keep_start = keep[0]
           string = @scanner.string
-          keep_data = string[keep_start, @scanner.pos - keep_start]
+          keep_data = string.byteslice(keep_start, @scanner.pos - keep_start)
           if keep_data
             keep_buffer = keep[1]
             if keep_buffer
@@ -170,7 +192,6 @@ class CSV
       @input = input
       @options = options
       @samples = []
-      @parsed = false
 
       prepare
     end
@@ -230,9 +251,7 @@ class CSV
     def parse(&block)
       return to_enum(__method__) unless block_given?
 
-      return if @parsed
-
-      if @return_headers and @headers
+      if @return_headers and @headers and @raw_headers
         headers = Row.new(@headers, @raw_headers, true)
         if @unconverted_fields
           headers = add_unconverted_fields(headers, [])
@@ -240,58 +259,23 @@ class CSV
         yield headers
       end
 
-      row = []
       begin
-        @scanner = build_scanner
-        skip_needless_lines
-        start_row
-        while true
-          @quoted_column_value = false
-          @unquoted_column_value = false
-          value = parse_column_value
-          if value and @field_size_limit and value.size >= @field_size_limit
-            raise MalformedCSVError.new("Field size exceeded", @lineno + 1)
-          end
-          if parse_column_end
-            row << value
-          elsif parse_row_end
-            if row.empty? and value.nil?
-              emit_row([], &block) unless @skip_blanks
-            else
-              row << value
-              emit_row(row, &block)
-              row = []
-            end
-            skip_needless_lines
-            start_row
-          elsif @scanner.eos?
-            break if row.empty? and value.nil?
-            row << value
-            emit_row(row, &block)
-            break
-          else
-            if @quoted_column_value
-              message = "Do not allow except col_sep_split_separator " +
-                "after quoted fields"
-              raise MalformedCSVError.new(message, @lineno + 1)
-            elsif @unquoted_column_value and @scanner.scan(@cr_or_lf)
-              message = "Unquoted fields do not allow \\r or \\n"
-              raise MalformedCSVError.new(message, @lineno + 1)
-            elsif @scanner.rest.start_with?(@quote_character)
-              message = "Illegal quoting"
-              raise MalformedCSVError.new(message, @lineno + 1)
-            else
-              raise MalformedCSVError.new("TODO: Meaningful message",
-                                          @lineno + 1)
-            end
-          end
+        @scanner ||= build_scanner
+        if quote_character.nil?
+          parse_no_quote(&block)
+        else
+          parse_quotable(&block)
         end
       rescue InvalidEncoding
+        if @scanner
+          ignore_broken_line
+          lineno = @lineno
+        else
+          lineno = @lineno + 1
+        end
         message = "Invalid byte sequence in #{@encoding}"
-        raise MalformedCSVError.new(message, @lineno + 1)
+        raise MalformedCSVError.new(message, lineno)
       end
-
-      @parsed = true
     end
 
     def use_headers?
@@ -301,7 +285,13 @@ class CSV
     private
     def prepare
       prepare_variable
-      prepare_regexp
+      prepare_backslash
+      prepare_quote_character
+      prepare_skip_lines
+      prepare_strip
+      prepare_separators
+      prepare_quoted
+      prepare_unquoted
       prepare_line
       prepare_header
       prepare_parser
@@ -315,11 +305,14 @@ class CSV
         if liberal_parsing.is_a?(Hash)
           @double_quote_outside_quote =
             liberal_parsing[:double_quote_outside_quote]
+          @backslash_quote = liberal_parsing[:backslash_quote]
         else
           @double_quote_outside_quote = false
+          @backslash_quote = false
         end
       else
         @liberal_parsing = false
+        @backslash_quote = false
       end
       @unconverted_fields = @options[:unconverted_fields]
       @field_size_limit = @options[:field_size_limit]
@@ -328,20 +321,33 @@ class CSV
       @header_fields_converter = @options[:header_fields_converter]
     end
 
-    def prepare_regexp
-      @column_separator = @options[:column_separator].to_s.encode(@encoding)
-      @row_separator =
-        resolve_row_separator(@options[:row_separator]).encode(@encoding)
-      @quote_character = @options[:quote_character].to_s.encode(@encoding)
-      if @quote_character.length != 1
-        raise ArgumentError, ":quote_char has to be a single character String"
-      end
+    def prepare_backslash
+      @backslash_character = "\\".encode(@encoding)
 
-      escaped_column_separator = Regexp.escape(@column_separator)
-      escaped_first_column_separator = Regexp.escape(@column_separator[0])
-      escaped_row_separator = Regexp.escape(@row_separator)
-      escaped_quote_character = Regexp.escape(@quote_character)
+      @escaped_backslash_character = Regexp.escape(@backslash_character)
+      @escaped_backslash = Regexp.new(@escaped_backslash_character)
+    end
+
+    def prepare_quote_character
+      @quote_character = @options[:quote_character]
+      if @quote_character.nil?
+        @escaped_quote_character = nil
+        @escaped_quote = nil
+        @backslash_quote_character = nil
+      else
+        @quote_character = @quote_character.to_s.encode(@encoding)
+        if @quote_character.length != 1
+          message = ":quote_char has to be nil or a single character String"
+          raise ArgumentError, message
+        end
+        @escaped_quote_character = Regexp.escape(@quote_character)
+        @escaped_quote = Regexp.new(@escaped_quote_character)
+        @backslash_quote_character =
+          @backslash_character + @escaped_quote_character
+      end
+    end
 
+    def prepare_skip_lines
       skip_lines = @options[:skip_lines]
       case skip_lines
       when String
@@ -356,18 +362,56 @@ class CSV
         end
         @skip_lines = skip_lines
       end
+    end
 
-      @column_end = Regexp.new(escaped_column_separator)
+    def prepare_strip
+      @strip = @options[:strip]
+      @escaped_strip = nil
+      @strip_value = nil
+      if @strip.is_a?(String)
+        case @strip.length
+        when 0
+          raise ArgumentError, ":strip must not be an empty String"
+        when 1
+          # ok
+        else
+          raise ArgumentError, ":strip doesn't support 2 or more characters yet"
+        end
+        @strip = @strip.encode(@encoding)
+        @escaped_strip = Regexp.escape(@strip)
+        if @quote_character
+          @strip_value = Regexp.new(@escaped_strip +
+                                    "+".encode(@encoding))
+        end
+      elsif @strip
+        strip_values = " \t\r\n\f\v"
+        @escaped_strip = strip_values.encode(@encoding)
+        if @quote_character
+          @strip_value = Regexp.new("[#{strip_values}]+".encode(@encoding))
+        end
+      end
+    end
+
+    def prepare_separators
+      @column_separator = @options[:column_separator].to_s.encode(@encoding)
+      @row_separator =
+        resolve_row_separator(@options[:row_separator]).encode(@encoding)
+
+      @escaped_column_separator = Regexp.escape(@column_separator)
+      @escaped_first_column_separator = Regexp.escape(@column_separator[0])
+      @column_end = Regexp.new(@escaped_column_separator)
       if @column_separator.size > 1
         @column_ends = @column_separator.each_char.collect do |char|
           Regexp.new(Regexp.escape(char))
         end
-        @first_column_separators = Regexp.new(escaped_first_column_separator +
+        @first_column_separators = Regexp.new(@escaped_first_column_separator +
                                               "+".encode(@encoding))
       else
         @column_ends = nil
         @first_column_separators = nil
       end
+
+      escaped_row_separator = Regexp.escape(@row_separator)
       @row_end = Regexp.new(escaped_row_separator)
       if @row_separator.size > 1
         @row_ends = @row_separator.each_char.collect do |char|
@@ -376,25 +420,55 @@ class CSV
       else
         @row_ends = nil
       end
-      @quotes = Regexp.new(escaped_quote_character +
-                           "+".encode(@encoding))
-      @quoted_value = Regexp.new("[^".encode(@encoding) +
-                                 escaped_quote_character +
-                                 "]+".encode(@encoding))
-      if @liberal_parsing
-        @unquoted_value = Regexp.new("[^".encode(@encoding) +
-                                     escaped_first_column_separator +
-                                     "\r\n]+".encode(@encoding))
-      else
-        @unquoted_value = Regexp.new("[^".encode(@encoding) +
-                                     escaped_quote_character +
-                                     escaped_first_column_separator +
-                                     "\r\n]+".encode(@encoding))
-      end
+
       @cr_or_lf = Regexp.new("[\r\n]".encode(@encoding))
       @not_line_end = Regexp.new("[^\r\n]+".encode(@encoding))
     end
 
+    def prepare_quoted
+      if @quote_character
+        @quotes = Regexp.new(@escaped_quote_character +
+                             "+".encode(@encoding))
+        no_quoted_values = @escaped_quote_character.dup
+        if @backslash_quote
+          no_quoted_values << @escaped_backslash_character
+        end
+        @quoted_value = Regexp.new("[^".encode(@encoding) +
+                                   no_quoted_values +
+                                   "]+".encode(@encoding))
+      else
+        if @escaped_strip
+          @split_column_separator = Regexp.new(@escaped_strip +
+                                               "*".encode(@encoding) +
+                                               @escaped_column_separator +
+                                               @escaped_strip +
+                                               "*".encode(@encoding))
+        else
+          if @column_separator == " ".encode(@encoding)
+            @split_column_separator = @column_end
+          else
+            @split_column_separator = @column_separator
+          end
+        end
+      end
+    end
+
+    def prepare_unquoted
+      return if @quote_character.nil?
+
+      no_unquoted_values = "\r\n".encode(@encoding)
+      no_unquoted_values << @escaped_first_column_separator
+      unless @liberal_parsing
+        no_unquoted_values << @escaped_quote_character
+      end
+      if @escaped_strip
+        no_unquoted_values << @escaped_strip
+      end
+      @unquoted_value = Regexp.new("[^".encode(@encoding) +
+                                   no_unquoted_values +
+                                   "]+".encode(@encoding))
+    end
+
     def resolve_row_separator(separator)
       if separator == :auto
         cr = "\r".encode(@encoding)
@@ -514,6 +588,8 @@ class CSV
     end
 
     def may_quoted?
+      return false if @quote_character.nil?
+
       if @input.is_a?(StringIO)
         sample = @input.string
       else
@@ -534,6 +610,10 @@ class CSV
           @io.gets(*args)
         end
 
+        def each_line(*args, &block)
+          @io.each_line(*args, &block)
+        end
+
         def eof?
           @io.eof?
         end
@@ -548,7 +628,10 @@ class CSV
         else
           inputs << @input
         end
-        InputsScanner.new(inputs, @encoding, chunk_size: 1)
+        chunk_size = ENV["CSV_PARSER_SCANNER_TEST_CHUNK_SIZE"] || "1"
+        InputsScanner.new(inputs,
+                          @encoding,
+                          chunk_size: Integer(chunk_size, 10))
       end
     else
       def build_scanner
@@ -560,8 +643,13 @@ class CSV
         end
         if string
           unless string.valid_encoding?
-            message = "Invalid byte sequence in #{@encoding}"
-            raise MalformedCSVError.new(message, @lineno + 1)
+            index = string.lines(@row_separator).index do |line|
+              !line.valid_encoding?
+            end
+            if index
+              message = "Invalid byte sequence in #{@encoding}"
+              raise MalformedCSVError.new(message, @lineno + index + 1)
+            end
           end
           Scanner.new(string)
         else
@@ -582,6 +670,7 @@ class CSV
         line = @scanner.scan_all(@not_line_end) || "".encode(@encoding)
         line << @row_separator if parse_row_end
         if skip_line?(line)
+          @lineno += 1
           @scanner.keep_drop
         else
           @scanner.keep_back
@@ -601,6 +690,96 @@ class CSV
       end
     end
 
+    def parse_no_quote(&block)
+      if @scanner.respond_to?(:string)
+        scanner = @scanner.string
+      else
+        scanner = @scanner
+      end
+      scanner.each_line(@row_separator) do |value|
+        next if @skip_lines and skip_line?(value)
+        value.chomp!
+
+        if value.empty?
+          next if @skip_blanks
+          row = []
+        else
+          value = strip_value(value)
+          row = value.split(@split_column_separator, -1)
+          n_columns = row.size
+          i = 0
+          while i < n_columns
+            row[i] = nil if row[i].empty?
+            i += 1
+          end
+        end
+        @last_line = value
+        emit_row(row, &block)
+      end
+    end
+
+    def parse_quotable(&block)
+      row = []
+      skip_needless_lines
+      start_row
+      while true
+        @quoted_column_value = false
+        @unquoted_column_value = false
+        @scanner.scan_all(@strip_value) if @strip_value
+        value = parse_column_value
+        if value
+          @scanner.scan_all(@strip_value) if @strip_value
+          if @field_size_limit and value.size >= @field_size_limit
+            ignore_broken_line
+            raise MalformedCSVError.new("Field size exceeded", @lineno)
+          end
+        end
+        if parse_column_end
+          row << value
+        elsif parse_row_end
+          if row.empty? and value.nil?
+            emit_row([], &block) unless @skip_blanks
+          else
+            row << value
+            emit_row(row, &block)
+            row = []
+          end
+          skip_needless_lines
+          start_row
+        elsif @scanner.eos?
+          break if row.empty? and value.nil?
+          row << value
+          emit_row(row, &block)
+          break
+        else
+          if @quoted_column_value
+            ignore_broken_line
+            message = "Any value after quoted field isn't allowed"
+            raise MalformedCSVError.new(message, @lineno)
+          elsif @unquoted_column_value and
+                (new_line = @scanner.scan(@cr_or_lf))
+            ignore_broken_line
+            message = "Unquoted fields do not allow new line " +
+                      "<#{new_line.inspect}>"
+            raise MalformedCSVError.new(message, @lineno)
+          elsif @scanner.rest.start_with?(@quote_character)
+            ignore_broken_line
+            message = "Illegal quoting"
+            raise MalformedCSVError.new(message, @lineno)
+          elsif (new_line = @scanner.scan(@cr_or_lf))
+            ignore_broken_line
+            message = "New line must be <#{@row_separator.inspect}> " +
+                      "not <#{new_line.inspect}>"
+            raise MalformedCSVError.new(message, @lineno)
+          else
+            ignore_broken_line
+            raise MalformedCSVError.new("TODO: Meaningful message",
+                                        @lineno)
+          end
+        end
+      end
+    end
+
     def parse_column_value
       if @liberal_parsing
         quoted_value = parse_quoted_column_value
@@ -651,6 +830,7 @@ class CSV
           value << sub_value
         end
       end
+      value.gsub!(@backslash_quote_character, @quote_character) if @backslash_quote
       value
     end
 
@@ -667,10 +847,22 @@ class CSV
         while true
           quoted_value = @scanner.scan_all(@quoted_value)
           value << quoted_value if quoted_value
+          if @backslash_quote
+            if @scanner.scan(@escaped_backslash)
+              if @scanner.scan(@escaped_quote)
+                value << @quote_character
+              else
+                value << @backslash_character
+              end
+              next
+            end
+          end
+
           quotes = @scanner.scan_all(@quotes)
           unless quotes
+            ignore_broken_line
             message = "Unclosed quoted field"
-            raise MalformedCSVError.new(message, @lineno + 1)
+            raise MalformedCSVError.new(message, @lineno)
           end
           n_quotes = quotes.size
           if n_quotes == 1
@@ -713,6 +905,33 @@ class CSV
       end
     end
 
+    def strip_value(value)
+      return value unless @strip
+      return nil if value.nil?
+
+      case @strip
+      when String
+        size = value.size
+        while value.start_with?(@strip)
+          size -= 1
+          value = value[1, size]
+        end
+        while value.end_with?(@strip)
+          size -= 1
+          value = value[0, size]
+        end
+      else
+        value.strip!
+      end
+      value
+    end
+
+    def ignore_broken_line
+      @scanner.scan_all(@not_line_end)
+      @scanner.scan_all(@cr_or_lf)
+      @lineno += 1
+    end
+
     def start_row
       if @last_line
         @last_line = nil

data/lib/csv/version.rb

@@ -2,5 +2,5 @@
 
 class CSV
   # The version of the installed library.
-  VERSION = "3.0.4"
+  VERSION = "3.0.5"
 end

data/lib/csv/writer.rb

@@ -18,6 +18,7 @@ class CSV
       if @options[:write_headers] and @headers
         self << @headers
       end
+      @fields_converter = @options[:fields_converter]
     end
 
     def <<(row)
@@ -31,6 +32,8 @@ class CSV
       @headers ||= row if @use_headers
       @lineno += 1
 
+      row = @fields_converter.convert(row, nil, lineno) if @fields_converter
+
       converted_row = row.collect do |field|
         quote(field)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.0.5
 platform: ruby
 authors:
 - James Edward Gray II
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-25 00:00:00.000000000 Z
+date: 2019-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler