csv 3.0.6 → 3.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0dfdbc61b4f75c06e3ea3ffd26980443aebe03d9a2fbdb8dc46a48d986d0447
-  data.tar.gz: 06cae607c0c77df4aa9f206d8d6d776a4c9207e359506558b9b07885c6f18c24
+  metadata.gz: 4a0aa4d6e8819de8616255194ac7fb7acf8669fb7a6f7580bd07e23e6ee798b0
+  data.tar.gz: 79044828a9b7232a6b671767541a6a860da3229e86e1101304b4a210490b867c
 SHA512:
-  metadata.gz: 0ec7e66bf0feb2f6dc9cea0cf72748e74e5a9ac817cd98254f17e2e42dca39265b067a8415c4edefd1879c878f4e1e96f68caa3289a97b26eafcd2e835d0160e
-  data.tar.gz: cbe775cdbf43c7eb86af541d2dbafc8359aee4e108cb62f34fa73adbb401ac96c2c3b6839f972f0d3285068d8fa7c96120ef4de1b10f14acd9457fe9384cc822
+  metadata.gz: f6626726217b3e967847f93a0a751b640f60d2aaca8b09e34a103b3b377c4623930e3264170fdd44b901a7e421c4953d0d05238e4c274de3e1eec9b97efad4b4
+  data.tar.gz: b722012b844524e1fd94ba7403dfa21aff33087789440f578b6f21f8b235e8f8cf8ef83dd1ebf98d37b64e958255120bd674c4b86715a2a37912bb0e9bf35fe3

data/NEWS.md

@@ -1,5 +1,18 @@
 # News
 
+## 3.0.7 - 2019-04-08
+
+### Improvements
+
+  * Improve parse performance 1.5x by introducing loose parser.
+
+### Fixes
+
+  * Fix performance regression in 3.0.5.
+
+  * Fix a bug that `CSV#line` returns wrong value when you
+    use `quote_char: nil`.
+
 ## 3.0.6 - 2019-03-30
 
 ### Improvements

data/lib/csv.rb

@@ -1232,16 +1232,8 @@ class CSV
   #
   # The data source must be open for reading.
   #
-  def each
-    return to_enum(__method__) unless block_given?
-    enumerator = parser_enumerator
-    begin
-      while true
-        yield enumerator.next
-      end
-    rescue StopIteration
-    end
-    self
+  def each(&block)
+    parser_enumerator.each(&block)
   end
 
   #

data/lib/csv/delete_suffix.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# This provides String#delete_suffix? for Ruby 2.4.
+unless String.method_defined?(:delete_suffix)
+  class CSV
+    module DeleteSuffix
+      refine String do
+        def delete_suffix(suffix)
+          if end_with?(suffix)
+            self[0..(-(suffix.size + 1))]
+          else
+            self
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csv/parser.rb

@@ -2,10 +2,12 @@
 
 require "strscan"
 
+require_relative "delete_suffix"
 require_relative "match_p"
 require_relative "row"
 require_relative "table"
 
+using CSV::DeleteSuffix if CSV.const_defined?(:DeleteSuffix)
 using CSV::MatchP if CSV.const_defined?(:MatchP)
 
 class CSV
@@ -21,6 +23,15 @@ class CSV
         @keeps = []
       end
 
+      def each_line(row_separator)
+        position = pos
+        rest.each_line(row_separator) do |line|
+          position += line.bytesize
+          self.pos = position
+          yield(line)
+        end
+      end
+
       def keep_start
         @keeps.push(pos)
       end
@@ -52,21 +63,43 @@ class CSV
       def each_line(row_separator)
         buffer = nil
         input = @scanner.rest
-        @scanner.terminate
-        while input
+        position = @scanner.pos
+        offset = 0
+        n_row_separator_chars = row_separator.size
+        while true
           input.each_line(row_separator) do |line|
+            @scanner.pos += line.bytesize
             if buffer
-              buffer << line
-              line = buffer
-              buffer = nil
+              if n_row_separator_chars == 2 and
+                buffer.end_with?(row_separator[0]) and
+                line.start_with?(row_separator[1])
+                buffer << line[0]
+                line = line[1..-1]
+                position += buffer.bytesize + offset
+                @scanner.pos = position
+                offset = 0
+                yield(buffer)
+                buffer = nil
+                next if line.empty?
+              else
+                buffer << line
+                line = buffer
+                buffer = nil
+              end
             end
             if line.end_with?(row_separator)
+              position += line.bytesize + offset
+              @scanner.pos = position
+              offset = 0
               yield(line)
             else
               buffer = line
             end
           end
-          input = @inputs.shift
+          break unless read_chunk
+          input = @scanner.rest
+          position = @scanner.pos
+          offset = -buffer.bytesize if buffer
         end
         yield(buffer) if buffer
       end
@@ -125,6 +158,7 @@ class CSV
         else
           @scanner.pos = start
         end
+        read_chunk if @scanner.eos?
       end
 
       def keep_drop
@@ -263,8 +297,10 @@ class CSV
         @scanner ||= build_scanner
         if quote_character.nil?
           parse_no_quote(&block)
+        elsif @need_robust_parsing
+          parse_quotable_robust(&block)
         else
-          parse_quotable(&block)
+          parse_quotable_loose(&block)
         end
       rescue InvalidEncoding
         if @scanner
@@ -285,8 +321,8 @@ class CSV
     private
     def prepare
       prepare_variable
-      prepare_backslash
       prepare_quote_character
+      prepare_backslash
       prepare_skip_lines
       prepare_strip
       prepare_separators
@@ -298,6 +334,7 @@ class CSV
     end
 
     def prepare_variable
+      @need_robust_parsing = false
       @encoding = @options[:encoding]
       liberal_parsing = @options[:liberal_parsing]
       if liberal_parsing
@@ -310,6 +347,7 @@ class CSV
           @double_quote_outside_quote = false
           @backslash_quote = false
         end
+        @need_robust_parsing = true
       else
         @liberal_parsing = false
         @backslash_quote = false
@@ -321,27 +359,33 @@ class CSV
       @header_fields_converter = @options[:header_fields_converter]
     end
 
-    def prepare_backslash
-      @backslash_character = "\\".encode(@encoding)
-
-      @escaped_backslash_character = Regexp.escape(@backslash_character)
-      @escaped_backslash = Regexp.new(@escaped_backslash_character)
-    end
-
     def prepare_quote_character
       @quote_character = @options[:quote_character]
       if @quote_character.nil?
         @escaped_quote_character = nil
         @escaped_quote = nil
-        @backslash_quote_character = nil
       else
         @quote_character = @quote_character.to_s.encode(@encoding)
         if @quote_character.length != 1
           message = ":quote_char has to be nil or a single character String"
           raise ArgumentError, message
         end
+        @double_quote_character = @quote_character * 2
         @escaped_quote_character = Regexp.escape(@quote_character)
         @escaped_quote = Regexp.new(@escaped_quote_character)
+      end
+    end
+
+    def prepare_backslash
+      return unless @backslash_quote
+
+      @backslash_character = "\\".encode(@encoding)
+
+      @escaped_backslash_character = Regexp.escape(@backslash_character)
+      @escaped_backslash = Regexp.new(@escaped_backslash_character)
+      if @quote_character.nil?
+        @backslash_quote_character = nil
+      else
         @backslash_quote_character =
           @backslash_character + @escaped_quote_character
       end
@@ -389,9 +433,18 @@ class CSV
         if @quote_character
           @strip_value = Regexp.new("[#{strip_values}]+".encode(@encoding))
         end
+        @need_robust_parsing = true
       end
     end
 
+    begin
+      StringScanner.new("x").scan("x")
+    rescue TypeError
+      @@string_scanner_scan_accept_string = false
+    else
+      @@string_scanner_scan_accept_string = true
+    end
+
     def prepare_separators
       @column_separator = @options[:column_separator].to_s.encode(@encoding)
       @row_separator =
@@ -399,14 +452,19 @@ class CSV
 
       @escaped_column_separator = Regexp.escape(@column_separator)
       @escaped_first_column_separator = Regexp.escape(@column_separator[0])
-      @column_end = Regexp.new(@escaped_column_separator)
       if @column_separator.size > 1
+        @column_end = Regexp.new(@escaped_column_separator)
         @column_ends = @column_separator.each_char.collect do |char|
           Regexp.new(Regexp.escape(char))
         end
         @first_column_separators = Regexp.new(@escaped_first_column_separator +
                                               "+".encode(@encoding))
       else
+        if @@string_scanner_scan_accept_string
+          @column_end = @column_separator
+        else
+          @column_end = Regexp.new(@escaped_column_separator)
+        end
         @column_ends = nil
         @first_column_separators = nil
       end
@@ -421,6 +479,8 @@ class CSV
         @row_ends = nil
       end
 
+      @cr = "\r".encode(@encoding)
+      @lf = "\n".encode(@encoding)
       @cr_or_lf = Regexp.new("[\r\n]".encode(@encoding))
       @not_line_end = Regexp.new("[^\r\n]+".encode(@encoding))
     end
@@ -436,19 +496,18 @@ class CSV
         @quoted_value = Regexp.new("[^".encode(@encoding) +
                                    no_quoted_values +
                                    "]+".encode(@encoding))
+      end
+      if @escaped_strip
+        @split_column_separator = Regexp.new(@escaped_strip +
+                                             "*".encode(@encoding) +
+                                             @escaped_column_separator +
+                                             @escaped_strip +
+                                             "*".encode(@encoding))
       else
-        if @escaped_strip
-          @split_column_separator = Regexp.new(@escaped_strip +
-                                               "*".encode(@encoding) +
-                                               @escaped_column_separator +
-                                               @escaped_strip +
-                                               "*".encode(@encoding))
+        if @column_separator == " ".encode(@encoding)
+          @split_column_separator = Regexp.new(@escaped_column_separator)
         else
-          if @column_separator == " ".encode(@encoding)
-            @split_column_separator = @column_end
-          else
-            @split_column_separator = @column_separator
-          end
+          @split_column_separator = @column_separator
         end
       end
     end
@@ -691,14 +750,10 @@ class CSV
     end
 
     def parse_no_quote(&block)
-      if @scanner.respond_to?(:string)
-        scanner = @scanner.string
-      else
-        scanner = @scanner
-      end
-      scanner.each_line(@row_separator) do |line|
+      @scanner.each_line(@row_separator) do |line|
         next if @skip_lines and skip_line?(line)
-        line.chomp!
+        original_line = line
+        line = line.delete_suffix(@row_separator)
 
         if line.empty?
           next if @skip_blanks
@@ -713,12 +768,67 @@ class CSV
             i += 1
           end
         end
-        @last_line = line
+        @last_line = original_line
+        emit_row(row, &block)
+      end
+    end
+
+    def parse_quotable_loose(&block)
+      @scanner.keep_start
+      @scanner.each_line(@row_separator) do |line|
+        if @skip_lines and skip_line?(line)
+          @scanner.keep_drop
+          @scanner.keep_start
+          next
+        end
+        original_line = line
+        line = line.delete_suffix(@row_separator)
+
+        if line.empty?
+          if @skip_blanks
+            @scanner.keep_drop
+            @scanner.keep_start
+            next
+          end
+          row = []
+        elsif line.include?(@cr) or line.include?(@lf)
+          @scanner.keep_back
+          @need_robust_parsing = true
+          return parse_quotable_robust(&block)
+        else
+          row = line.split(@split_column_separator, -1)
+          n_columns = row.size
+          i = 0
+          while i < n_columns
+            column = row[i]
+            if column.empty?
+              row[i] = nil
+            else
+              n_quotes = column.count(@quote_character)
+              if n_quotes.zero?
+                # no quote
+              elsif n_quotes == 2 and
+                   column.start_with?(@quote_character) and
+                   column.end_with?(@quote_character)
+                row[i] = column[1..-2]
+              else
+                @scanner.keep_back
+                @need_robust_parsing = true
+                return parse_quotable_robust(&block)
+              end
+            end
+            i += 1
+          end
+        end
+        @scanner.keep_drop
+        @scanner.keep_start
+        @last_line = original_line
         emit_row(row, &block)
       end
+      @scanner.keep_drop
     end
 
-    def parse_quotable(&block)
+    def parse_quotable_robust(&block)
       row = []
       skip_needless_lines
       start_row

data/lib/csv/version.rb

@@ -2,5 +2,5 @@
 
 class CSV
   # The version of the installed library.
-  VERSION = "3.0.6"
+  VERSION = "3.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv
 version: !ruby/object:Gem::Version
-  version: 3.0.6
+  version: 3.0.7
 platform: ruby
 authors:
 - James Edward Gray II
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-29 00:00:00.000000000 Z
+date: 2019-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -83,6 +83,7 @@ files:
 - lib/csv.rb
 - lib/csv/core_ext/array.rb
 - lib/csv/core_ext/string.rb
+- lib/csv/delete_suffix.rb
 - lib/csv/fields_converter.rb
 - lib/csv/match_p.rb
 - lib/csv/parser.rb