dsv 0.12.6 → 0.12.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7322674ac0843543bf2f813312290dd2c4d460c7c3d0e8e8d604ccfcb680a196
-  data.tar.gz: 866e0a95d7854265444854111c0f1b61ed5e0faf9eb1f55ba20061f60fe52447
+  metadata.gz: 190e66c0b25b64c18dac0b955ae59fe64591d14d369cafa950d31b1aa56a2b30
+  data.tar.gz: 6a7ea034b8f8da40716efb89a555951307ee312c38142b84c71f2712c57e0622
 SHA512:
-  metadata.gz: 219226e29fa873569b197d731813421aa478d18719bb25ede6501b84ab42efd574112a1131c7e3526933fe5099c8c1c3fffb8345d1fcdb26392dd8ef9fe445c9
-  data.tar.gz: e57b19a5fc7ce1855f4fd40cca7a496bd946464c43f84e00e13f8a3b031d2e8b7c104e6995e749d2955a968c941896d80951f8190f557ff17ea92734c65aba66
+  metadata.gz: e6f6cb3c8f04131f26bf26be33f7dc1e000a4e39a58537f3c327c250837d880477b07e478be8688a57b124dfd85f24f92d569109de15e95ec61b95631652e493
+  data.tar.gz: f0787b59e38dbac938fcd7474b815e24d2660c33438289a9720ca239b439a3d7435ce650db9dd5279b2ae2266ccfe763d1dfe8005d92cdb2dd1a8c1369f16cb8

data/CHANGELOG

@@ -1,7 +1,25 @@
 # dsv/CHANGELOG
 
+## 20261028
+
+0.12.8: a stray quote no longer takes in the next line.
+
+1. /complete_quoted_row()/row_fields()/: takes the next line into a row while a quoted field in it is open, rather than while its quotes are odd in number. A stray quote had made the count odd, and one beside an open field even.
+2. ~ reassemble_quoted_fields(), split_row(): nil where a quoted field is still open at the end of the row.
+3. ~ split_double_quoted(): takes a row as wholly quoted only where it ends in a closing quote.
+4. + parse_fields(), from parse_row(), so that the read loop splits each row once.
+5. ~ test/dsv_test.rb: - the skip on a stray quote; + a quoted field across lines after a stray quote.
+6. ~ lib/DSV/VERSION.rb: /0.12.7/0.12.8/.
+
+
 ## 20261027
 
+0.12.7: ~ reassemble_quoted_fields(): an escaped quote before a separator no longer closes a quoted field.
+
+1. ~ reassemble_quoted_fields(): a piece closes a field where it ends in an odd run of quotes; + trailing_quotes().
+2. ~ test/dsv_test.rb: - the skip on a doubled quote before a separator.
+3. ~ lib/DSV/VERSION.rb: /0.12.6/0.12.7/.
+
 0.12.6: + quoting examples.
 
 1. ~ test/dsv_test.rb: + a doubled quote before a separator, skipped; + a quote not opening a field read as data; + a stray quote not taking in the next line, skipped; + an unclosed quoted field raising UnterminatedQuote, skipped.

data/lib/DSV/VERSION.rb

@@ -2,5 +2,5 @@
 # DSV::VERSION
 
 class DSV
-  VERSION = '0.12.6'
+  VERSION = '0.12.8'
 end

data/lib/dsv.rb

@@ -180,7 +180,7 @@ class DSV
     else
       read_header
       @rows = []
-      @source.each(@row_separator){|raw_row| @rows << parse_row(complete_quoted_row(raw_row), *selected_columns)}
+      @source.each(@row_separator){|raw_row| @rows << parse_fields(row_fields(raw_row), *selected_columns)}
       @rows
     end
   end
@@ -211,7 +211,7 @@ class DSV
     @columns = (
       if header_row? && ['r', 'r+', 'a+'].include?(@mode) && (first_row = first_row?)
         columns, i = {}, -1
-        split_row(first_row).each do |column_name|
+        (split_row(first_row) || []).each do |column_name|
           columns[column_name] = columns.key?(column_name) ? [*columns[column_name], i += 1] : (i += 1)
         end
         columns
@@ -263,7 +263,7 @@ class DSV
     value.nil? || value.empty?
   end
 
-  # One row into its fields, by the quote mode: :none splits; :double splits a row as it promises, wholly quoted or wholly unquoted, and one that is neither as unspecified would; unspecified splits and reassembles a quoted field that held the separator, a doubled quote inside it reading as one. Direct string operations throughout; the scanner-based parser is on the branch scanner-parser.
+  # One row into its fields, by the quote mode: :none splits; :double splits a row as it promises, wholly quoted or wholly unquoted, and one that is neither as unspecified would; unspecified splits and reassembles a quoted field that held the separator, a doubled quote inside it reading as one. nil where a quoted field in the row is still open at its end. Direct string operations throughout; the scanner-based parser is on the branch scanner-parser.
   def split_row(raw_row)
     row = raw_row.chomp(@row_separator)
     case @quote
@@ -302,10 +302,10 @@ class DSV
     end
   end
 
-  # The :double contract: a wholly quoted row split on quote-separator-quote, a wholly unquoted one plainly; nil for a row that is neither.
+  # The :double contract: a wholly quoted row split on quote-separator-quote, a wholly unquoted one plainly; nil for a row that is neither, or whose last field does not close.
   def split_double_quoted(row)
     if row.start_with?('"')
-      unescaped_fields(split_on(row.delete_prefix('"').delete_suffix('"'), quoted_separator))
+      unescaped_fields(split_on(row.delete_prefix('"').delete_suffix('"'), quoted_separator)) if trailing_quotes(row, 0).odd?
     else
       split_on(row, @column_separator) unless row.include?('"')
     end
@@ -332,25 +332,32 @@ class DSV
     )
   end
 
-  # Pieces of a row split on the separator, the pieces of a quoted field that held the separator joined back with it, or with the text it matched where the separator is a Regexp; a piece that is a lone quote opens or closes a field.
+  # Pieces of a row split on the separator, the pieces of a quoted field that held the separator joined back with it, or with the text it matched where the separator is a Regexp; a piece closes a field where it ends in an odd run of quotes, a doubled quote being escaped, and a piece that is a lone quote opens or closes one; nil where a field is still open after the last piece.
   def reassemble_quoted_fields(pieces, separators = nil)
     fields = []
     buffer = nil
     pieces.each_with_index do |piece, i|
-      if buffer && !piece.end_with?('"')
+      if buffer && trailing_quotes(piece, -1).even?
         buffer << separator_before(i, separators) << piece
       elsif buffer
         fields << (buffer << separator_before(i, separators) << piece.delete_suffix('"')).gsub('""', '"')
         buffer = nil
-      elsif piece.start_with?('"') && (piece.length == 1 || !piece.end_with?('"'))
+      elsif piece.start_with?('"') && trailing_quotes(piece, 0).even?
         buffer = +piece.delete_prefix('"')
-      elsif piece.start_with?('"') && piece.end_with?('"')
+      elsif piece.start_with?('"')
         fields << piece.delete_prefix('"').delete_suffix('"').gsub('""', '"')
       else
         fields << piece
       end
     end
-    fields
+    fields unless buffer
+  end
+
+  # How many quotes a piece ends with, counting back to but not including the byte given, so that a field's opening quote is not taken for part of its closing. Read by byte, a quote being 34 in any ASCII-compatible encoding, so that nothing is allocated per character.
+  def trailing_quotes(piece, floor)
+    i = piece.bytesize - 1
+    i -= 1 while i > floor && piece.getbyte(i) == 34
+    piece.bytesize - 1 - i
   end
 
   # What stood between a piece and the one before it: the separator, or the text a Regexp separator matched there.
@@ -358,30 +365,34 @@ class DSV
     separators ? separators[i - 1] : @column_separator
   end
 
-  # A quoted field may hold the row separator: while the quotes in a row are unbalanced the next line belongs to it, in every quote mode which reads quotes as structure, :none and :unquoted reading them as data.
-  def complete_quoted_row(raw_row)
-    raw_row << @source.gets(@row_separator).to_s while @quote != :none && @quote != :unquoted && raw_row.count('"').odd? && !@source.eof?
-    raw_row
+  # A row's fields, the next line taken into the row while a quoted field in it is open, a quoted field being able to hold the row separator; a field still open at the end of the source is dropped, with its row.
+  def row_fields(raw_row)
+    raw_row << @source.gets(@row_separator) until (fields = split_row(raw_row)) || @source.eof?
+    fields || []
   end
 
   def parse_row(raw_row, *selected_columns)
+    parse_fields(split_row(raw_row) || [], *selected_columns)
+  end
+
+  def parse_fields(fields, *selected_columns)
     parsed_row = {}
     i = -1
     if selected_columns.empty?
       if blank?(@columns)
-        split_row(raw_row).each{|column_value| parsed_row[i += 1] = column_value}
+        fields.each{|column_value| parsed_row[i += 1] = column_value}
       elsif repeated_names.empty?
-        split_row(raw_row).each{|column_value| parsed_row[attributes[i += 1]] = column_value}
+        fields.each{|column_value| parsed_row[attributes[i += 1]] = column_value}
       else
-        split_row(raw_row).each{|column_value| store_field(parsed_row, attributes[i += 1], column_value)}
+        fields.each{|column_value| store_field(parsed_row, attributes[i += 1], column_value)}
       end
     else
       selected_columns.flatten!
       case selected_columns[0]
       when Integer
-        split_row(raw_row).each{|column_value| parsed_row[i] = column_value unless !selected_columns.include?(i += 1)}
+        fields.each{|column_value| parsed_row[i] = column_value unless !selected_columns.include?(i += 1)}
       else
-        split_row(raw_row).each{|column_value| store_field(parsed_row, attributes[i], column_value) unless !selected_columns.include?(attributes[i += 1])}
+        fields.each{|column_value| store_field(parsed_row, attributes[i], column_value) unless !selected_columns.include?(attributes[i += 1])}
       end
     end
     if @as_array

data/test/dsv_test.rb

@@ -243,7 +243,6 @@ describe DSV do
     end
 
     it "reads a doubled quote just before a separator inside a quoted field" do
-      skip "Observation: a piece of a quoted field ending in a quote is taken as the field's end, an escaped quote ending in one as a closing quote does"
       _(DSV.parse_line('"He said ""no"", then left",x')).must_equal ['He said "no", then left', 'x']
       _(DSV.parse_line('"a"",""b",c')).must_equal ['a","b', 'c']
       _(DSV.read("\"a\"\",\"\"b\",\"c\"\n", quote: :double)).must_equal [{0 => 'a","b', 1 => 'c'}]
@@ -563,13 +562,17 @@ describe DSV do
     end
 
     it "reads a stray quote in an unquoted field as data, without taking in the next line" do
-      skip "Observation: the quotes in a row are counted, so a stray quote leaves them unbalanced and the next line is taken into the row; unspecified since 0.11.4, :double since 0.12.5"
       _(DSV.read("12\" pizza,9.99\nnext,row\n", as_array: true)).must_equal [['12" pizza', '9.99'], ['next', 'row']]
       _(DSV.read("12\" pizza,9.99\nnext,row\n", as_array: true, quote: :double)).must_equal [['12" pizza', '9.99'], ['next', 'row']]
     end
 
+    it "reads a quoted field across lines after a stray quote in the same row" do
+      _(DSV.read("a\"b,\"c\nd\"\n", as_array: true)).must_equal [['a"b', "c\nd"]]
+      _(DSV.read("a\"b,\"c\nd\"\n", as_array: true, quote: :double)).must_equal [['a"b', "c\nd"]]
+    end
+
     it "raises UnterminatedQuote on a quoted field which does not close" do
-      skip "Observation: a quoted field which does not close is dropped silently, with the rest of its row"
+      skip "Observation: a quoted field which does not close is dropped silently, with its row"
       _{DSV.read("\"abc,d\n")}.must_raise DSV::UnterminatedQuote
       _{DSV.read("\"ab\"c,d\n")}.must_raise DSV::UnterminatedQuote
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dsv
 version: !ruby/object:Gem::Version
-  version: 0.12.6
+  version: 0.12.8
 platform: ruby
 authors:
 - thoran