redshift-connector-data_file 7.2.0 → 7.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6e59a40168cc873c02377298bc0ed3ff93dfdf7
-  data.tar.gz: 4eb2c6f3c78ef2b742dfea6e5ca795344a686f58
+  metadata.gz: 701f14f9327e2b44caf41f66a410518da508a2b9
+  data.tar.gz: 21afe712a15a3ebd31ef0c86fb03150c124608dc
 SHA512:
-  metadata.gz: 3beaaacb6c6ea5b743bed86d6f0e86ba95f424f6ffa72b60a725482da151f47deda1b489bbe8b7d54bb8254c632b33b7d9e92e70361efc1b48de52bdf7b0ef0d
-  data.tar.gz: 3189ab027a2e2a5626471475cf18e942c6e4b84223e7f67b07c1572e7a258a20ff851a84fb1b6dcf1406a757e0687650f56abd48fa775aeecb094a5a8c90aca8
+  metadata.gz: 3ed483e215690fc976b60969b3ab6ada05a68941ef7e17899fdfde6fc3d28f442b05626f1257c8f351a222078daffcedee2c74030baf3aabbe6dd5e77a74de96
+  data.tar.gz: 8dd76812b9a5c79f4c3cd4da6994442f2703546d4750e78770f9faff747dcab9a313843de9f8061b3f443944c9f8233f6ee316f19f26ef7990cabdb0a89785cf

data/lib/redshift_connector/reader/redshift_csv.rb

@@ -15,41 +15,105 @@ module RedshiftConnector
     # f :: IO
     def initialize(f)
       @f = f
+      @s = ScanBuffer.new(@f)
     end
 
-    def each
-      # We can use simple #each_line to read single row
-      # because line terminators are always escaped by UNLOAD.
-      @f.each_line do |line|
-        yield parse_row(line, @f.lineno)
+    def each_row
+      s = @s
+      while row = parse_row(@s)
+        yield row
       end
     end
 
-    def parse_row(line, lineno = nil)
+    alias each each_row
+
+    def read_row
+      return nil if @s.eof?
+      parse_row(@s)
+    end
+
+    private
+
+    def parse_row(s)
+      s.next_row or return nil
       row = []
-      s = StringScanner.new(line)
-      s.skip(/\s+/)
-      until s.eos?
-        col = s.scan(/"(?:\\.|[^"\\]+)*"/) or raise Reader::MalformedCSVException, "CSV parse error at line #{lineno}"
-        row.push unescape_column(col)
-        s.skip(/\s*/)    # skip line terminator on line ends
-        s.skip(/,\s*/)
+      begin
+        first = false
+        column = s.scan_column
+        unless column
+          raise Reader::MalformedCSVException, "CSV parse error: unterminated column or row at line #{s.lineno}"
+        end
+        row.push unescape_column(column)
+      end while s.read_separator
+      unless s.read_eol
+        raise Reader::MalformedCSVException, "CSV parse error: missing column separator at line #{s.lineno}"
       end
       row
     end
 
     UNESCAPE_MAP = {
-      '\\"' => '"',
-      "\\'" => "'",
-      '\\,' => ',',
+      '\\t' => "\t",
       '\\r' => "\r",
       '\\n' => "\n",
-      '\\\\' => '\\'
     }
 
     def unescape_column(col)
       charmap = UNESCAPE_MAP
-      col[1...-1].gsub(/\\./) {|s| charmap[s] }
+      col[1...-1].gsub(/\\./m) {|s| charmap[s] || s[1,1] }
+    end
+
+    class ScanBuffer
+      def initialize(f)
+        @f = f
+        @s = StringScanner.new("")
+        @eof = false
+      end
+
+      def eof?
+        @s.eos? && @eof
+      end
+
+      def lineno
+        @f.lineno
+      end
+
+      def next_row
+        fill_buffer
+      end
+
+      MAX_COLUMN_LENGTH = (1.2 * (1024 ** 3)).to_i   # 1.2MB
+
+      def scan_column
+        s = @s
+        s.skip(/[ \t]+/)
+        until column = s.scan(/"(?:\\.|[^"\\]+)*"/m)
+          fill_buffer or return nil
+          return nil if s.eos?
+          if s.rest_size > MAX_COLUMN_LENGTH
+            raise Reader::MalformedCSVException, "CSV parse error: too long column at line #{@f.lineno}"
+          end
+        end
+        column
+      end
+
+      def fill_buffer
+        line = @f.gets
+        if line
+          @s << line
+          true
+        else
+          @eof = true
+          false
+        end
+      end
+
+      def read_separator
+        @s.skip(/[ \t]*,/)
+      end
+
+      def read_eol
+        @s.skip(/[ \t\r]*(?:\n|\z)/)
+      end
     end
   end
 end

data/redshift-connector-data_file.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "redshift-connector-data_file"
-  spec.version       = "7.2.0"
+  spec.version       = "7.3.0"
   spec.authors       = ["Hidekazu Kobayashi", "Minero Aoki"]
   spec.email         = ["kobahide789@gmail.com", "aamine@loveruby.net"]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector-data_file
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 7.3.0
 platform: ruby
 authors:
 - Hidekazu Kobayashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-30 00:00:00.000000000 Z
+date: 2018-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Utility classes for exported data files from Redshift