redshift_csv_file 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 988bff806bddf57141c4d2819c6c440fbdc25955a5b83060299e66699dfdca2b
+  data.tar.gz: 013c197fbffc732dc8ade4554059c00d2f9cd4764a25b4b91aca50102a245df3
+SHA512:
+  metadata.gz: 19f59a3acde973ef62d222aeb40624555b11c0add2b0d7b194891e2929282e8b56d5a2f175ad5569fa46bfea58b9b7a02f78064c8ca316ed1c29afcea9bd3e7d
+  data.tar.gz: 2a65476327e518e612f1011e73a9c1870d0fe1cdeffcb52338c7739c158242ac6b1f2e31a483de808aecff27fc2a4fa7ec27debcc24164441c0a6f935b85a78a

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Bricolage Org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+# redshift_csv_file library
+
+redshift_csv_file is a Redshift-specific CSV file parser in Ruby.
+Amazon Redshift generates non-standard CSV format, special handling is required.
+
+## Usage
+
+```
+require 'redshift_csv_file'
+
+File.open('unloaded_file.csv') {|f|
+  RedshiftCsvFile.new(f).each_row do |col1, col2, col3|
+    p [col1, col2, col3]   # => ["value1", "value2", "value3"]
+  end
+}
+```
+
+## License
+
+MIT license.
+See LICENSE file for details.
+
+## Author
+
+Minero Aoki

data/lib/redshift_csv_file.rb

@@ -0,0 +1,111 @@
+require 'strscan'
+
+# Reads CSV file generated by Redshift UNLOAD statement (with option ADDQUOTES ESCAPE).
+# UNLOAD escapes data by '\' (backslash character), we cannot use standard CSV class.
+class RedshiftCsvFile
+  class MalformedCSVException < StandardError; end
+
+  # f :: IO
+  def initialize(f)
+    @f = f
+    @s = ScanBuffer.new(@f)
+  end
+
+  def each_row
+    s = @s
+    while row = parse_row(@s)
+      yield row
+    end
+  end
+
+  alias each each_row
+
+  def read_row
+    return nil if @s.eof?
+    parse_row(@s)
+  end
+
+  private
+
+  def parse_row(s)
+    s.next_row or return nil
+    row = []
+    begin
+      first = false
+      column = s.scan_column
+      unless column
+        raise MalformedCSVException, "CSV parse error: unterminated column or row at line #{s.lineno}"
+      end
+      row.push unescape_column(column)
+    end while s.read_separator
+    unless s.read_eol
+      raise MalformedCSVException, "CSV parse error: missing column separator at line #{s.lineno}"
+    end
+    row
+  end
+
+  UNESCAPE_MAP = {
+    '\\t' => "\t",
+    '\\r' => "\r",
+    '\\n' => "\n",
+  }
+
+  def unescape_column(col)
+    charmap = UNESCAPE_MAP
+    col[1...-1].gsub(/\\./m) {|s| charmap[s] || s[1,1] }
+  end
+
+  class ScanBuffer
+    def initialize(f)
+      @f = f
+      @s = StringScanner.new("")
+      @eof = false
+    end
+
+    def eof?
+      @s.eos? && @eof
+    end
+
+    def lineno
+      @f.lineno
+    end
+
+    def next_row
+      fill_buffer
+    end
+
+    MAX_COLUMN_LENGTH = (1.2 * (1024 ** 3)).to_i   # 1.2MB
+
+    def scan_column
+      s = @s
+      s.skip(/[ \t]+/)
+      until column = s.scan(/"(?:\\.|[^"\\]+)*"/m)
+        fill_buffer or return nil
+        return nil if s.eos?
+        if s.rest_size > MAX_COLUMN_LENGTH
+          raise MalformedCSVException, "CSV parse error: too long column at line #{@f.lineno}"
+        end
+      end
+      column
+    end
+
+    def fill_buffer
+      line = @f.gets
+      if line
+        @s << line
+        true
+      else
+        @eof = true
+        false
+      end
+    end
+
+    def read_separator
+      @s.skip(/[ \t]*,/)
+    end
+
+    def read_eol
+      @s.skip(/[ \t\r]*(?:\n|\z)/)
+    end
+  end
+end

data/redshift_csv_file.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.name = 'redshift_csv_file'
+  s.version = '1.0.0'
+  s.summary = 'Redshift unloaded CSV file parser'
+  s.description = 'redshift_csv_file is a Redshift-specific CSV file parser.  Amazon Redshift generates non-standard CSV format, special handling is required.'
+  s.license = 'MIT'
+
+  s.author = ['Minero Aoki']
+  s.email = 'aamine@loveruby.net'
+  s.homepage = 'https://github.com/bricolages/redshift_csv_file'
+
+  s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  s.require_path = 'lib'
+
+  s.required_ruby_version = '>= 2.3.0'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: redshift_csv_file
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: redshift_csv_file is a Redshift-specific CSV file parser.  Amazon Redshift
+  generates non-standard CSV format, special handling is required.
+email: aamine@loveruby.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- lib/redshift_csv_file.rb
+- redshift_csv_file.gemspec
+homepage: https://github.com/bricolages/redshift_csv_file
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Redshift unloaded CSV file parser
+test_files: []