redshift-connector 5.4.1 → 5.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c334f32bf3c597fc80cdf2e783f6eea993e58166
-  data.tar.gz: ec1e1f4b3bb6d5ae2561c5c401ac4bf6644fc4f6
+  metadata.gz: 64ec55df09c992e2e792de2b6dea9554cba4d4be
+  data.tar.gz: 4c37b910361878dfb6830b79da024e968ab78726
 SHA512:
-  metadata.gz: c7693bbacee2b71405da3d48d244775c0cf39f7225d9d2f1d67fc2e48ff5132b9cadd7ccf5c523076ed5404928a2d40e8e13debf159be7213b89c8df7015cda2
-  data.tar.gz: df952ee8ef13d61c7efd1dfd9643e4018a6978cb9cfe6d3e1a36bd845ccbafa59b6bfa726e80978a1d62114bbf001287b3a9e249cb242902c6c823859549a730
+  metadata.gz: a268d84364402590e59c115831fddce59be352b795676715a07f4d54a966a14fdf5a19a926783ea5f8c38073d5fa283c21822b1ea91600e224061a4932006dfa
+  data.tar.gz: be94cf2ae56df70b8c9e1be2d67619e26f6edb942957a4bb977c824042a55729b550a7a699c6400ac3f6566c1834a243bbcf773e1898a19b4e51da5ecc26e03d

data/lib/redshift-connector/s3_data_file.rb

@@ -1,4 +1,4 @@
-require 'redshift-connector/abstract_data_file'
+require 'redshift-connector/data_file'
 
 module RedshiftConnector
   class S3DataFile < AbstractDataFile

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -1,8 +1,7 @@
 require 'redshift-connector/s3_bucket'
 require 'redshift-connector/s3_data_file'
-require 'redshift-connector/reader'
 require 'redshift-connector/logger'
-require 'redshift-connector/abstract_data_file_bundle'
+require 'redshift-connector/data_file'
 require 'aws-sdk'
 
 module RedshiftConnector

data/lib/redshift-connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '5.4.1'
+  VERSION = '5.5.0'
 end

data/test/reader/test_redshift_csv.rb

@@ -1,5 +1,5 @@
 require 'test/unit'
-require 'redshift-connector/reader'
+require 'redshift-connector/data_file'
 
 module RedshiftConnector
   module Reader

data/test/test_reader.rb

@@ -1,5 +1,4 @@
 require 'test/unit'
-require 'redshift-connector/reader'
 
 module RedshiftConnector
   class TestReader < Test::Unit::TestCase

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 5.4.1
+  version: 5.5.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-04 00:00:00.000000000 Z
+date: 2017-05-12 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: redshift-connector-data_file
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -130,8 +144,6 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/redshift-connector.rb
-- lib/redshift-connector/abstract_data_file.rb
-- lib/redshift-connector/abstract_data_file_bundle.rb
 - lib/redshift-connector/connector.rb
 - lib/redshift-connector/exporter.rb
 - lib/redshift-connector/importer.rb
@@ -142,17 +154,9 @@ files:
 - lib/redshift-connector/importer/upsert.rb
 - lib/redshift-connector/logger.rb
 - lib/redshift-connector/query.rb
-- lib/redshift-connector/reader.rb
-- lib/redshift-connector/reader/abstract.rb
-- lib/redshift-connector/reader/csv.rb
-- lib/redshift-connector/reader/exception.rb
-- lib/redshift-connector/reader/redshift_csv.rb
-- lib/redshift-connector/reader/tsv.rb
 - lib/redshift-connector/s3_bucket.rb
 - lib/redshift-connector/s3_data_file.rb
 - lib/redshift-connector/s3_data_file_bundle.rb
-- lib/redshift-connector/url_data_file.rb
-- lib/redshift-connector/url_data_file_bundle.rb
 - lib/redshift-connector/version.rb
 - test/all.rb
 - test/config.rb

data/lib/redshift-connector/abstract_data_file.rb

@@ -1,24 +0,0 @@
-require 'zlib'
-
-module RedshiftConnector
-  class AbstractDataFile
-    def each_row(&block)
-      f = if gzipped_object?
-            Zlib::GzipReader.new(content)
-          else
-            content
-          end
-      @reader_class.new(f).each(&block)
-    ensure
-      content.close
-    end
-
-    def data_object?
-      @reader_class.data_object?(key)
-    end
-
-    def gzipped_object?
-      File.extname(key) == '.gz'
-    end
-  end
-end

data/lib/redshift-connector/abstract_data_file_bundle.rb

@@ -1,22 +0,0 @@
-module RedshiftConnector
-  class AbstractDataFileBundle
-    def each_row(&block)
-      each_object do |obj|
-        obj.each_row(&block)
-      end
-    end
-
-    alias each each_row
-
-    def each_object(&block)
-      all_data_objects.each do |obj|
-        @logger.info "processing s3 object: #{obj.key}"
-        yield obj
-      end
-    end
-
-    def all_data_objects
-      data_files.select {|obj| obj.data_object? }
-    end
-  end
-end

data/lib/redshift-connector/reader.rb

@@ -1,18 +0,0 @@
-# create module
-module RedshiftConnector
-  module Reader
-  end
-end
-
-require 'redshift-connector/reader/redshift_csv'
-require 'redshift-connector/reader/csv'
-require 'redshift-connector/reader/tsv'
-require 'redshift-connector/reader/exception'
-
-module RedshiftConnector
-  module Reader
-    def Reader.get(id)
-      Abstract.get_reader_class(id)
-    end
-  end
-end

data/lib/redshift-connector/reader/abstract.rb

@@ -1,18 +0,0 @@
-module RedshiftConnector
-  class Reader::Abstract
-    READER_CLASSES = {}   # {Symbol => Class}
-
-    def self.declare_reader(id)
-      READER_CLASSES[id.to_sym] = self
-    end
-
-    def self.get_reader_class(id)
-      READER_CLASSES[id.to_sym] or
-          raise ArgumentError, "unknown data file reader type: #{id.inspect}"
-    end
-  end
-
-  def self.get_reader_class(id)
-    Reader::Abstract.get_reader_class(id)
-  end
-end

data/lib/redshift-connector/reader/csv.rb

@@ -1,24 +0,0 @@
-require 'redshift-connector/reader/abstract'
-require 'redshift-connector/reader/exception'
-require 'csv'
-
-module RedshiftConnector
-  # Parses (standard) CSV files.
-  # For UNLOAD-generated CSV, use RedshiftCSV class.
-  class Reader::CSV < Reader::Abstract
-    declare_reader :csv
-
-    def self.data_object?(key)
-      /\.csv(?:\.|\z)/ =~ File.basename(key)
-    end
-
-    def initialize(f)
-      @f = f
-    end
-
-    def each(&block)
-      csv = CSV.new(@f)
-      csv.each(&block)
-    end
-  end
-end

data/lib/redshift-connector/reader/exception.rb

@@ -1,3 +0,0 @@
-module RedshiftConnector
-  class Reader::MalformedCSVException < StandardError; end
-end

data/lib/redshift-connector/reader/redshift_csv.rb

@@ -1,54 +0,0 @@
-require 'redshift-connector/reader/abstract'
-require 'redshift-connector/reader/exception'
-
-module RedshiftConnector
-  # Reads CSV file generated by Redshift UNLOAD statement (with option ADDQUOTES ESCAPE).
-  # UNLOAD escapes data by '\' (backslash character), we cannot use standard CSV class.
-  class Reader::RedshiftCSV < Reader::Abstract
-    declare_reader :redshift_csv
-
-    def self.data_object?(key)
-      /\.csv(?:\.|\z)/ =~ File.basename(key)
-    end
-
-    # f :: IO
-    def initialize(f)
-      @f = f
-    end
-
-    def each
-      # We can use simple #each_line to read single row
-      # because line terminators are always escaped by UNLOAD.
-      @f.each_line do |line|
-        yield parse_row(line, @f.lineno)
-      end
-    end
-
-    def parse_row(line, lineno = nil)
-      row = []
-      s = StringScanner.new(line)
-      s.skip(/\s+/)
-      until s.eos?
-        col = s.scan(/"(?:\\.|[^"\\]+)*"/) or raise MalformedCSVException, "CSV parse error at line #{lineno}"
-        row.push unescape_column(col)
-        s.skip(/\s*/)    # skip line terminator on line ends
-        s.skip(/,\s*/)
-      end
-      row
-    end
-
-    UNESCAPE_MAP = {
-      '\\"' => '"',
-      "\\'" => "'",
-      '\\,' => ',',
-      '\\r' => "\r",
-      '\\n' => "\n",
-      '\\\\' => '\\'
-    }
-
-    def unescape_column(col)
-      charmap = UNESCAPE_MAP
-      col[1...-1].gsub(/\\./) {|s| charmap[s] }
-    end
-  end
-end

data/lib/redshift-connector/reader/tsv.rb

@@ -1,24 +0,0 @@
-require 'redshift-connector/reader/abstract'
-require 'redshift-connector/reader/exception'
-require 'csv'
-
-module RedshiftConnector
-  # Parses TSV (Tab Separated Format) files.
-  class Reader::TSV < Reader::Abstract
-    declare_reader :tsv
-
-    def self.data_object?(key)
-      /\.tsv(?:\.|\z)/ =~ File.basename(key)
-    end
-
-    def initialize(f)
-      @f = f
-    end
-
-    def each(&block)
-      @f.each_line do |line|
-        yield line.chomp.split("\t", -1)
-      end
-    end
-  end
-end

data/lib/redshift-connector/url_data_file.rb

@@ -1,23 +0,0 @@
-require 'redshift-connector/abstract_data_file'
-require 'uri'
-require 'zlib'
-require 'open3'
-
-module RedshiftConnector
-  class UrlDataFile < AbstractDataFile
-    def initialize(url, reader_class:)
-      @url = url
-      @reader_class = reader_class
-    end
-
-    def key
-      URI.parse(@url).path
-    end
-
-    def content
-      stdin, stdout, stderr, wait_th = Open3.popen3('curl', @url)
-      stdin.close
-      stdout
-    end
-  end
-end

data/lib/redshift-connector/url_data_file_bundle.rb

@@ -1,21 +0,0 @@
-require 'redshift-connector/reader'
-require 'redshift-connector/logger'
-require 'redshift-connector/abstract_data_file_bundle'
-require 'redshift-connector/url_data_file'
-
-module RedshiftConnector
-  class UrlDataFileBundle < AbstractDataFileBundle
-    def initialize(data_file_urls, format: :redshift_csv, filter: nil, logger: RedshiftConnector.logger)
-      @data_file_urls = data_file_urls
-      @filter = filter || lambda {|*row| row }
-      @logger = logger
-      @reader_class = Reader.get(format)
-    end
-
-    def data_files
-      @data_file_urls.map do |url|
-        UrlDataFile.new(url, reader_class: @reader_class)
-      end
-    end
-  end
-end