redshift-connector 4.4.1 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 905cd291f9b03a51619ed92e3aa21a88775928df
-  data.tar.gz: 1d7616a5482652d634a4004c790825f3f638225a
+  metadata.gz: cb3953798104bd92d6d856b8fd3b2d835ba9d54a
+  data.tar.gz: e2f51e74ec4969e1187ce35e457a2af357bda1bf
 SHA512:
-  metadata.gz: d70d4f5ea5a0ba4369b1d9f730b0a861779c3f66b72eb036ca34653817af3c2d32e6d42190f4f61438a9324851a77eba2f2651e3b6472520aff5f13b2f500edf
-  data.tar.gz: 6d2683f7361f3706a916d6bab4571a5bfa0a6ca8baeff3155c25005174ea1d07468c0b8959765be7548540a2a3c74e316e358e841ba0ff4c42f3e8df373c5a0e
+  metadata.gz: cec111e3ca92096a3a0071b7a76a36b3c259213ab03aecb5f75ca1e3b2cc41fd1b4e140384a5a482944d7f02819cfe8da5010ae0bd8c9e9b0619001e09aa3542
+  data.tar.gz: ac5dbcc924a7ffb5ee7387d4cf3f56a78371c10a870d4b3b3215502d19f256daff0bcc970ed2d7d946b6ecd62f8305a4ab55ecd6adbd0e4b1896cbf42b671067

data/lib/redshift-connector/s3_data_file.rb

@@ -1,4 +1,4 @@
-require 'redshift-connector/abstract_data_file'
+require 'redshift-connector/data_file'
 
 module RedshiftConnector
   class S3DataFile < AbstractDataFile

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -1,8 +1,7 @@
 require 'redshift-connector/s3_bucket'
 require 'redshift-connector/s3_data_file'
-require 'redshift-connector/reader'
 require 'redshift-connector/logger'
-require 'redshift-connector/abstract_data_file_bundle'
+require 'redshift-connector/data_file'
 require 'aws-sdk'
 
 module RedshiftConnector

data/lib/redshift-connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '4.4.1'
+  VERSION = '4.5.0'
 end

data/test/reader/test_redshift_csv.rb

@@ -1,5 +1,5 @@
 require 'test/unit'
-require 'redshift-connector/reader'
+require 'redshift-connector/data_file'
 
 module RedshiftConnector
   module Reader

data/test/test_reader.rb

@@ -1,5 +1,4 @@
 require 'test/unit'
-require 'redshift-connector/reader'
 
 module RedshiftConnector
   class TestReader < Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 4.4.1
+  version: 4.5.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-04 00:00:00.000000000 Z
+date: 2017-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redshift-connector-data_file
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -130,8 +144,6 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/redshift-connector.rb
-- lib/redshift-connector/abstract_data_file.rb
-- lib/redshift-connector/abstract_data_file_bundle.rb
 - lib/redshift-connector/connector.rb
 - lib/redshift-connector/exporter.rb
 - lib/redshift-connector/importer.rb
@@ -142,17 +154,9 @@ files:
 - lib/redshift-connector/importer/upsert.rb
 - lib/redshift-connector/logger.rb
 - lib/redshift-connector/query.rb
-- lib/redshift-connector/reader.rb
-- lib/redshift-connector/reader/abstract.rb
-- lib/redshift-connector/reader/csv.rb
-- lib/redshift-connector/reader/exception.rb
-- lib/redshift-connector/reader/redshift_csv.rb
-- lib/redshift-connector/reader/tsv.rb
 - lib/redshift-connector/s3_bucket.rb
 - lib/redshift-connector/s3_data_file.rb
 - lib/redshift-connector/s3_data_file_bundle.rb
-- lib/redshift-connector/url_data_file.rb
-- lib/redshift-connector/url_data_file_bundle.rb
 - lib/redshift-connector/version.rb
 - test/all.rb
 - test/config.rb

data/lib/redshift-connector/abstract_data_file.rb

@@ -1,24 +0,0 @@
-require 'zlib'
-
-module RedshiftConnector
-  class AbstractDataFile
-    def each_row(&block)
-      f = if gzipped_object?
-            Zlib::GzipReader.new(content)
-          else
-            content
-          end
-      @reader_class.new(f).each(&block)
-    ensure
-      content.close
-    end
-
-    def data_object?
-      @reader_class.data_object?(key)
-    end
-
-    def gzipped_object?
-      File.extname(key) == '.gz'
-    end
-  end
-end

data/lib/redshift-connector/abstract_data_file_bundle.rb

@@ -1,22 +0,0 @@
-module RedshiftConnector
-  class AbstractDataFileBundle
-    def each_row(&block)
-      each_object do |obj|
-        obj.each_row(&block)
-      end
-    end
-
-    alias each each_row
-
-    def each_object(&block)
-      all_data_objects.each do |obj|
-        @logger.info "processing s3 object: #{obj.key}"
-        yield obj
-      end
-    end
-
-    def all_data_objects
-      data_files.select {|obj| obj.data_object? }
-    end
-  end
-end

data/lib/redshift-connector/reader.rb

@@ -1,18 +0,0 @@
-# create module
-module RedshiftConnector
-  module Reader
-  end
-end
-
-require 'redshift-connector/reader/redshift_csv'
-require 'redshift-connector/reader/csv'
-require 'redshift-connector/reader/tsv'
-require 'redshift-connector/reader/exception'
-
-module RedshiftConnector
-  module Reader
-    def Reader.get(id)
-      Abstract.get_reader_class(id)
-    end
-  end
-end

data/lib/redshift-connector/reader/abstract.rb

@@ -1,18 +0,0 @@
-module RedshiftConnector
-  class Reader::Abstract
-    READER_CLASSES = {}   # {Symbol => Class}
-
-    def self.declare_reader(id)
-      READER_CLASSES[id.to_sym] = self
-    end
-
-    def self.get_reader_class(id)
-      READER_CLASSES[id.to_sym] or
-          raise ArgumentError, "unknown data file reader type: #{id.inspect}"
-    end
-  end
-
-  def self.get_reader_class(id)
-    Reader::Abstract.get_reader_class(id)
-  end
-end

data/lib/redshift-connector/reader/csv.rb

@@ -1,24 +0,0 @@
-require 'redshift-connector/reader/abstract'
-require 'redshift-connector/reader/exception'
-require 'csv'
-
-module RedshiftConnector
-  # Parses (standard) CSV files.
-  # For UNLOAD-generated CSV, use RedshiftCSV class.
-  class Reader::CSV < Reader::Abstract
-    declare_reader :csv
-
-    def self.data_object?(key)
-      /\.csv(?:\.|\z)/ =~ File.basename(key)
-    end
-
-    def initialize(f)
-      @f = f
-    end
-
-    def each(&block)
-      csv = CSV.new(@f)
-      csv.each(&block)
-    end
-  end
-end

data/lib/redshift-connector/reader/exception.rb

@@ -1,3 +0,0 @@
-module RedshiftConnector
-  class Reader::MalformedCSVException < StandardError; end
-end

data/lib/redshift-connector/reader/redshift_csv.rb

@@ -1,54 +0,0 @@
-require 'redshift-connector/reader/abstract'
-require 'redshift-connector/reader/exception'
-
-module RedshiftConnector
-  # Reads CSV file generated by Redshift UNLOAD statement (with option ADDQUOTES ESCAPE).
-  # UNLOAD escapes data by '\' (backslash character), we cannot use standard CSV class.
-  class Reader::RedshiftCSV < Reader::Abstract
-    declare_reader :redshift_csv
-
-    def self.data_object?(key)
-      /\.csv(?:\.|\z)/ =~ File.basename(key)
-    end
-
-    # f :: IO
-    def initialize(f)
-      @f = f
-    end
-
-    def each
-      # We can use simple #each_line to read single row
-      # because line terminators are always escaped by UNLOAD.
-      @f.each_line do |line|
-        yield parse_row(line, @f.lineno)
-      end
-    end
-
-    def parse_row(line, lineno = nil)
-      row = []
-      s = StringScanner.new(line)
-      s.skip(/\s+/)
-      until s.eos?
-        col = s.scan(/"(?:\\.|[^"\\]+)*"/) or raise MalformedCSVException, "CSV parse error at line #{lineno}"
-        row.push unescape_column(col)
-        s.skip(/\s*/)    # skip line terminator on line ends
-        s.skip(/,\s*/)
-      end
-      row
-    end
-
-    UNESCAPE_MAP = {
-      '\\"' => '"',
-      "\\'" => "'",
-      '\\,' => ',',
-      '\\r' => "\r",
-      '\\n' => "\n",
-      '\\\\' => '\\'
-    }
-
-    def unescape_column(col)
-      charmap = UNESCAPE_MAP
-      col[1...-1].gsub(/\\./) {|s| charmap[s] }
-    end
-  end
-end

data/lib/redshift-connector/reader/tsv.rb

@@ -1,24 +0,0 @@
-require 'redshift-connector/reader/abstract'
-require 'redshift-connector/reader/exception'
-require 'csv'
-
-module RedshiftConnector
-  # Parses TSV (Tab Separated Format) files.
-  class Reader::TSV < Reader::Abstract
-    declare_reader :tsv
-
-    def self.data_object?(key)
-      /\.tsv(?:\.|\z)/ =~ File.basename(key)
-    end
-
-    def initialize(f)
-      @f = f
-    end
-
-    def each(&block)
-      @f.each_line do |line|
-        yield line.chomp.split("\t", -1)
-      end
-    end
-  end
-end

data/lib/redshift-connector/url_data_file.rb

@@ -1,23 +0,0 @@
-require 'redshift-connector/abstract_data_file'
-require 'uri'
-require 'zlib'
-require 'open3'
-
-module RedshiftConnector
-  class UrlDataFile < AbstractDataFile
-    def initialize(url, reader_class:)
-      @url = url
-      @reader_class = reader_class
-    end
-
-    def key
-      URI.parse(@url).path
-    end
-
-    def content
-      stdin, stdout, stderr, wait_th = Open3.popen3('curl', @url)
-      stdin.close
-      stdout
-    end
-  end
-end

data/lib/redshift-connector/url_data_file_bundle.rb

@@ -1,21 +0,0 @@
-require 'redshift-connector/reader'
-require 'redshift-connector/logger'
-require 'redshift-connector/abstract_data_file_bundle'
-require 'redshift-connector/url_data_file'
-
-module RedshiftConnector
-  class UrlDataFileBundle < AbstractDataFileBundle
-    def initialize(data_file_urls, format: :redshift_csv, filter: nil, logger: RedshiftConnector.logger)
-      @data_file_urls = data_file_urls
-      @filter = filter || lambda {|*row| row }
-      @logger = logger
-      @reader_class = Reader.get(format)
-    end
-
-    def data_files
-      @data_file_urls.map do |url|
-        UrlDataFile.new(url, reader_class: @reader_class)
-      end
-    end
-  end
-end