redshift-connector-data_file 7.0.0 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a91f1bb3c4764f126a7cd5db3c97b23626a8f54
-  data.tar.gz: 4d604f99c42e55ed8343a6240139df3ce723e63d
+  metadata.gz: 5f242fc490379db815a588378910c758a3aa2d84
+  data.tar.gz: 2a315d9407eb66562b32d83993ba577a5b5063ac
 SHA512:
-  metadata.gz: a64f93698aeee74754b6c0b4452b8bd9d06031235e0737a57f17752a0aba36659322a9e6a467e5c3f359f82fcd1e6f8ae1fef128a30c080370e08898d89216e7
-  data.tar.gz: 00e1d01e5da780e861a8c700ef2c45860bbef1e36e56ced8c19c4afc8c79750d1b50674c61474e8826904030ef6f13e4ec0eff82dca4d7bfa1a65caf18843a68
+  metadata.gz: 141660793595766a703b5bd798ee4a61c8f9dab6db02ea3084d7b9f5eb92305bf2ad7c6d3164a55eb9dfa5797593675ef161d4789509d1b35928bda840825bc4
+  data.tar.gz: dcfebdb474cd2f78be55f47d4afb2e2488bd3e43f72f88fc3806bcff80ab24619eb0afc4b054845d3f57d4961897b753b32b62aca0c9b99a1d883c3b4387cc08

data/lib/redshift_connector/abstract_data_file.rb

@@ -2,17 +2,24 @@ require 'zlib'
 
 module RedshiftConnector
   class AbstractDataFile
+    def initialize(reader_class:)
+      @reader_class = reader_class
+    end
+
     def each_row(&block)
-      f = if gzipped_object?
-            Zlib::GzipReader.new(content)
-          else
-            content
-          end
-      @reader_class.new(f).each(&block)
-    ensure
-      content.close
+      f = open
+      begin
+        if gzipped_object?
+          f = Zlib::GzipReader.new(f)
+        end
+        @reader_class.new(f).each(&block)
+      ensure
+        f.close
+      end
     end
 
+    # abstract open
+
     def data_object?
       @reader_class.data_object?(key)
     end

data/lib/redshift_connector/abstract_data_file_bundle.rb

@@ -1,5 +1,14 @@
 module RedshiftConnector
   class AbstractDataFileBundle
+    def initialize(filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
+      @filter = filter || lambda {|*row| row }
+      @batch_size = batch_size || 1000
+      @logger = logger
+    end
+
+    attr_reader :batch_size
+    attr_reader :logger
+
     def each_row(&block)
       each_object do |obj|
         obj.each_row(&block)
@@ -19,10 +28,11 @@ module RedshiftConnector
       data_files.select {|obj| obj.data_object? }
     end
 
+    # abstract data_files
+
     REPORT_SIZE = 10_0000
 
     def each_batch(report: true)
-      @logger.info "reader: #{@reader_class}"
       n = 0
       reported = 0
       do_each_batch(@batch_size) do |rows|

data/lib/redshift_connector/reader/redshift_csv.rb

@@ -1,5 +1,6 @@
 require 'redshift_connector/reader/abstract'
 require 'redshift_connector/reader/exception'
+require 'strscan'
 
 module RedshiftConnector
   # Reads CSV file generated by Redshift UNLOAD statement (with option ADDQUOTES ESCAPE).
@@ -29,7 +30,7 @@ module RedshiftConnector
       s = StringScanner.new(line)
       s.skip(/\s+/)
       until s.eos?
-        col = s.scan(/"(?:\\.|[^"\\]+)*"/) or raise MalformedCSVException, "CSV parse error at line #{lineno}"
+        col = s.scan(/"(?:\\.|[^"\\]+)*"/) or raise Reader::MalformedCSVException, "CSV parse error at line #{lineno}"
         row.push unescape_column(col)
         s.skip(/\s*/)    # skip line terminator on line ends
         s.skip(/,\s*/)

data/lib/redshift_connector/s3_data_file.rb

@@ -3,15 +3,15 @@ require 'redshift_connector/data_file'
 module RedshiftConnector
   class S3DataFile < AbstractDataFile
     def initialize(object, reader_class:)
+      super reader_class: reader_class
       @object = object
-      @reader_class = reader_class
     end
 
     def key
       @object.key
     end
 
-    def content
+    def open
       @object.get.body
     end
 

data/lib/redshift_connector/s3_data_file_bundle.rb

@@ -32,13 +32,12 @@ module RedshiftConnector
     end
 
     def initialize(bucket, prefix, format: :csv, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
+      super filter: filter, batch_size: batch_size, logger: logger
       @bucket = bucket
       @prefix = prefix
       @format = format
-      @filter = filter || lambda {|*row| row }
-      @batch_size = batch_size
-      @logger = logger
       @reader_class = Reader.get(format)
+      logger.info "reader: #{@reader_class}"
     end
 
     attr_reader :bucket
@@ -61,7 +60,7 @@ module RedshiftConnector
       pref = File.dirname(@prefix) + '/'
       keys = @bucket.objects(prefix: pref).map(&:key)
       unless keys.empty?
-        @logger.info "DELETE #{pref}*"
+        logger.info "DELETE #{pref}*"
         @bucket.delete_objects(keys)
       end
     end

data/lib/redshift_connector/url_data_file.rb

@@ -1,22 +1,23 @@
 require 'redshift_connector/abstract_data_file'
-require 'uri'
-require 'zlib'
 require 'open3'
 
 module RedshiftConnector
   class UrlDataFile < AbstractDataFile
     def initialize(url, reader_class:)
+      super reader_class: reader_class
       @url = url
-      @reader_class = reader_class
     end
 
+    attr_reader :url
+
     def key
-      URI.parse(@url).path
+      @url.path
     end
 
-    def content
-      stdin, stdout, stderr, wait_th = Open3.popen3('curl', @url)
+    def open
+      stdin, stdout, stderr, wait_th = Open3.popen3('curl', @url.to_s)
       stdin.close
+      stderr.close
       stdout
     end
   end

data/lib/redshift_connector/url_data_file_bundle.rb

@@ -2,21 +2,18 @@ require 'redshift_connector/reader'
 require 'redshift_connector/abstract_data_file_bundle'
 require 'redshift_connector/url_data_file'
 require 'redshift_connector/logger'
+require 'uri'
 
 module RedshiftConnector
   class UrlDataFileBundle < AbstractDataFileBundle
-    def initialize(data_file_urls, format: :redshift_csv, filter: nil, logger: RedshiftConnector.logger)
-      raise ArgumentError, 'data_file_urls is empty' if data_file_urls.empty?
-      @data_file_urls = data_file_urls
-      @filter = filter || lambda {|*row| row }
-      @logger = logger
-      @reader_class = Reader.get(format)
+    def initialize(urls, format: :redshift_csv, filter: nil, logger: RedshiftConnector.logger)
+      super filter: filter, batch_size: 1000, logger: logger
+      reader_class = Reader.get(format)
+      raise ArgumentError, 'no URL given' if urls.empty?
+      @data_files = urls.map {|url| UrlDataFile.new(URI.parse(url), reader_class: reader_class) }
+      logger.info "reader: #{reader_class}"
     end
 
-    def data_files
-      @data_file_urls.map do |url|
-        UrlDataFile.new(url, reader_class: @reader_class)
-      end
-    end
+    attr_reader :data_files
   end
 end

data/redshift-connector-data_file.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "redshift-connector-data_file"
-  spec.version       = "7.0.0"
+  spec.version       = "7.1.0"
   spec.authors       = ["Hidekazu Kobayashi", "Minero Aoki"]
   spec.email         = ["kobahide789@gmail.com", "aamine@loveruby.net"]
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector-data_file
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.1.0
 platform: ruby
 authors:
 - Hidekazu Kobayashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-19 00:00:00.000000000 Z
+date: 2017-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Utility classes for exported data files from Redshift