redshift-connector 4.3.1 → 4.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ed37ac9fef3effe7b2e9c2c288be44f2d942d35
-  data.tar.gz: 5b8b27e51b258609c83b2f06d3df3268e78cccfa
+  metadata.gz: b12a5bb9e0027d0251a3d3e98766d60c1a7a2bf1
+  data.tar.gz: 2e27ed62452041a2dd6bac8471014bdbf4702ef6
 SHA512:
-  metadata.gz: db50e26f55babb5e0e897d08799001e5f3dd5f4a69375711e9275f52edc6d4a6ada9b6543f185ce8de80459d39e51c7576b7de7f1990b3b9f824c70d5fb669e8
-  data.tar.gz: a1147ff8f8d20624c80674c49857c80a00ac0684235b5bd90c0f0ab8e86e32ab316e343cefd362b5d1fd8139c5fe79ffb9b79c3795de0a478d43a169611eb2b7
+  metadata.gz: 2ff4c494a726ae8749e1f3e3e13bfa070908e79aba4c037742117b5afb500a27ed84474d43a65ea6a2d57b12bfdf379b6126c3a1d76bf3962e625bee419efea7
+  data.tar.gz: 702273bc9689811b99345cd4fa685dd0916252a6695e462201323b77224809deb95dfc9a663076b9a54b1dd4ca1c7abc724ff29faf29d759ad4f9559ec400f77

data/lib/redshift-connector/abstract_data_file.rb

@@ -0,0 +1,24 @@
+require 'zlib'
+
+module RedshiftConnector
+  class AbstractDataFile
+    def each_row(&block)
+      f = if gzipped_object?
+            Zlib::GzipReader.new(content)
+          else
+            content
+          end
+      @reader_class.new(f).each(&block)
+    ensure
+      content.close
+    end
+
+    def data_object?
+      @reader_class.data_object?(key)
+    end
+
+    def gzipped_object?
+      File.extname(key) == '.gz'
+    end
+  end
+end

data/lib/redshift-connector/abstract_data_file_bundle.rb

@@ -0,0 +1,22 @@
+module RedshiftConnector
+  class AbstractDataFileBundle
+    def each_row(&block)
+      each_object do |obj|
+        obj.each_row(&block)
+      end
+    end
+
+    alias each each_row
+
+    def each_object(&block)
+      all_data_objects.each do |obj|
+        @logger.info "processing s3 object: #{obj.key}"
+        yield obj
+      end
+    end
+
+    def all_data_objects
+      data_files.select {|obj| obj.data_object? }
+    end
+  end
+end

data/lib/redshift-connector/reader/csv.rb

@@ -8,8 +8,8 @@ module RedshiftConnector
   class Reader::CSV < Reader::Abstract
     declare_reader :csv
 
-    def self.data_object?(obj)
-      /\.csv(?:\.|\z)/ =~ File.basename(obj.key)
+    def self.data_object?(key)
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
     end
 
     def initialize(f)

data/lib/redshift-connector/reader/redshift_csv.rb

@@ -7,8 +7,8 @@ module RedshiftConnector
   class Reader::RedshiftCSV < Reader::Abstract
     declare_reader :redshift_csv
 
-    def self.data_object?(obj)
-      /\.csv(?:\.|\z)/ =~ File.basename(obj.key)
+    def self.data_object?(key)
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
     end
 
     # f :: IO

data/lib/redshift-connector/reader/tsv.rb

@@ -7,8 +7,8 @@ module RedshiftConnector
   class Reader::TSV < Reader::Abstract
     declare_reader :tsv
 
-    def self.data_object?(obj)
-      /\.tsv(?:\.|\z)/ =~ File.basename(obj.key)
+    def self.data_object?(key)
+      /\.tsv(?:\.|\z)/ =~ File.basename(key)
     end
 
     def initialize(f)

data/lib/redshift-connector/s3_data_file.rb

@@ -1,7 +1,7 @@
-require 'zlib'
+require 'redshift-connector/abstract_data_file'
 
 module RedshiftConnector
-  class S3DataFile
+  class S3DataFile < AbstractDataFile
     def initialize(object, reader_class:)
       @object = object
       @reader_class = reader_class
@@ -11,24 +11,10 @@ module RedshiftConnector
       @object.key
     end
 
-    def each_row(&block)
-      response = @object.get
-      f = if gzipped_object?
-        Zlib::GzipReader.new(response.body)
-      else
-        response.body
-      end
-      @reader_class.new(f).each(&block)
-    ensure
-      response.body.close if response
+    def content
+      @object.get.body
     end
 
-    def data_object?
-      @reader_class.data_object?(@object)
-    end
-
-    def gzipped_object?
-      File.extname(@object.key) == '.gz'
-    end
+    delegate :presigned_url, to: :@object
   end
 end

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -2,10 +2,11 @@ require 'redshift-connector/s3_bucket'
 require 'redshift-connector/s3_data_file'
 require 'redshift-connector/reader'
 require 'redshift-connector/logger'
+require 'redshift-connector/abstract_data_file_bundle'
 require 'aws-sdk'
 
 module RedshiftConnector
-  class S3DataFileBundle
+  class S3DataFileBundle < AbstractDataFileBundle
     def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
       real_prefix = "#{bucket.prefix}/#{prefix}"
       new(bucket, real_prefix, format: format, filter: filter, batch_size: batch_size, logger: logger)
@@ -68,25 +69,9 @@ module RedshiftConnector
     end
     private :do_each_batch
 
-    def each_row(&block)
-      each_object do |obj|
-        obj.each_row(&block)
-      end
-    end
-
-    alias each each_row
-
-    def each_object(&block)
-      all_data_objects.each do |obj|
-        @logger.info "processing s3 object: #{obj.key}"
-        yield obj
-      end
-    end
-
-    def all_data_objects
+    def data_files
       @bucket.objects(prefix: @prefix)
         .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }
-        .select {|obj| obj.data_object? }
     end
 
     def clear

data/lib/redshift-connector/url_data_file.rb

@@ -0,0 +1,23 @@
+require 'redshift-connector/abstract_data_file'
+require 'uri'
+require 'zlib'
+require 'open3'
+
+module RedshiftConnector
+  class UrlDataFile < AbstractDataFile
+    def initialize(url, reader_class:)
+      @url = url
+      @reader_class = reader_class
+    end
+
+    def key
+      URI.parse(@url).path
+    end
+
+    def content
+      stdin, stdout, stderr, wait_th = Open3.popen3('curl', @url)
+      stdin.close
+      stdout
+    end
+  end
+end

data/lib/redshift-connector/url_data_file_bundle.rb

@@ -0,0 +1,21 @@
+require 'redshift-connector/reader'
+require 'redshift-connector/logger'
+require 'redshift-connector/abstract_data_file_bundle'
+require 'redshift-connector/url_data_file'
+
+module RedshiftConnector
+  class UrlDataFileBundle < AbstractDataFileBundle
+    def initialize(data_file_urls, format: :redshift_csv, filter: nil, logger: RedshiftConnector.logger)
+      @data_file_urls = data_file_urls
+      @filter = filter || lambda {|*row| row }
+      @logger = logger
+      @reader_class = Reader.get(format)
+    end
+
+    def data_files
+      @data_file_urls.map do |url|
+        UrlDataFile.new(url, reader_class: @reader_class)
+      end
+    end
+  end
+end

data/lib/redshift-connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '4.3.1'
+  VERSION = '4.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 4.3.1
+  version: 4.3.2
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-20 00:00:00.000000000 Z
+date: 2017-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -130,6 +130,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/redshift-connector.rb
+- lib/redshift-connector/abstract_data_file.rb
+- lib/redshift-connector/abstract_data_file_bundle.rb
 - lib/redshift-connector/connector.rb
 - lib/redshift-connector/exporter.rb
 - lib/redshift-connector/importer.rb
@@ -149,6 +151,8 @@ files:
 - lib/redshift-connector/s3_bucket.rb
 - lib/redshift-connector/s3_data_file.rb
 - lib/redshift-connector/s3_data_file_bundle.rb
+- lib/redshift-connector/url_data_file.rb
+- lib/redshift-connector/url_data_file_bundle.rb
 - lib/redshift-connector/version.rb
 - test/all.rb
 - test/config.rb