redshift-connector 5.3.1 → 5.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fd434428d24b3d41029d89fbbf48c8415416fc46
-  data.tar.gz: a81a921c7b1bdde79851684137cf43d3bc5f26e7
+  metadata.gz: 92e4cbc013cea23b354fcd85bc91e320efd234ae
+  data.tar.gz: b5bbe711725abdf11ba8f138138fee8422fd6822
 SHA512:
-  metadata.gz: c2dfd23d53f217190dcc85d99128700a40432eec27f89cf7a4100fbd61df81d3330ea7ddcc25fdd2164afdda3aeb736b5dd416f9fece9bc8c3fff905063925eb
-  data.tar.gz: 6beed3edae728c20b33fbedaa858b7c27d49439cd86732ecf78d556cd3836cc2885ef83a60d0526935c4a2f05fb9908693ac76995b4a4096605db668953f0101
+  metadata.gz: 25731aa8fa269ded37717adb36e4d8a6ce81de171c36e6968e3bb1f29aadad93abfe068b6e3509ce70ff820f6f9b196c2ebc66373feded7bea99f945b0d645f7
+  data.tar.gz: ea26a509da98e7e6cf1fe7247b430db216687f216889d8181a1b5ea353de5d3a77a5bbf4de563f5ea4402c1782f8c57751b8cff167a9e1f7eed8e65cad955fcd

data/lib/redshift-connector/abstract_data_file.rb

@@ -0,0 +1,24 @@
+require 'zlib'
+
+module RedshiftConnector
+  class AbstractDataFile
+    def each_row(&block)
+      f = if gzipped_object?
+            Zlib::GzipReader.new(content)
+          else
+            content
+          end
+      @reader_class.new(f).each(&block)
+    ensure
+      content.close
+    end
+
+    def data_object?
+      @reader_class.data_object?(key)
+    end
+
+    def gzipped_object?
+      File.extname(key) == '.gz'
+    end
+  end
+end

data/lib/redshift-connector/abstract_data_file_bundle.rb

@@ -0,0 +1,22 @@
+module RedshiftConnector
+  class AbstractDataFileBundle
+    def each_row(&block)
+      each_object do |obj|
+        obj.each_row(&block)
+      end
+    end
+
+    alias each each_row
+
+    def each_object(&block)
+      all_data_objects.each do |obj|
+        @logger.info "processing s3 object: #{obj.key}"
+        yield obj
+      end
+    end
+
+    def all_data_objects
+      data_files.select {|obj| obj.data_object? }
+    end
+  end
+end

data/lib/redshift-connector/reader/csv.rb

@@ -8,8 +8,8 @@ module RedshiftConnector
   class Reader::CSV < Reader::Abstract
     declare_reader :csv
 
-    def self.data_object?(obj)
-      /\.csv(?:\.|\z)/ =~ File.basename(obj.key)
+    def self.data_object?(key)
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
     end
 
     def initialize(f)

data/lib/redshift-connector/reader/redshift_csv.rb

@@ -7,8 +7,8 @@ module RedshiftConnector
   class Reader::RedshiftCSV < Reader::Abstract
     declare_reader :redshift_csv
 
-    def self.data_object?(obj)
-      /\.csv(?:\.|\z)/ =~ File.basename(obj.key)
+    def self.data_object?(key)
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
     end
 
     # f :: IO

data/lib/redshift-connector/reader/tsv.rb

@@ -7,8 +7,8 @@ module RedshiftConnector
   class Reader::TSV < Reader::Abstract
     declare_reader :tsv
 
-    def self.data_object?(obj)
-      /\.tsv(?:\.|\z)/ =~ File.basename(obj.key)
+    def self.data_object?(key)
+      /\.tsv(?:\.|\z)/ =~ File.basename(key)
     end
 
     def initialize(f)

data/lib/redshift-connector/s3_data_file.rb

@@ -1,7 +1,7 @@
-require 'zlib'
+require 'redshift-connector/abstract_data_file'
 
 module RedshiftConnector
-  class S3DataFile
+  class S3DataFile < AbstractDataFile
     def initialize(object, reader_class:)
       @object = object
       @reader_class = reader_class
@@ -11,24 +11,10 @@ module RedshiftConnector
       @object.key
     end
 
-    def each_row(&block)
-      response = @object.get
-      f = if gzipped_object?
-        Zlib::GzipReader.new(response.body)
-      else
-        response.body
-      end
-      @reader_class.new(f).each(&block)
-    ensure
-      response.body.close if response
+    def content
+      @object.get.body
     end
 
-    def data_object?
-      @reader_class.data_object?(@object)
-    end
-
-    def gzipped_object?
-      File.extname(@object.key) == '.gz'
-    end
+    delegate :presigned_url, to: :@object
   end
 end

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -2,10 +2,11 @@ require 'redshift-connector/s3_bucket'
 require 'redshift-connector/s3_data_file'
 require 'redshift-connector/reader'
 require 'redshift-connector/logger'
+require 'redshift-connector/abstract_data_file_bundle'
 require 'aws-sdk'
 
 module RedshiftConnector
-  class S3DataFileBundle
+  class S3DataFileBundle < AbstractDataFileBundle
     def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
       real_prefix = "#{bucket.prefix}/#{prefix}"
       new(bucket, real_prefix, format: format, filter: filter, batch_size: batch_size, logger: logger)
@@ -68,25 +69,9 @@ module RedshiftConnector
     end
     private :do_each_batch
 
-    def each_row(&block)
-      each_object do |obj|
-        obj.each_row(&block)
-      end
-    end
-
-    alias each each_row
-
-    def each_object(&block)
-      all_data_objects.each do |obj|
-        @logger.info "processing s3 object: #{obj.key}"
-        yield obj
-      end
-    end
-
-    def all_data_objects
+    def data_files
       @bucket.objects(prefix: @prefix)
         .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }
-        .select {|obj| obj.data_object? }
     end
 
     def clear

data/lib/redshift-connector/url_data_file.rb

@@ -0,0 +1,23 @@
+require 'redshift-connector/abstract_data_file'
+require 'uri'
+require 'zlib'
+require 'open3'
+
+module RedshiftConnector
+  class UrlDataFile < AbstractDataFile
+    def initialize(url, reader_class:)
+      @url = url
+      @reader_class = reader_class
+    end
+
+    def key
+      URI.parse(@url).path
+    end
+
+    def content
+      stdin, stdout, stderr, wait_th = Open3.popen3('curl', @url)
+      stdin.close
+      stdout
+    end
+  end
+end

data/lib/redshift-connector/url_data_file_bundle.rb

@@ -0,0 +1,21 @@
+require 'redshift-connector/reader'
+require 'redshift-connector/logger'
+require 'redshift-connector/abstract_data_file_bundle'
+require 'redshift-connector/url_data_file'
+
+module RedshiftConnector
+  class UrlDataFileBundle < AbstractDataFileBundle
+    def initialize(data_file_urls, format: :redshift_csv, filter: nil, logger: RedshiftConnector.logger)
+      @data_file_urls = data_file_urls
+      @filter = filter || lambda {|*row| row }
+      @logger = logger
+      @reader_class = Reader.get(format)
+    end
+
+    def data_files
+      @data_file_urls.map do |url|
+        UrlDataFile.new(url, reader_class: @reader_class)
+      end
+    end
+  end
+end

data/lib/redshift-connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '5.3.1'
+  VERSION = '5.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 5.3.1
+  version: 5.3.2
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-27 00:00:00.000000000 Z
+date: 2017-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -130,6 +130,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/redshift-connector.rb
+- lib/redshift-connector/abstract_data_file.rb
+- lib/redshift-connector/abstract_data_file_bundle.rb
 - lib/redshift-connector/connector.rb
 - lib/redshift-connector/exporter.rb
 - lib/redshift-connector/importer.rb
@@ -149,6 +151,8 @@ files:
 - lib/redshift-connector/s3_bucket.rb
 - lib/redshift-connector/s3_data_file.rb
 - lib/redshift-connector/s3_data_file_bundle.rb
+- lib/redshift-connector/url_data_file.rb
+- lib/redshift-connector/url_data_file_bundle.rb
 - lib/redshift-connector/version.rb
 - test/all.rb
 - test/config.rb