redshift-connector 4.5.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb3953798104bd92d6d856b8fd3b2d835ba9d54a
-  data.tar.gz: e2f51e74ec4969e1187ce35e457a2af357bda1bf
+  metadata.gz: b3d56a36e8dba44e03271c37ecedd89871e5f0d2
+  data.tar.gz: 3237a2be89f2b86c524d99ce0064773a3807b41e
 SHA512:
-  metadata.gz: cec111e3ca92096a3a0071b7a76a36b3c259213ab03aecb5f75ca1e3b2cc41fd1b4e140384a5a482944d7f02819cfe8da5010ae0bd8c9e9b0619001e09aa3542
-  data.tar.gz: ac5dbcc924a7ffb5ee7387d4cf3f56a78371c10a870d4b3b3215502d19f256daff0bcc970ed2d7d946b6ecd62f8305a4ab55ecd6adbd0e4b1896cbf42b671067
+  metadata.gz: 0de405e11f29b3efad2e9161436396cd436396ceb525c284d505a42abb3893da882b6d3c1767b88589736776bc26cc8442be350ed77e40ca26e5ed73a1da096c
+  data.tar.gz: 6be60fc0ac03acb59f990dfb605abba2f49711ec32441d11d366ffa322b54835b9625deb3224429ff88f7528f2433a777028283f49fc81132d04f6a79d417cab

data/README.md

@@ -8,32 +8,3 @@ Add following block to your Gemfile and bundle.
 ```
 gem 'redshift-connector'
 ```
-Add config/initializers/redshift-connector.rb like following:
-```
-module RedshiftConnector
-  Exporter.default_data_source = Any_ActiveRecord_Class_Bound_To_Redshift
-
-  S3Bucket.add('primary', default: true,
-    region: 'YOUR_AWS_REGION_NAME',
-    bucket: 'YOUR_BUCKET_NAME',
-    prefix: 'YOUR_PREFIX',
-    iam_role: 'arn:aws:iam::XXXXXXXXXXXX:role/RedshiftReadOnly'
-    # For explicit S3 access, use following:
-    # aws_access_key_id: 'XXXXXXXXXXXXX',
-    # aws_secret_access_key: 'XXXXXXXXXXXXX'
-  )
-end
-```
-
-## Usage
-
-### Fetching rows
-
-```
-RedshiftConnector.foreach(schema: 'app_mst', table: 'shops', query: 'select id, name from app_mst.shops') do |id, name|
-  p [id, name]
-end
-```
-`schema` and `table` is the source table name (written in the query).
-This method executes Redshift UNLOAD statement with given query and
-unload result to the intermediate S3, then read contents.

data/lib/redshift-connector/connector.rb

@@ -116,18 +116,26 @@ module RedshiftConnector
       not ENV['IMPORT_ONLY']
     end
 
+    def export_forced?
+      !! (ENV['EXPORT_ONLY'] or ENV['FORCE'])
+    end
+
     def import_enabled?
       not ENV['EXPORT_ONLY']
     end
 
     def execute
-      export if export_enabled?
+      export(forced: export_forced?) if export_enabled?
       import if import_enabled?
     end
 
-    def export
+    def export(forced: false)
       @logger.info "==== export task =================================================="
-      @exporter.execute
+      if not forced and @exporter.completed?
+        @logger.info "export task is already executed; skip"
+      else
+        @exporter.execute
+      end
     end
 
     def import

data/lib/redshift-connector/exporter.rb

@@ -41,7 +41,6 @@ module RedshiftConnector
         query:,
         txn_id: "#{Time.now.strftime('%Y%m%d_%H%M%S')}_#{$$}",
         filter: nil,
-        enable_sort: false,
         logger: RedshiftConnector.logger,
         quiet: false
     )
@@ -57,7 +56,7 @@ module RedshiftConnector
       )
       exporter = Exporter.new(
         ds: ds,
-        query: UnloadQuery.wrap(query: query, bundle: bundle, enable_sort: enable_sort),
+        query: UnloadQuery.wrap(query: query, bundle: bundle),
         bundle: bundle,
         logger: logger
       )
@@ -75,6 +74,19 @@ module RedshiftConnector
     attr_reader :bundle
     attr_reader :logger
 
+    def completed?
+      @bundle.bucket.object(flag_object_key).exists?
+    end
+
+    def create_flag_object
+      @logger.info "TOUCH #{flag_object_key}"
+      @bundle.bucket.object(flag_object_key).put(body: "OK")
+    end
+
+    def flag_object_key
+      "#{File.dirname(@bundle.prefix)}/00completed"
+    end
+
     def execute
       @bundle.clear
       @logger.info "EXPORT #{@query.description} -> #{@bundle.url}*"
@@ -83,6 +95,7 @@ module RedshiftConnector
         @logger.info "[SQL/Redshift] #{batch_job_label}#{stmt.strip}"
         conn.execute(batch_job_label + stmt)
       end
+      create_flag_object
     end
 
     def batch_job_label

data/lib/redshift-connector/query.rb

@@ -44,14 +44,13 @@ module RedshiftConnector
   end
 
   class UnloadQuery
-    def UnloadQuery.wrap(query:, bundle:, enable_sort: false)
-      new(query: ArbitraryQuery.new(query), bundle: bundle, enable_sort: enable_sort)
+    def UnloadQuery.wrap(query:, bundle:)
+      new(query: ArbitraryQuery.new(query), bundle: bundle)
     end
 
-    def initialize(query:, bundle:, enable_sort: false)
+    def initialize(query:, bundle:)
       @query = query
       @bundle = bundle
-      @enable_sort = enable_sort
     end
 
     def table_spec
@@ -69,7 +68,6 @@ module RedshiftConnector
         credentials '#{@bundle.credential_string}'
         gzip
         allowoverwrite
-        parallel #{@enable_sort ? 'off' : 'on'}
         delimiter ',' escape addquotes
       EndSQL
     end

data/lib/redshift-connector/reader.rb

@@ -0,0 +1,18 @@
+# create module
+module RedshiftConnector
+  module Reader
+  end
+end
+
+require 'redshift-connector/reader/redshift_csv'
+require 'redshift-connector/reader/csv'
+require 'redshift-connector/reader/tsv'
+require 'redshift-connector/reader/exception'
+
+module RedshiftConnector
+  module Reader
+    def Reader.get(id)
+      Abstract.get_reader_class(id)
+    end
+  end
+end

data/lib/redshift-connector/reader/abstract.rb

@@ -0,0 +1,18 @@
+module RedshiftConnector
+  class Reader::Abstract
+    READER_CLASSES = {}   # {Symbol => Class}
+
+    def self.declare_reader(id)
+      READER_CLASSES[id.to_sym] = self
+    end
+
+    def self.get_reader_class(id)
+      READER_CLASSES[id.to_sym] or
+          raise ArgumentError, "unknown data file reader type: #{id.inspect}"
+    end
+  end
+
+  def self.get_reader_class(id)
+    Reader::Abstract.get_reader_class(id)
+  end
+end

data/lib/redshift-connector/reader/csv.rb

@@ -0,0 +1,24 @@
+require 'redshift-connector/reader/abstract'
+require 'redshift-connector/reader/exception'
+require 'csv'
+
+module RedshiftConnector
+  # Parses (standard) CSV files.
+  # For UNLOAD-generated CSV, use RedshiftCSV class.
+  class Reader::CSV < Reader::Abstract
+    declare_reader :csv
+
+    def self.data_object?(obj)
+      /\.csv(?:\.|\z)/ =~ File.basename(obj.key)
+    end
+
+    def initialize(f)
+      @f = f
+    end
+
+    def each(&block)
+      csv = CSV.new(@f)
+      csv.each(&block)
+    end
+  end
+end

data/lib/redshift-connector/reader/exception.rb

@@ -0,0 +1,3 @@
+module RedshiftConnector
+  class Reader::MalformedCSVException < StandardError; end
+end

data/lib/redshift-connector/reader/redshift_csv.rb

@@ -0,0 +1,54 @@
+require 'redshift-connector/reader/abstract'
+require 'redshift-connector/reader/exception'
+
+module RedshiftConnector
+  # Reads CSV file generated by Redshift UNLOAD statement (with option ADDQUOTES ESCAPE).
+  # UNLOAD escapes data by '\' (backslash character), we cannot use standard CSV class.
+  class Reader::RedshiftCSV < Reader::Abstract
+    declare_reader :redshift_csv
+
+    def self.data_object?(obj)
+      /\.csv(?:\.|\z)/ =~ File.basename(obj.key)
+    end
+
+    # f :: IO
+    def initialize(f)
+      @f = f
+    end
+
+    def each
+      # We can use simple #each_line to read single row
+      # because line terminators are always escaped by UNLOAD.
+      @f.each_line do |line|
+        yield parse_row(line, @f.lineno)
+      end
+    end
+
+    def parse_row(line, lineno = nil)
+      row = []
+      s = StringScanner.new(line)
+      s.skip(/\s+/)
+      until s.eos?
+        col = s.scan(/"(?:\\.|[^"\\]+)*"/) or raise MalformedCSVException, "CSV parse error at line #{lineno}"
+        row.push unescape_column(col)
+        s.skip(/\s*/)    # skip line terminator on line ends
+        s.skip(/,\s*/)
+      end
+      row
+    end
+
+    UNESCAPE_MAP = {
+      '\\"' => '"',
+      "\\'" => "'",
+      '\\,' => ',',
+      '\\r' => "\r",
+      '\\n' => "\n",
+      '\\\\' => '\\'
+    }
+
+    def unescape_column(col)
+      charmap = UNESCAPE_MAP
+      col[1...-1].gsub(/\\./) {|s| charmap[s] }
+    end
+  end
+end

data/lib/redshift-connector/reader/tsv.rb

@@ -0,0 +1,24 @@
+require 'redshift-connector/reader/abstract'
+require 'redshift-connector/reader/exception'
+require 'csv'
+
+module RedshiftConnector
+  # Parses TSV (Tab Separated Format) files.
+  class Reader::TSV < Reader::Abstract
+    declare_reader :tsv
+
+    def self.data_object?(obj)
+      /\.tsv(?:\.|\z)/ =~ File.basename(obj.key)
+    end
+
+    def initialize(f)
+      @f = f
+    end
+
+    def each(&block)
+      @f.each_line do |line|
+        yield line.chomp.split("\t", -1)
+      end
+    end
+  end
+end

data/lib/redshift-connector/s3_bucket.rb

@@ -21,8 +21,7 @@ module RedshiftConnector
       @buckets[name.to_s] or raise ArgumentError, "no such S3 bucket configured: #{name.inspect}"
     end
 
-    def initialize(region: nil, bucket:, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil)
-      @region = region
+    def initialize(bucket:, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil)
       @name = bucket
       @prefix = prefix
       @access_key_id = access_key_id
@@ -38,10 +37,7 @@ module RedshiftConnector
     end
 
     def client
-      @client ||= begin
-        args = { region: @region, access_key_id: @access_key_id, secret_access_key: @secret_access_key }.reject {|k, v| v.nil? }
-        Aws::S3::Client.new(**args)
-      end
+      @client ||= Aws::S3::Client.new(access_key_id: @access_key_id, secret_access_key: @secret_access_key)
     end
 
     def bucket

data/lib/redshift-connector/s3_data_file.rb

@@ -1,7 +1,7 @@
-require 'redshift-connector/data_file'
+require 'zlib'
 
 module RedshiftConnector
-  class S3DataFile < AbstractDataFile
+  class S3DataFile
     def initialize(object, reader_class:)
       @object = object
       @reader_class = reader_class
@@ -11,10 +11,24 @@ module RedshiftConnector
       @object.key
     end
 
-    def content
-      @object.get.body
+    def each_row(&block)
+      response = @object.get
+      f = if gzipped_object?
+        Zlib::GzipReader.new(response.body)
+      else
+        response.body
+      end
+      @reader_class.new(f).each(&block)
+    ensure
+      response.body.close if response
     end
 
-    delegate :presigned_url, to: :@object
+    def data_object?
+      @reader_class.data_object?(@object)
+    end
+
+    def gzipped_object?
+      File.extname(@object.key) == '.gz'
+    end
   end
 end

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -1,11 +1,11 @@
 require 'redshift-connector/s3_bucket'
 require 'redshift-connector/s3_data_file'
+require 'redshift-connector/reader'
 require 'redshift-connector/logger'
-require 'redshift-connector/data_file'
 require 'aws-sdk'
 
 module RedshiftConnector
-  class S3DataFileBundle < AbstractDataFileBundle
+  class S3DataFileBundle
     def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
       real_prefix = "#{bucket.prefix}/#{prefix}"
       new(bucket, real_prefix, format: format, filter: filter, batch_size: batch_size, logger: logger)
@@ -68,9 +68,25 @@ module RedshiftConnector
     end
     private :do_each_batch
 
-    def data_files
+    def each_row(&block)
+      each_object do |obj|
+        obj.each_row(&block)
+      end
+    end
+
+    alias each each_row
+
+    def each_object(&block)
+      all_data_objects.each do |obj|
+        @logger.info "processing s3 object: #{obj.key}"
+        yield obj
+      end
+    end
+
+    def all_data_objects
       @bucket.objects(prefix: @prefix)
         .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }
+        .select {|obj| obj.data_object? }
     end
 
     def clear

data/lib/redshift-connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '4.5.0'
+  VERSION = '5.0.0'
 end

data/test/foreach.rb

@@ -1,5 +1,5 @@
 require_relative 'helper'
 
-RedshiftConnector.foreach(schema: 'tabemiru', table: 'items', query: 'select id from tabemiru.items where id < 50 order by 1', enable_sort: true) do |row|
+RedshiftConnector.foreach(schema: 'tabemiru', table: 'items', query: 'select * from tabemiru.items where id < 10') do |row|
   p row
 end

data/test/reader/test_redshift_csv.rb

@@ -1,5 +1,5 @@
 require 'test/unit'
-require 'redshift-connector/data_file'
+require 'redshift-connector/reader'
 
 module RedshiftConnector
   module Reader

data/test/test_reader.rb

@@ -1,4 +1,5 @@
 require 'test/unit'
+require 'redshift-connector/reader'
 
 module RedshiftConnector
   class TestReader < Test::Unit::TestCase

metadata

@@ -1,31 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 4.5.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-12 00:00:00.000000000 Z
+date: 2017-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: activerecord4-redshift-adapter
+  name: activerecord5-redshift-adapter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: redshift-connector-data_file
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +140,12 @@ files:
 - lib/redshift-connector/importer/upsert.rb
 - lib/redshift-connector/logger.rb
 - lib/redshift-connector/query.rb
+- lib/redshift-connector/reader.rb
+- lib/redshift-connector/reader/abstract.rb
+- lib/redshift-connector/reader/csv.rb
+- lib/redshift-connector/reader/exception.rb
+- lib/redshift-connector/reader/redshift_csv.rb
+- lib/redshift-connector/reader/tsv.rb
 - lib/redshift-connector/s3_bucket.rb
 - lib/redshift-connector/s3_data_file.rb
 - lib/redshift-connector/s3_data_file_bundle.rb