redshift_connector 8.0.0

data/lib/redshift_connector/logger.rb

@@ -0,0 +1,20 @@
+module RedshiftConnector
+  @logger = nil
+
+  def RedshiftConnector.logger
+    # Defer to access Rails
+    @logger || Rails.logger
+  end
+
+  def RedshiftConnector.logger=(logger)
+    @logger = logger
+  end
+
+  class NullLogger
+    def noop(*args) end
+    alias error noop
+    alias warn noop
+    alias info noop
+    alias debug noop
+  end
+end

data/lib/redshift_connector/query.rb

@@ -0,0 +1,95 @@
+module RedshiftConnector
+  class DeltaQuery
+    def initialize(schema:, table:, columns:, condition: nil)
+      @schema = schema
+      @table = table
+      @columns = columns
+      @condition = condition
+    end
+
+    def table_spec
+      "#{@schema}.#{@table}"
+    end
+
+    def description
+      "#{table_spec} (#{@columns.join(', ')}) where (#{@condition})"
+    end
+
+    def to_sql
+      "select #{@columns.map {|c| %Q("#{c}") }.join(', ')}" \
+          + " from #{table_spec}" \
+          + (@condition ? " where #{@condition}" : '')
+    end
+  end
+
+  class SelectAllQuery
+    def initialize(schema:, table:, columns:)
+      @schema = schema
+      @table = table
+      @columns = columns
+    end
+
+    def table_spec
+      "#{@schema}.#{@table}"
+    end
+
+    def description
+      "#{table_spec} (#{@columns.join(', ')})"
+    end
+
+    def to_sql
+      "select #{@columns.map {|c| %Q("#{c}") }.join(', ')}" \
+          + " from #{table_spec}"
+    end
+  end
+
+  class UnloadQuery
+    def UnloadQuery.wrap(query:, bundle:, enable_sort: false)
+      new(query: ArbitraryQuery.new(query), bundle: bundle, enable_sort: enable_sort)
+    end
+
+    def initialize(query:, bundle:, enable_sort: false)
+      @query = query
+      @bundle = bundle
+      @enable_sort = enable_sort
+    end
+
+    def table_spec
+      @query.table_spec
+    end
+
+    def description
+      @query.description
+    end
+
+    def to_sql
+      <<-EndSQL.gsub(/^\s+/, '')
+        unload ('#{escape_query(@query.to_sql)}')
+        to '#{@bundle.url}'
+        credentials '#{@bundle.credential_string}'
+        gzip
+        allowoverwrite
+        parallel #{@enable_sort ? 'off' : 'on'}
+        delimiter ',' escape addquotes
+      EndSQL
+    end
+
+    def escape_query(query)
+      query.gsub("'", "\\\\'")
+    end
+  end
+
+  class ArbitraryQuery
+    def initialize(query)
+      @query = query
+    end
+
+    def description
+      @query
+    end
+
+    def to_sql
+      @query
+    end
+  end
+end

data/lib/redshift_connector/reader.rb

@@ -0,0 +1,18 @@
+# create module
+module RedshiftConnector
+  module Reader
+  end
+end
+
+require 'redshift_connector/reader/redshift_csv'
+require 'redshift_connector/reader/csv'
+require 'redshift_connector/reader/tsv'
+require 'redshift_connector/reader/exception'
+
+module RedshiftConnector
+  module Reader
+    def Reader.get(id)
+      Abstract.get_reader_class(id)
+    end
+  end
+end

data/lib/redshift_connector/reader/abstract.rb

@@ -0,0 +1,18 @@
+module RedshiftConnector
+  class Reader::Abstract
+    READER_CLASSES = {}   # {Symbol => Class}
+
+    def self.declare_reader(id)
+      READER_CLASSES[id.to_sym] = self
+    end
+
+    def self.get_reader_class(id)
+      READER_CLASSES[id.to_sym] or
+          raise ArgumentError, "unknown data file reader type: #{id.inspect}"
+    end
+  end
+
+  def self.get_reader_class(id)
+    Reader::Abstract.get_reader_class(id)
+  end
+end

data/lib/redshift_connector/reader/csv.rb

@@ -0,0 +1,24 @@
+require 'redshift_connector/reader/abstract'
+require 'redshift_connector/reader/exception'
+require 'csv'
+
+module RedshiftConnector
+  # Parses (standard) CSV files.
+  # For UNLOAD-generated CSV, use RedshiftCSV class.
+  class Reader::CSV < Reader::Abstract
+    declare_reader :csv
+
+    def self.data_object?(key)
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def initialize(f)
+      @f = f
+    end
+
+    def each(&block)
+      csv = CSV.new(@f)
+      csv.each(&block)
+    end
+  end
+end

data/lib/redshift_connector/reader/exception.rb

@@ -0,0 +1,3 @@
+module RedshiftConnector
+  class Reader::MalformedCSVException < StandardError; end
+end

data/lib/redshift_connector/reader/redshift_csv.rb

@@ -0,0 +1,25 @@
+require 'redshift_connector/reader/abstract'
+require 'redshift_connector/reader/exception'
+require 'redshift_csv_file'
+require 'forwardable'
+
+module RedshiftConnector
+  # Reads CSV file generated by Redshift UNLOAD statement (with option ADDQUOTES ESCAPE).
+  # UNLOAD escapes data by '\' (backslash character), we cannot use standard CSV class.
+  class Reader::RedshiftCSV < Reader::Abstract
+    declare_reader :redshift_csv
+
+    def self.data_object?(key)
+      /\.csv(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    # f :: IO
+    def initialize(f)
+      @f = f
+      @csv = RedshiftCsvFile.new(@f)
+    end
+
+    extend Forwardable
+    def_delegators '@csv', :each, :each_row, :read_row
+  end
+end

data/lib/redshift_connector/reader/tsv.rb

@@ -0,0 +1,24 @@
+require 'redshift_connector/reader/abstract'
+require 'redshift_connector/reader/exception'
+require 'csv'
+
+module RedshiftConnector
+  # Parses TSV (Tab Separated Format) files.
+  class Reader::TSV < Reader::Abstract
+    declare_reader :tsv
+
+    def self.data_object?(key)
+      /\.tsv(?:\.|\z)/ =~ File.basename(key)
+    end
+
+    def initialize(f)
+      @f = f
+    end
+
+    def each(&block)
+      @f.each_line do |line|
+        yield line.chomp.split("\t", -1)
+      end
+    end
+  end
+end

data/lib/redshift_connector/s3_bucket.rb

@@ -0,0 +1,76 @@
+require 'aws-sdk-s3'
+
+module RedshiftConnector
+  class S3Bucket
+    @buckets = {}
+    @default = nil
+
+    def S3Bucket.add(name, default: false, **params)
+      instance = new(**params)
+      @buckets[name.to_s] = instance
+      if !@default or default
+        @default = instance
+      end
+    end
+
+    def S3Bucket.default
+      @default or raise ArgumentError, "no default S3 bucket configured"
+    end
+
+    def S3Bucket.get(name)
+      @buckets[name.to_s] or raise ArgumentError, "no such S3 bucket configured: #{name.inspect}"
+    end
+
+    def initialize(region: nil, bucket:, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil)
+      @region = region
+      @name = bucket
+      @prefix = prefix
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @iam_role = iam_role
+    end
+
+    attr_reader :name
+    attr_reader :prefix
+
+    def url
+      "s3://#{@bucket.name}/#{@prefix}/"
+    end
+
+    def client
+      @client ||= begin
+        args = { region: @region, access_key_id: @access_key_id, secret_access_key: @secret_access_key }.reject {|k, v| v.nil? }
+        Aws::S3::Client.new(**args)
+      end
+    end
+
+    def bucket
+      @bucket ||= begin
+        resource = Aws::S3::Resource.new(client: client)
+        resource.bucket(@name)
+      end
+    end
+
+    def object(key)
+      bucket.object(key)
+    end
+
+    def objects(prefix:)
+      bucket.objects(prefix: prefix)
+    end
+
+    def delete_objects(keys)
+      bucket.delete_objects(delete: {objects: keys.map {|k| {key: k} }})
+    end
+
+    def credential_string
+      if @iam_role
+        "aws_iam_role=#{@iam_role}"
+      elsif @access_key_id
+        "aws_access_key_id=#{@access_key_id};aws_secret_access_key=#{@secret_access_key}"
+      else
+        raise ArgumentError, "no credential given for Redshift S3 access"
+      end
+    end
+  end
+end

data/lib/redshift_connector/s3_data_file.rb

@@ -0,0 +1,20 @@
+require 'redshift_connector/data_file'
+
+module RedshiftConnector
+  class S3DataFile < DataFile
+    def initialize(object, reader_class:)
+      super reader_class: reader_class
+      @object = object
+    end
+
+    def key
+      @object.key
+    end
+
+    def open
+      @object.get.body
+    end
+
+    delegate :presigned_url, to: :@object
+  end
+end

data/lib/redshift_connector/s3_data_file_bundle.rb

@@ -0,0 +1,68 @@
+require 'redshift_connector/s3_bucket'
+require 'redshift_connector/s3_data_file'
+require 'redshift_connector/reader'
+require 'redshift_connector/logger'
+require 'aws-sdk-s3'
+
+module RedshiftConnector
+  class S3DataFileBundle
+    def self.for_params(params)
+      unless params.txn_id
+        raise ArgumentError, "cannot create bundle: missing txn_id"
+      end
+      s3bucket = params.bucket ? S3Bucket.get(params.bucket) : S3Bucket.default
+      for_table(
+        bucket: s3bucket,
+        schema: params.schema,
+        table: params.table,
+        txn_id: params.txn_id,
+        logger: params.logger
+      )
+    end
+
+    def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, logger: RedshiftConnector.logger)
+      real_prefix = "#{bucket.prefix}/#{prefix}"
+      new(bucket, real_prefix, format: format, logger: logger)
+    end
+
+    def self.for_table(bucket: S3Bucket.default, schema:, table:, txn_id:, logger: RedshiftConnector.logger)
+      prefix = "#{bucket.prefix}/#{schema}_export/#{table}/#{txn_id}/#{table}.csv."
+      new(bucket, prefix, format: :redshift_csv, logger: logger)
+    end
+
+    def initialize(bucket, prefix, format: :csv, logger: RedshiftConnector.logger)
+      @bucket = bucket
+      @prefix = prefix
+      @format = format
+      @logger = logger
+      @reader_class = Reader.get(format)
+      logger.info "reader: #{@reader_class}"
+    end
+
+    attr_reader :bucket
+    attr_reader :prefix
+    attr_reader :logger
+
+    def url
+      "s3://#{@bucket.name}/#{@prefix}"
+    end
+
+    def credential_string
+      @bucket.credential_string
+    end
+
+    def data_files
+      @bucket.objects(prefix: @prefix)
+        .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }
+    end
+
+    def clear
+      pref = File.dirname(@prefix) + '/'
+      keys = @bucket.objects(prefix: pref).map(&:key)
+      unless keys.empty?
+        logger.info "DELETE #{pref}*"
+        @bucket.delete_objects(keys)
+      end
+    end
+  end
+end

data/lib/redshift_connector/version.rb

@@ -0,0 +1,3 @@
+module RedshiftConnector
+  VERSION = '8.0.0'
+end

data/redshift_connector.gemspec

@@ -0,0 +1,27 @@
+require_relative 'lib/redshift_connector/version'
+
+Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.name = 'redshift_connector'
+  s.version = RedshiftConnector::VERSION
+  s.summary = 'Redshift bulk data connector'
+  s.description = 'redshift_connector is a bulk data connector for Rails (ActiveRecord).'
+  s.license = 'MIT'
+
+  s.author = ['Minero Aoki']
+  s.email = 'aamine@loveruby.net'
+  s.homepage = 'https://github.com/bricolages/redshift_connector'
+
+  s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  s.require_path = 'lib'
+
+  s.required_ruby_version = '>= 2.1.0'
+  s.add_dependency 'activerecord'
+  s.add_dependency 'activerecord-redshift'
+  s.add_dependency 'pg', '~> 0.18.0'
+  s.add_dependency 'activerecord-import'
+  s.add_dependency 'redshift_csv_file', '~> 1.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1.0'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,190 @@
+--- !ruby/object:Gem::Specification
+name: redshift_connector
+version: !ruby/object:Gem::Version
+  version: 8.0.0
+platform: ruby
+authors:
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-redshift
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redshift_csv_file
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: redshift_connector is a bulk data connector for Rails (ActiveRecord).
+email: aamine@loveruby.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- RELEASE.md
+- Rakefile
+- lib/redshift_connector.rb
+- lib/redshift_connector/active_record_data_source.rb
+- lib/redshift_connector/active_record_exporter.rb
+- lib/redshift_connector/connector.rb
+- lib/redshift_connector/data_file.rb
+- lib/redshift_connector/data_file_bundle_params.rb
+- lib/redshift_connector/data_file_bundle_reader.rb
+- lib/redshift_connector/exception.rb
+- lib/redshift_connector/exporter.rb
+- lib/redshift_connector/exporter_builder.rb
+- lib/redshift_connector/immediate_exporter.rb
+- lib/redshift_connector/importer.rb
+- lib/redshift_connector/importer/activerecord-import.rb
+- lib/redshift_connector/importer/insert_delta.rb
+- lib/redshift_connector/importer/rebuild_rename.rb
+- lib/redshift_connector/importer/rebuild_truncate.rb
+- lib/redshift_connector/importer/upsert.rb
+- lib/redshift_connector/logger.rb
+- lib/redshift_connector/query.rb
+- lib/redshift_connector/reader.rb
+- lib/redshift_connector/reader/abstract.rb
+- lib/redshift_connector/reader/csv.rb
+- lib/redshift_connector/reader/exception.rb
+- lib/redshift_connector/reader/redshift_csv.rb
+- lib/redshift_connector/reader/tsv.rb
+- lib/redshift_connector/s3_bucket.rb
+- lib/redshift_connector/s3_data_file.rb
+- lib/redshift_connector/s3_data_file_bundle.rb
+- lib/redshift_connector/version.rb
+- redshift_connector.gemspec
+homepage: https://github.com/bricolages/redshift_connector
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Redshift bulk data connector
+test_files: []