redshift-connector 6.0.0 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f31b6eadc1161cb752be8f735f6ab3cdf2e87d32
-  data.tar.gz: edb474edc4bd93ee6ab5efc26d9b9770f05bd109
+  metadata.gz: bc258b0cae8475c5c78a9e37ebb80e16d186245d
+  data.tar.gz: 9c8c846b8bfb89714986146b019aa063bcfee9af
 SHA512:
-  metadata.gz: 6d8c0229f3bc0a7beae47859ac7aee5b0df82d82e838ad33803ab9d9e2321e660cd6befd74fee7e800f9394efff425856f20bc0fb77c69dbac407315a65d8333
-  data.tar.gz: c7514f15467cf56f6c3b60c0670a4f338645d0b05b96e418500aa9172b696a3f22c3277d498a4e309a779c2ab3202cd4f5a8dd2ad9fcda3e1527390a6a36b74b
+  metadata.gz: 8e9001439475fc25dfbc6c0eda67306044dccb7c5eeda5309fa2995df4adeaa5b028e3a177f33ca938b99d150ea10a8e48206479b0532f26f2eb17b59b5bf791
+  data.tar.gz: 1d83a4852630ab542bb14cc4fac376a08f549aec0d4225d92f65852f5af290c06d3c44ba31660991370707d9763322ad9d01daba9ff0a1dad352f88d4c0db6ff

data/lib/redshift-connector.rb

@@ -1,31 +1 @@
-module RedshiftConnector
-end
-
-require 'redshift-connector/connector'
-require 'redshift-connector/exporter'
-require 'redshift-connector/importer'
-require 'redshift-connector/s3_bucket'
-require 'redshift-connector/s3_data_file_bundle'
-require 'redshift-connector/version'
-
-module RedshiftConnector
-  def RedshiftConnector.transport_delta(**params)
-    Connector.transport_delta(**params)
-  end
-
-  def RedshiftConnector.transport_all(**params)
-    Connector.transport_all(**params)
-  end
-
-  def RedshiftConnector.transport_delta_from_s3(**params)
-    Importer.transport_delta_from_s3(**params)
-  end
-
-  def RedshiftConnector.transport_all_from_s3(**params)
-    Importer.transport_all_from_s3(**params)
-  end
-
-  def RedshiftConnector.foreach(**params, &block)
-    Exporter.foreach(**params, &block)
-  end
-end
+require 'redshift_connector'

data/lib/redshift_connector.rb

@@ -0,0 +1,34 @@
+module RedshiftConnector
+end
+
+require 'redshift_connector/connector'
+require 'redshift_connector/exporter'
+require 'redshift_connector/active_record_data_source'
+require 'redshift_connector/active_record_exporter'
+require 'redshift_connector/importer'
+require 'redshift_connector/s3_bucket'
+require 'redshift_connector/s3_data_file_bundle'
+require 'redshift_connector/exception'
+require 'redshift_connector/version'
+
+module RedshiftConnector
+  def RedshiftConnector.transport_delta(**params)
+    Connector.transport_delta(**params)
+  end
+
+  def RedshiftConnector.transport_all(**params)
+    Connector.transport_all(**params)
+  end
+
+  def RedshiftConnector.transport_delta_from_s3(**params)
+    Importer.transport_delta_from_s3(**params)
+  end
+
+  def RedshiftConnector.transport_all_from_s3(**params)
+    Importer.transport_all_from_s3(**params)
+  end
+
+  def RedshiftConnector.foreach(**params, &block)
+    Exporter.foreach(**params, &block)
+  end
+end

data/lib/redshift_connector/active_record_data_source.rb

@@ -0,0 +1,23 @@
+require 'redshift_connector/exporter_builder'
+
+module RedshiftConnector
+  class ActiveRecordDataSource
+    def ActiveRecordDataSource.for_dao(dao)
+      new(dao)
+    end
+
+    def initialize(dao)
+      @dao = dao
+    end
+
+    def exporter_builder
+      ExporterBuilder.new(ds: self, exporter_class: ActiveRecordExporter)
+    end
+
+    def execute_query(query_str)
+      @dao.connection_pool.with_connection {|conn|
+        conn.execute(query_str)
+      }
+    end
+  end
+end

data/lib/redshift_connector/active_record_exporter.rb

@@ -0,0 +1,47 @@
+require 'redshift_connector/s3_data_file_bundle'
+require 'redshift_connector/query'
+require 'redshift_connector/logger'
+
+module RedshiftConnector
+  class ActiveRecordExporter
+    def initialize(ds:, query:, bundle_params:, enable_sort: false, logger: RedshiftConnector.logger)
+      @ds = ds
+      @query = query
+      @bundle_params = bundle_params
+      @enable_sort = enable_sort
+      @logger = logger
+
+      @bundle = S3DataFileBundle.for_params(bundle_params)
+    end
+
+    attr_reader :query
+    attr_reader :bundle_params
+    attr_reader :bundle
+    attr_reader :logger
+
+    def execute
+      @bundle.clear
+      unload_query = UnloadQuery.new(query: @query, bundle: @bundle, enable_sort: @enable_sort)
+      @logger.info "EXPORT #{unload_query.description} -> #{@bundle.url}*"
+      stmt = unload_query.to_sql
+      @logger.info "[SQL/Redshift] #{batch_job_label}#{stmt.strip}"
+      @ds.execute_query(batch_job_label + stmt)
+      @bundle
+    end
+
+    def batch_job_label
+      @batch_job_label ||= begin
+        components = Dir.getwd.split('/')
+        app = if components.last == 'current'
+            # is Capistrano environment
+            components[-2]
+          else
+            components[-1]
+          end
+        batch_file = caller.detect {|c| /redshift_connector|active_record/ !~ c }
+        path = batch_file ? batch_file.split(':').first : '?'
+        "/* Job: #{app}:#{path} */ "
+      end
+    end
+  end
+end

data/lib/{redshift-connector → redshift_connector}/connector.rb

@@ -1,7 +1,7 @@
-require 'redshift-connector/exporter'
-require 'redshift-connector/importer'
-require 'redshift-connector/s3_data_file_bundle'
-require 'redshift-connector/logger'
+require 'redshift_connector/exporter'
+require 'redshift_connector/importer'
+require 'redshift_connector/data_file_bundle_params'
+require 'redshift_connector/logger'
 
 module RedshiftConnector
   class Connector
@@ -15,16 +15,16 @@ module RedshiftConnector
         delete_cond: nil,
         upsert_columns: nil,
         bucket: nil,
-        txn_id:, filter:,
+        txn_id: nil,
+        filter:,
         logger: RedshiftConnector.logger,
         quiet: false
     )
       unless src_table and dest_table
         raise ArgumentError, "missing :table, :src_table or :dest_table"
       end
-      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
       logger = NullLogger.new if quiet
-      bundle = S3DataFileBundle.for_table(
+      bundle_params = DataFileBundleParams.new(
         bucket: bucket,
         schema: schema,
         table: src_table,
@@ -33,18 +33,19 @@ module RedshiftConnector
         logger: logger
       )
       exporter = Exporter.for_table_delta(
-        bundle: bundle,
+        bundle_params: bundle_params,
         schema: schema,
         table: src_table,
         columns: columns,
         condition: condition,
         logger: logger
       )
-      importer = Importer.transport_delta_from_bundle(
-        bundle: bundle,
-        table: dest_table, columns: columns,
-        delete_cond: delete_cond, upsert_columns: upsert_columns,
-        logger: logger, quiet: quiet
+      importer = Importer.for_delta_upsert(
+        table: dest_table,
+        columns: columns,
+        delete_cond: delete_cond,
+        upsert_columns: upsert_columns,
+        logger: logger
       )
       new(exporter: exporter, importer: importer, logger: logger)
     end
@@ -57,33 +58,32 @@ module RedshiftConnector
         dest_table: table,
         columns:,
         bucket: nil,
-        txn_id:,
+        txn_id: nil,
         filter:,
         logger: RedshiftConnector.logger,
         quiet: false
     )
-      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
       logger = NullLogger.new if quiet
-      bundle = S3DataFileBundle.for_table(
+      bundle_params = DataFileBundleParams.new(
         bucket: bucket,
         schema: schema,
-        table: table,
+        table: src_table,
         txn_id: txn_id,
         filter: filter,
         logger: logger
       )
       exporter = Exporter.for_table(
-        bundle: bundle,
+        bundle_params: bundle_params,
         schema: schema,
-        table: table,
+        table: src_table,
         columns: columns,
         logger: logger
       )
-      importer = Importer.transport_all_from_bundle(
+      importer = Importer.for_rebuild(
         strategy: strategy,
-        bundle: bundle,
-        table: table, columns: columns,
-        logger: logger, quiet: quiet
+        table: dest_table,
+        columns: columns,
+        logger: logger
       )
       new(exporter: exporter, importer: importer, logger: logger)
     end
@@ -92,6 +92,7 @@ module RedshiftConnector
       @exporter = exporter
       @importer = importer
       @logger = logger
+      @bundle = nil
     end
 
     def export_enabled?
@@ -109,12 +110,12 @@ module RedshiftConnector
 
     def export
       @logger.info "==== export task =================================================="
-      @exporter.execute
+      @bundle = @exporter.execute
     end
 
     def import
       @logger.info "==== import task =================================================="
-      @importer.execute
+      @importer.execute(@bundle)
     end
   end
 end

data/lib/redshift_connector/data_file_bundle_params.rb

@@ -0,0 +1,28 @@
+require 'redshift_connector/logger'
+
+module RedshiftConnector
+  class DataFileBundleParams
+    def initialize(
+      bucket: nil,
+      schema:,
+      table:,
+      txn_id: nil,
+      filter:,
+      logger: RedshiftConnector.logger
+    )
+      @bucket = bucket
+      @schema = schema
+      @table = table
+      @txn_id = txn_id
+      @filter = filter
+      @logger = logger
+    end
+
+    attr_reader :bucket
+    attr_reader :schema
+    attr_reader :table
+    attr_reader :txn_id
+    attr_reader :filter
+    attr_reader :logger
+  end
+end

data/lib/redshift_connector/exception.rb

@@ -0,0 +1,5 @@
+module RedshiftConnector
+  class Error < ::StandardError; end
+  class ExportError < Error; end
+  class ImportError < Error; end
+end

data/lib/redshift_connector/exporter.rb

@@ -0,0 +1,39 @@
+module RedshiftConnector
+  module Exporter
+    @default_data_source = nil
+
+    def Exporter.default_data_source=(ds)
+      @default_data_source = ds
+    end
+
+    def Exporter.default_data_source
+      @default_data_source or raise ArgumentError, "RedshiftConnector::Exporter.default_data_source was not set"
+    end
+
+    def Exporter.builder
+      default_data_source.exporter_builder
+    end
+
+    def Exporter.for_table_delta(**params)
+      builder.build_for_table_delta(**params)
+    end
+
+    def Exporter.for_table(**params)
+      builder.build_for_table(**params)
+    end
+
+    def Exporter.for_query(**params)
+      builder.build_for_query(**params)
+    end
+
+    def Exporter.foreach(**params, &block)
+      exporter = for_query(**params)
+      begin
+        exporter.execute
+        exporter.bundle.each_row(&block)
+      ensure
+        exporter.bundle.clear
+      end
+    end
+  end
+end

data/lib/redshift_connector/exporter_builder.rb

@@ -0,0 +1,51 @@
+require 'redshift_connector/query'
+require 'redshift_connector/logger'
+
+module RedshiftConnector
+  class ExporterBuilder
+    def initialize(ds:, exporter_class:)
+      @ds = ds
+      @exporter_class = exporter_class
+    end
+
+    def build_for_table_delta(schema:, table:, condition:, columns:, bundle_params:, logger: RedshiftConnector.logger)
+      query = DeltaQuery.new(schema: schema, table: table, columns: columns, condition: condition)
+      @exporter_class.new(ds: @ds, query: query, bundle_params: bundle_params, logger: logger)
+    end
+
+    def build_for_table(schema:, table:, columns:, bundle_params:, logger: RedshiftConnector.logger)
+      query = SelectAllQuery.new(schema: schema, table: table, columns: columns)
+      @exporter_class.new(ds: @ds, query: query, bundle_params: bundle_params, logger: logger)
+    end
+
+    def build_for_query(
+        schema:,
+        table:,
+        bucket: nil,
+        query:,
+        txn_id: "#{Time.now.strftime('%Y%m%d_%H%M%S')}_#{$$}",
+        filter: nil,
+        enable_sort: false,
+        logger: RedshiftConnector.logger,
+        quiet: false
+    )
+      logger = NullLogger.new if quiet
+      bundle_params = DataFileBundleParams.new(
+        bucket: bucket,
+        schema: schema,
+        table: table,
+        txn_id: txn_id,
+        filter: filter,
+        logger: logger
+      )
+      @exporter_class.new(
+        ds: @ds,
+        query: ArbitraryQuery.new(query),
+        bundle_params: bundle_params,
+        enable_sort: enable_sort,
+        logger: logger
+      )
+    end
+
+  end
+end

data/lib/redshift_connector/importer.rb

@@ -0,0 +1,58 @@
+# create module
+module RedshiftConnector
+  module Importer
+  end
+end
+
+require 'redshift_connector/importer/upsert'
+require 'redshift_connector/importer/insert_delta'
+require 'redshift_connector/importer/rebuild_rename'
+require 'redshift_connector/importer/rebuild_truncate'
+require 'redshift_connector/logger'
+
+module RedshiftConnector
+  module Importer
+    def Importer.for_delta_upsert(table:, columns:, delete_cond: nil, upsert_columns: nil, logger: RedshiftConnector.logger)
+      if delete_cond and upsert_columns
+        raise ArgumentError, "delete_cond and upsert_columns are exclusive"
+      end
+      importer =
+        if delete_cond
+          Importer::InsertDelta.new(
+            dao: table.classify.constantize,
+            columns: columns,
+            delete_cond: delete_cond,
+            logger: logger
+          )
+        elsif upsert_columns
+          Importer::Upsert.new(
+            dao: table.classify.constantize,
+            columns: columns,
+            upsert_columns: upsert_columns,
+            logger: logger
+          )
+        else
+          raise ArgumentError, "either of delete_cond or upsert_columns is required for delta import"
+        end
+      importer
+    end
+
+    def Importer.for_rebuild(strategy: 'rename', table:, columns:, logger: RedshiftConnector.logger)
+      c = get_rebuild_class(strategy)
+      c.new(
+        dao: table.classify.constantize,
+        columns: columns,
+        logger: logger
+      )
+    end
+
+    def Importer.get_rebuild_class(strategy)
+      case strategy.to_s
+      when 'rename' then RebuildRename
+      when 'truncate' then RebuildTruncate
+      else
+        raise ArgumentError, "unsupported rebuild strategy: #{strategy.inspect}"
+      end
+    end
+  end
+end

data/lib/{redshift-connector → redshift_connector}/importer/insert_delta.rb

@@ -1,19 +1,18 @@
-require 'redshift-connector/importer/activerecord-import'
-require 'redshift-connector/logger'
+require 'redshift_connector/importer/activerecord-import'
+require 'redshift_connector/logger'
 
 module RedshiftConnector
   class Importer::InsertDelta
-    def initialize(dao:, bundle:, columns:, delete_cond:, logger: RedshiftConnector.logger)
+    def initialize(dao:, columns:, delete_cond:, logger: RedshiftConnector.logger)
       @dao = dao
-      @bundle = bundle
       @columns = columns
       @delete_cond = delete_cond
       @logger = logger
     end
 
-    def execute
+    def execute(bundle)
       delete_rows(@delete_cond)
-      import
+      import(bundle)
     end
 
     def delete_rows(cond_expr)
@@ -22,9 +21,9 @@ module RedshiftConnector
       @logger.info "deleted."
     end
 
-    def import
-      @logger.info "IMPORT #{@bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')})"
-      @bundle.each_batch do |rows|
+    def import(bundle)
+      @logger.info "IMPORT #{bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')})"
+      bundle.each_batch do |rows|
         @dao.import(@columns, rows)
       end
     end

data/lib/{redshift-connector → redshift_connector}/importer/rebuild_rename.rb

@@ -1,16 +1,15 @@
-require 'redshift-connector/importer/activerecord-import'
-require 'redshift-connector/logger'
+require 'redshift_connector/importer/activerecord-import'
+require 'redshift_connector/logger'
 
 module RedshiftConnector
   class Importer::RebuildRename
-    def initialize(dao:, bundle:, columns:, logger: RedshiftConnector.logger)
+    def initialize(dao:, columns:, logger: RedshiftConnector.logger)
       @dao = dao
-      @bundle = bundle
       @columns = columns
       @logger = logger
     end
 
-    def execute
+    def execute(bundle)
       dest_table = @dao.table_name
       tmp_table = "#{dest_table}_new"
       old_table = "#{dest_table}_old"
@@ -20,7 +19,7 @@ module RedshiftConnector
 
       exec_update "drop table if exists #{tmp_table}"
       exec_update "create table #{tmp_table} like #{dest_table}"
-      import(tmp_dao)
+      import(tmp_dao, bundle)
       exec_update "drop table if exists #{old_table}"
       # Atomic table exchange
       exec_update "rename table #{dest_table} to #{old_table}, #{tmp_table} to #{dest_table}"
@@ -31,9 +30,9 @@ module RedshiftConnector
       @dao.connection.execute(query)
     end
 
-    def import(dao)
-      @logger.info "IMPORT #{@bundle.url}* -> #{dao.table_name} (#{@columns.join(', ')})"
-      @bundle.each_batch do |rows|
+    def import(dao, bundle)
+      @logger.info "IMPORT #{bundle.url}* -> #{dao.table_name} (#{@columns.join(', ')})"
+      bundle.each_batch do |rows|
         dao.import(@columns, rows)
       end
     end

data/lib/{redshift-connector → redshift_connector}/importer/rebuild_truncate.rb

@@ -1,18 +1,17 @@
-require 'redshift-connector/importer/activerecord-import'
-require 'redshift-connector/logger'
+require 'redshift_connector/importer/activerecord-import'
+require 'redshift_connector/logger'
 
 module RedshiftConnector
   class Importer::RebuildTruncate
-    def initialize(dao:, bundle:, columns:, logger: RedshiftConnector.logger)
+    def initialize(dao:, columns:, logger: RedshiftConnector.logger)
       @dao = dao
-      @bundle = bundle
       @columns = columns
       @logger = logger
     end
 
-    def execute
+    def execute(bundle)
       truncate_table(@dao.table_name)
-      import
+      import(bundle)
     end
 
     def truncate_table(table_name)
@@ -21,9 +20,9 @@ module RedshiftConnector
       @logger.info "truncated."
     end
 
-    def import
-      @logger.info "IMPORT #{@bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')})"
-      @bundle.each_batch do |rows|
+    def import(bundle)
+      @logger.info "IMPORT #{bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')})"
+      bundle.each_batch do |rows|
         @dao.import(@columns, rows)
       end
     end

data/lib/redshift_connector/importer/upsert.rb

@@ -0,0 +1,24 @@
+require 'redshift_connector/importer/activerecord-import'
+require 'redshift_connector/logger'
+
+module RedshiftConnector
+  class Importer::Upsert
+    def initialize(dao:, columns:, upsert_columns:, logger: RedshiftConnector.logger)
+      @dao = dao
+      @columns = columns
+      @upsert_columns = upsert_columns
+      @logger = logger
+    end
+
+    def execute(bundle)
+      import(bundle)
+    end
+
+    def import(bundle)
+      @logger.info "IMPORT #{bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')}) upsert (#{@upsert_columns.join(', ')})"
+      bundle.each_batch do |rows|
+        @dao.import(@columns, rows, on_duplicate_key_update: @upsert_columns)
+      end
+    end
+  end
+end

data/lib/{redshift-connector → redshift_connector}/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '6.0.0'
+  VERSION = '7.0.0'
 end

data/test/config.rb

@@ -1,18 +1,13 @@
-module RedshiftConnector
-  # For test only
-  $TEST_SCHEMA = 'hidekazukobayashi'
+$TEST_SCHEMA = 'aamine'
 
-  Exporter.default_data_source = Redshift
+module RedshiftConnector
+  Exporter.default_data_source = RedshiftConnector::ActiveRecordDataSource.new(Redshift)
 
   S3Bucket.add(
-    'test',
-    bucket: 'misc-internal.ap-northeast-1',
-    prefix: 'development/hidekazu-kobayashi',
-    # When using IAM role
-    #iam_role: 'arn:aws:iam::NNNNNNNNNNNN:role/RRRRRRRRR',
-    # When using explicit access key
-    access_key_id: 'AKIAJJGEKUU2MXO3X4NA',
-    secret_access_key: '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',
+    'redshift-copy-buffer',
+    bucket: 'redshift-copy-buffer',
+    prefix: 'development',
+    iam_role: 'arn:aws:iam::789035092620:role/RedshiftDevelopers',
     default: true
   )
 end

data/test/database.yml

@@ -1,7 +1,7 @@
 mysql:
   adapter: mysql2
   host: localhost
-  username: test
+  username: minero-aoki
   database: test
   encoding: utf8
 
@@ -10,6 +10,6 @@ redshift:
   host: dwh.ckpd.co
   port: 5439
   database: production
-  username: hidekazukobayashi
-  password: '?N6s3oH#jc9k3d+P'
+  username: aamine
+  password: "3edCVfr$"
   encoding: utf8

data/test/foreach.rb

@@ -0,0 +1,5 @@
+require_relative 'helper'
+
+RedshiftConnector.foreach(schema: 'tabemiru', table: 'items', query: 'select id from tabemiru.items where id < 50 order by 1', enable_sort: true) do |row|
+  p row
+end

metadata

@@ -1,31 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 7.0.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2017-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: redshift-connector-data_file
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 7.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 7.0.0
 - !ruby/object:Gem::Dependency
-  name: activerecord-redshift
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,33 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redshift-connector-data_file
+  name: activerecord-redshift
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: 0.18.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: 0.18.0
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -144,33 +144,35 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/redshift-connector.rb
-- lib/redshift-connector/connector.rb
-- lib/redshift-connector/exporter.rb
-- lib/redshift-connector/importer.rb
-- lib/redshift-connector/importer/activerecord-import.rb
-- lib/redshift-connector/importer/insert_delta.rb
-- lib/redshift-connector/importer/rebuild_rename.rb
-- lib/redshift-connector/importer/rebuild_truncate.rb
-- lib/redshift-connector/importer/upsert.rb
-- lib/redshift-connector/logger.rb
-- lib/redshift-connector/query.rb
-- lib/redshift-connector/s3_bucket.rb
-- lib/redshift-connector/s3_data_file.rb
-- lib/redshift-connector/s3_data_file_bundle.rb
-- lib/redshift-connector/version.rb
+- lib/redshift_connector.rb
+- lib/redshift_connector/active_record_data_source.rb
+- lib/redshift_connector/active_record_exporter.rb
+- lib/redshift_connector/connector.rb
+- lib/redshift_connector/data_file_bundle_params.rb
+- lib/redshift_connector/exception.rb
+- lib/redshift_connector/exporter.rb
+- lib/redshift_connector/exporter_builder.rb
+- lib/redshift_connector/importer.rb
+- lib/redshift_connector/importer/activerecord-import.rb
+- lib/redshift_connector/importer/insert_delta.rb
+- lib/redshift_connector/importer/rebuild_rename.rb
+- lib/redshift_connector/importer/rebuild_truncate.rb
+- lib/redshift_connector/importer/upsert.rb
+- lib/redshift_connector/query.rb
+- lib/redshift_connector/version.rb
 - test/all.rb
 - test/config.rb
 - test/config.rb.example
 - test/database.yml
 - test/database.yml.example
+- test/foreach.rb
 - test/helper.rb
 - test/item_pvs.ct.mysql
 - test/item_pvs.ct.redshift
 - test/reader/test_redshift_csv.rb
 - test/test_connector.rb
-- test/test_reader.rb
 - test/test_s3_import.rb
-homepage: https://github.com/aamine/redshift-connector
+homepage: https://github.com/bricolages/redshift-connector
 licenses:
 - MIT
 metadata: {}
@@ -190,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Redshift bulk data connector

data/lib/redshift-connector/exporter.rb

@@ -1,103 +0,0 @@
-require 'redshift-connector/query'
-require 'redshift-connector/logger'
-
-module RedshiftConnector
-  class Exporter
-    def Exporter.default_data_source=(ds)
-      @default_data_source = ds
-    end
-
-    def Exporter.default_data_source
-      @default_data_source or raise ArgumentError, "RedshiftConnector::Exporter.default_data_source was not set"
-    end
-
-    def Exporter.for_table_delta(ds: default_data_source, schema:, table:, condition:, columns:, bundle:, logger: RedshiftConnector.logger)
-      delta_query = DeltaQuery.new(schema: schema, table: table, columns: columns, condition: condition)
-      unload_query = UnloadQuery.new(query: delta_query, bundle: bundle)
-      new(ds: ds, query: unload_query, bundle: bundle, logger: logger)
-    end
-
-    def Exporter.for_table(ds: default_data_source, schema:, table:, columns:, bundle:, logger: RedshiftConnector.logger)
-      query = SelectAllQuery.new(schema: schema, table: table, columns: columns)
-      unload_query = UnloadQuery.new(query: query, bundle: bundle)
-      new(ds: ds, query: unload_query, bundle: bundle, logger: logger)
-    end
-
-    def Exporter.foreach(**params, &block)
-      exporter = Exporter.for_query(**params)
-      begin
-        exporter.execute
-        exporter.bundle.each_row(&block)
-      ensure
-        exporter.bundle.clear
-      end
-    end
-
-    def Exporter.for_query(
-        ds: default_data_source,
-        schema:,
-        table:,
-        bucket: nil,
-        query:,
-        txn_id: "#{Time.now.strftime('%Y%m%d_%H%M%S')}_#{$$}",
-        filter: nil,
-        enable_sort: false,
-        logger: RedshiftConnector.logger,
-        quiet: false
-    )
-      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
-      logger = NullLogger.new if quiet
-      bundle = S3DataFileBundle.for_table(
-        bucket: bucket,
-        schema: schema,
-        table: table,
-        txn_id: txn_id,
-        filter: filter,
-        logger: logger
-      )
-      exporter = Exporter.new(
-        ds: ds,
-        query: UnloadQuery.wrap(query: query, bundle: bundle, enable_sort: enable_sort),
-        bundle: bundle,
-        logger: logger
-      )
-      exporter
-    end
-
-    def initialize(ds: self.class.default_data_source, query:, bundle:, logger: RedshiftConnector.logger)
-      @ds = ds
-      @query = query
-      @bundle = bundle
-      @logger = logger
-    end
-
-    attr_reader :query
-    attr_reader :bundle
-    attr_reader :logger
-
-    def execute
-      @bundle.clear
-      @logger.info "EXPORT #{@query.description} -> #{@bundle.url}*"
-      @ds.connection_pool.with_connection do |conn|
-        stmt = @query.to_sql
-        @logger.info "[SQL/Redshift] #{batch_job_label}#{stmt.strip}"
-        conn.execute(batch_job_label + stmt)
-      end
-    end
-
-    def batch_job_label
-      @batch_job_label ||= begin
-        components = Dir.getwd.split('/')
-        app = if components.last == 'current'
-            # is Capistrano environment
-            components[-2]
-          else
-            components[-1]
-          end
-        batch_file = caller.detect {|c| /redshift-connector|active_record/ !~ c }
-        path = batch_file ? batch_file.split(':').first : '?'
-        "/* Job: #{app}:#{path} */ "
-      end
-    end
-  end
-end

data/lib/redshift-connector/importer.rb

@@ -1,117 +0,0 @@
-# create module
-module RedshiftConnector
-  module Importer
-  end
-end
-
-require 'redshift-connector/importer/upsert'
-require 'redshift-connector/importer/insert_delta'
-require 'redshift-connector/importer/rebuild_rename'
-require 'redshift-connector/importer/rebuild_truncate'
-
-require 'redshift-connector/s3_data_file_bundle'
-require 'redshift-connector/logger'
-
-module RedshiftConnector
-  module Importer
-    def Importer.transport_delta_from_s3(
-        bucket: nil, prefix:, format:, filter: nil,
-        table:, columns:,
-        delete_cond: nil, upsert_columns: nil,
-        logger: RedshiftConnector.logger, quiet: false)
-      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
-      logger = NullLogger.new if quiet
-      bundle = S3DataFileBundle.for_prefix(
-        bucket: bucket,
-        prefix: prefix,
-        format: format,
-        filter: filter,
-        logger: logger
-      )
-      transport_delta_from_bundle(
-        bundle: bundle,
-        table: table, columns: columns,
-        delete_cond: delete_cond, upsert_columns: upsert_columns,
-        logger: logger, quiet: quiet
-      )
-    end
-
-    def Importer.transport_delta_from_bundle(
-      bundle:,
-      table:, columns:,
-      delete_cond: nil, upsert_columns: nil,
-      logger: RedshiftConnector.logger, quiet: false
-    )
-      if delete_cond and upsert_columns
-        raise ArgumentError, "delete_cond and upsert_columns are exclusive"
-      end
-      importer =
-        if delete_cond
-          Importer::InsertDelta.new(
-            dao: table.classify.constantize,
-            bundle: bundle,
-            columns: columns,
-            delete_cond: delete_cond,
-            logger: logger
-          )
-        elsif upsert_columns
-          Importer::Upsert.new(
-            dao: table.classify.constantize,
-            bundle: bundle,
-            columns: columns,
-            upsert_columns: upsert_columns,
-            logger: logger
-          )
-        else
-          raise ArgumentError, "either of delete_cond or upsert_columns is required for transport_delta"
-        end
-      importer
-    end
-
-    def Importer.transport_all_from_s3(
-        strategy: 'rename',
-        bucket: nil, prefix:, format:, filter: nil,
-        table:, columns:,
-        logger: RedshiftConnector.logger, quiet: false)
-      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
-      logger = NullLogger.new if quiet
-      bundle = S3DataFileBundle.for_prefix(
-        bucket: bucket,
-        prefix: prefix,
-        format: format,
-        filter: filter,
-        logger: logger
-      )
-      transport_all_from_bundle(
-        strategy: strategy,
-        bundle: bundle,
-        table: table, columns: columns,
-        logger: logger, quiet: quiet
-      )
-    end
-
-    def Importer.transport_all_from_bundle(
-      strategy: 'rename',
-      bundle:,
-      table:, columns:,
-      logger: RedshiftConnector.logger, quiet: false
-    )
-      importer = get_rebuild_class(strategy).new(
-        dao: table.classify.constantize,
-        bundle: bundle,
-        columns: columns,
-        logger: logger
-      )
-      importer
-    end
-
-    def Importer.get_rebuild_class(strategy)
-      case strategy.to_s
-      when 'rename' then RebuildRename
-      when 'truncate' then RebuildTruncate
-      else
-        raise ArgumentError, "unsupported rebuild strategy: #{strategy.inspect}"
-      end
-    end
-  end
-end

data/lib/redshift-connector/importer/upsert.rb

@@ -1,25 +0,0 @@
-require 'redshift-connector/importer/activerecord-import'
-require 'redshift-connector/logger'
-
-module RedshiftConnector
-  class Importer::Upsert
-    def initialize(dao:, bundle:, columns:, upsert_columns:, logger: RedshiftConnector.logger)
-      @dao = dao
-      @bundle = bundle
-      @columns = columns
-      @upsert_columns = upsert_columns
-      @logger = logger
-    end
-
-    def execute
-      import
-    end
-
-    def import
-      @logger.info "IMPORT #{@bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')}) upsert (#{@upsert_columns.join(', ')})"
-      @bundle.each_batch do |rows|
-        @dao.import(@columns, rows, on_duplicate_key_update: @upsert_columns)
-      end
-    end
-  end
-end

data/lib/redshift-connector/logger.rb

@@ -1,20 +0,0 @@
-module RedshiftConnector
-  @logger = nil
-
-  def RedshiftConnector.logger
-    # Defer to access Rails
-    @logger || Rails.logger
-  end
-
-  def RedshiftConnector.logger=(logger)
-    @logger = logger
-  end
-
-  class NullLogger
-    def noop(*args) end
-    alias error noop
-    alias warn noop
-    alias info noop
-    alias debug noop
-  end
-end

data/lib/redshift-connector/s3_bucket.rb

@@ -1,76 +0,0 @@
-require 'aws-sdk'
-
-module RedshiftConnector
-  class S3Bucket
-    @buckets = {}
-    @default = nil
-
-    def S3Bucket.add(name, default: false, **params)
-      instance = new(**params)
-      @buckets[name.to_s] = instance
-      if !@default or default
-        @default = instance
-      end
-    end
-
-    def S3Bucket.default
-      @default or raise ArgumentError, "no default S3 bucket configured"
-    end
-
-    def S3Bucket.get(name)
-      @buckets[name.to_s] or raise ArgumentError, "no such S3 bucket configured: #{name.inspect}"
-    end
-
-    def initialize(region: nil, bucket:, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil)
-      @region = region
-      @name = bucket
-      @prefix = prefix
-      @access_key_id = access_key_id
-      @secret_access_key = secret_access_key
-      @iam_role = iam_role
-    end
-
-    attr_reader :name
-    attr_reader :prefix
-
-    def url
-      "s3://#{@bucket.name}/#{@prefix}/"
-    end
-
-    def client
-      @client ||= begin
-        args = { region: @region, access_key_id: @access_key_id, secret_access_key: @secret_access_key }.reject {|k, v| v.nil? }
-        Aws::S3::Client.new(**args)
-      end
-    end
-
-    def bucket
-      @bucket ||= begin
-        resource = Aws::S3::Resource.new(client: client)
-        resource.bucket(@name)
-      end
-    end
-
-    def object(key)
-      bucket.object(key)
-    end
-
-    def objects(prefix:)
-      bucket.objects(prefix: prefix)
-    end
-
-    def delete_objects(keys)
-      bucket.delete_objects(delete: {objects: keys.map {|k| {key: k} }})
-    end
-
-    def credential_string
-      if @iam_role
-        "aws_iam_role=#{@iam_role}"
-      elsif @access_key_id
-        "aws_access_key_id=#{@access_key_id};aws_secret_access_key=#{@secret_access_key}"
-      else
-        raise ArgumentError, "no credential given for Redshift S3 access"
-      end
-    end
-  end
-end

data/lib/redshift-connector/s3_data_file.rb

@@ -1,20 +0,0 @@
-require 'redshift-connector/data_file'
-
-module RedshiftConnector
-  class S3DataFile < AbstractDataFile
-    def initialize(object, reader_class:)
-      @object = object
-      @reader_class = reader_class
-    end
-
-    def key
-      @object.key
-    end
-
-    def content
-      @object.get.body
-    end
-
-    delegate :presigned_url, to: :@object
-  end
-end

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -1,54 +0,0 @@
-require 'redshift-connector/s3_bucket'
-require 'redshift-connector/s3_data_file'
-require 'redshift-connector/logger'
-require 'redshift-connector/data_file'
-require 'aws-sdk'
-
-module RedshiftConnector
-  class S3DataFileBundle < AbstractDataFileBundle
-    def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
-      real_prefix = "#{bucket.prefix}/#{prefix}"
-      new(bucket, real_prefix, format: format, filter: filter, batch_size: batch_size, logger: logger)
-    end
-
-    def self.for_table(bucket: S3Bucket.default, schema:, table:, txn_id:, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
-      prefix = "#{bucket.prefix}/#{schema}_export/#{table}/#{txn_id}/#{table}.csv."
-      new(bucket, prefix, format: :redshift_csv, filter: filter, batch_size: batch_size, logger: logger)
-    end
-
-    def initialize(bucket, prefix, format: :csv, filter: nil, batch_size: 1000, logger: RedshiftConnector.logger)
-      @bucket = bucket
-      @prefix = prefix
-      @format = format
-      @filter = filter || lambda {|*row| row }
-      @batch_size = batch_size
-      @logger = logger
-      @reader_class = Reader.get(format)
-    end
-
-    attr_reader :bucket
-    attr_reader :prefix
-
-    def url
-      "s3://#{@bucket.name}/#{@prefix}"
-    end
-
-    def credential_string
-      @bucket.credential_string
-    end
-
-    def data_files
-      @bucket.objects(prefix: @prefix)
-        .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }
-    end
-
-    def clear
-      pref = File.dirname(@prefix) + '/'
-      keys = @bucket.objects(prefix: pref).map(&:key)
-      unless keys.empty?
-        @logger.info "DELETE #{pref}*"
-        @bucket.delete_objects(keys)
-      end
-    end
-  end
-end

data/test/test_reader.rb

@@ -1,9 +0,0 @@
-require 'test/unit'
-
-module RedshiftConnector
-  class TestReader < Test::Unit::TestCase
-    def test_get
-      assert_equal Reader::RedshiftCSV, Reader.get(:redshift_csv)
-    end
-  end
-end