redshift-connector 4.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c5d5bf307943f19b5187a9244ecd89717241d72f
+  data.tar.gz: 1651cd723205ee50247c5e407f39aaa1eafd287d
+SHA512:
+  metadata.gz: ff3ef2d4b7e647617529313df5b1d24eb9bf537672ba4aeba3de6542d2fa60589576854b308bc7db5b34c853e1cf30a1a517f6cfd1c58b876018dd6c6fcab7c8
+  data.tar.gz: 51de6b97fd7099a2dfaf401b1f230e49f991c623cd43949bbf9e8a4e5f822180040b024638938613aaa89fdbaced93f5c9e27bf6e5e57c8e3d6c24a486f8c06b

data/README.md

@@ -0,0 +1,10 @@
+# Redshift Connector for Rails
+
+redshift-connector is a Redshift bulk data connector for Rails (ActiveRecord).
+
+## Settings
+
+Add following block to your Gemfile and bundle.
+```
+gem 'redshift-connector'
+```

data/lib/redshift-connector.rb

@@ -0,0 +1,31 @@
+module RedshiftConnector
+end
+
+require 'redshift-connector/connector'
+require 'redshift-connector/exporter'
+require 'redshift-connector/importer'
+require 'redshift-connector/s3_bucket'
+require 'redshift-connector/s3_data_file_bundle'
+require 'redshift-connector/version'
+
+module RedshiftConnector
+  def RedshiftConnector.transport_delta(**params)
+    Connector.transport_delta(**params)
+  end
+
+  def RedshiftConnector.transport_all(**params)
+    Connector.transport_all(**params)
+  end
+
+  def RedshiftConnector.transport_delta_from_s3(**params)
+    Importer.transport_delta_from_s3(**params)
+  end
+
+  def RedshiftConnector.transport_all_from_s3(**params)
+    Importer.transport_all_from_s3(**params)
+  end
+
+  def RedshiftConnector.foreach(**params, &block)
+    Exporter.foreach(**params, &block)
+  end
+end

data/lib/redshift-connector/connector.rb

@@ -0,0 +1,146 @@
+require 'redshift-connector/exporter'
+require 'redshift-connector/importer'
+require 'redshift-connector/s3_data_file_bundle'
+require 'redshift-connector/logger'
+
+module RedshiftConnector
+  class Connector
+    def Connector.transport_delta(
+        schema:,
+        table: nil,
+        src_table: table,
+        dest_table: table,
+        condition:,
+        columns:,
+        delete_cond: nil,
+        upsert_columns: nil,
+        bucket: nil,
+        txn_id:, filter:,
+        logger: RedshiftConnector.logger,
+        quiet: false
+    )
+      unless src_table and dest_table
+        raise ArgumentError, "missing :table, :src_table or :dest_table"
+      end
+      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
+      logger = NullLogger.new if quiet
+      bundle = S3DataFileBundle.for_table(
+        bucket: bucket,
+        schema: schema,
+        table: src_table,
+        txn_id: txn_id,
+        filter: filter,
+        logger: logger
+      )
+      exporter = Exporter.for_table_delta(
+        bundle: bundle,
+        schema: schema,
+        table: src_table,
+        columns: columns,
+        condition: condition,
+        logger: logger
+      )
+      if delete_cond and upsert_columns
+        raise ArgumentError, "delete_cond and upsert_columns are exclusive"
+      end
+      dao = dest_table.classify.constantize
+      importer =
+        if delete_cond
+          Importer::InsertDelta.new(
+            dao: dao,
+            bundle: bundle,
+            columns: columns,
+            delete_cond: delete_cond,
+            logger: logger
+          )
+        elsif upsert_columns
+          Importer::Upsert.new(
+            dao: dao,
+            bundle: bundle,
+            columns: columns,
+            upsert_columns: upsert_columns,
+            logger: logger
+          )
+        else
+          raise ArgumentError, "either of delete_cond or upsert_columns is required for transport_delta"
+        end
+      new(exporter: exporter, importer: importer, logger: logger)
+    end
+
+    def Connector.transport_all(
+        strategy: 'rename',
+        schema:,
+        table:,
+        src_table: table,
+        dest_table: table,
+        columns:,
+        bucket: nil,
+        txn_id:,
+        filter:,
+        logger: RedshiftConnector.logger,
+        quiet: false
+    )
+      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
+      logger = NullLogger.new if quiet
+      bundle = S3DataFileBundle.for_table(
+        bucket: bucket,
+        schema: schema,
+        table: table,
+        txn_id: txn_id,
+        filter: filter,
+        logger: logger
+      )
+      exporter = Exporter.for_table(
+        bundle: bundle,
+        schema: schema,
+        table: table,
+        columns: columns,
+        logger: logger
+      )
+      importer = Importer.get_rebuild_class(strategy).new(
+        dao: table.classify.constantize,
+        bundle: bundle,
+        columns: columns,
+        logger: logger
+      )
+      new(exporter: exporter, importer: importer, logger: logger)
+    end
+
+    def initialize(exporter:, importer:, logger:)
+      @exporter = exporter
+      @importer = importer
+      @logger = logger
+    end
+
+    def export_enabled?
+      not ENV['IMPORT_ONLY']
+    end
+
+    def export_forced?
+      !! (ENV['EXPORT_ONLY'] or ENV['FORCE'])
+    end
+
+    def import_enabled?
+      not ENV['EXPORT_ONLY']
+    end
+
+    def execute
+      export(forced: export_forced?) if export_enabled?
+      import if import_enabled?
+    end
+
+    def export(forced: false)
+      @logger.info "==== export task =================================================="
+      if not forced and @exporter.completed?
+        @logger.info "export task is already executed; skip"
+      else
+        @exporter.execute
+      end
+    end
+
+    def import
+      @logger.info "==== import task =================================================="
+      @importer.execute
+    end
+  end
+end

data/lib/redshift-connector/exporter.rb

@@ -0,0 +1,116 @@
+require 'redshift-connector/query'
+require 'redshift-connector/logger'
+
+module RedshiftConnector
+  class Exporter
+    def Exporter.default_data_source=(ds)
+      @default_data_source = ds
+    end
+
+    def Exporter.default_data_source
+      @default_data_source or raise ArgumentError, "RedshiftConnector::Exporter.default_data_source was not set"
+    end
+
+    def Exporter.for_table_delta(ds: default_data_source, schema:, table:, condition:, columns:, bundle:, logger: RedshiftConnector.logger)
+      delta_query = DeltaQuery.new(schema: schema, table: table, columns: columns, condition: condition)
+      unload_query = UnloadQuery.new(query: delta_query, bundle: bundle)
+      new(ds: ds, query: unload_query, bundle: bundle, logger: logger)
+    end
+
+    def Exporter.for_table(ds: default_data_source, schema:, table:, columns:, bundle:, logger: RedshiftConnector.logger)
+      query = SelectAllQuery.new(schema: schema, table: table, columns: columns)
+      unload_query = UnloadQuery.new(query: query, bundle: bundle)
+      new(ds: ds, query: unload_query, bundle: bundle, logger: logger)
+    end
+
+    def Exporter.foreach(**params, &block)
+      exporter = Exporter.for_query(**params)
+      begin
+        exporter.execute
+        exporter.bundle.each_row(&block)
+      ensure
+        exporter.bundle.clear
+      end
+    end
+
+    def Exporter.for_query(
+        ds: default_data_source,
+        schema:,
+        table:,
+        bucket: nil,
+        query:,
+        txn_id: "#{Time.now.strftime('%Y%m%d_%H%M%S')}_#{$$}",
+        filter: nil,
+        logger: RedshiftConnector.logger,
+        quiet: false
+    )
+      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
+      logger = NullLogger.new if quiet
+      bundle = S3DataFileBundle.for_table(
+        bucket: bucket,
+        schema: schema,
+        table: table,
+        txn_id: txn_id,
+        filter: filter,
+        logger: logger
+      )
+      exporter = Exporter.new(
+        ds: ds,
+        query: UnloadQuery.wrap(query: query, bundle: bundle),
+        bundle: bundle,
+        logger: logger
+      )
+      exporter
+    end
+
+    def initialize(ds: self.class.default_data_source, query:, bundle:, logger: RedshiftConnector.logger)
+      @ds = ds
+      @query = query
+      @bundle = bundle
+      @logger = logger
+    end
+
+    attr_reader :query
+    attr_reader :bundle
+    attr_reader :logger
+
+    def completed?
+      @bundle.bucket.object(flag_object_key).exists?
+    end
+
+    def create_flag_object
+      @logger.info "TOUCH #{flag_object_key}"
+      @bundle.bucket.object(flag_object_key).put(body: "OK")
+    end
+
+    def flag_object_key
+      "#{File.dirname(@bundle.prefix)}/00completed"
+    end
+
+    def execute
+      @bundle.clear
+      @logger.info "EXPORT #{@query.description} -> #{@bundle.url}*"
+      @ds.connection_pool.with_connection do |conn|
+        stmt = @query.to_sql
+        @logger.info "[SQL/Redshift] #{batch_job_label}#{stmt.strip}"
+        conn.execute(batch_job_label + stmt)
+      end
+      create_flag_object
+    end
+
+    def batch_job_label
+      @batch_job_label ||= begin
+        components = Dir.getwd.split('/')
+        app = if components.last == 'current'
+            # is Capistrano environment
+            components[-2]
+          else
+            components[-1]
+          end
+        batch_file = caller.detect {|c| /redshift-connector|active_record/ !~ c }
+        path = batch_file ? batch_file.split(':').first : '?'
+        "/* Job: #{app}:#{path} */ "
+      end
+    end
+  end
+end

data/lib/redshift-connector/importer.rb

@@ -0,0 +1,89 @@
+# create module
+module RedshiftConnector
+  module Importer
+  end
+end
+
+require 'redshift-connector/importer/upsert'
+require 'redshift-connector/importer/insert_delta'
+require 'redshift-connector/importer/rebuild_rename'
+require 'redshift-connector/importer/rebuild_truncate'
+
+require 'redshift-connector/s3_data_file_bundle'
+require 'redshift-connector/logger'
+
+module RedshiftConnector
+  module Importer
+    def Importer.transport_delta_from_s3(
+        bucket: nil, prefix:, format:, filter: nil,
+        table:, columns:,
+        delete_cond: nil, upsert_columns: nil,
+        logger: RedshiftConnector.logger, quiet: false)
+      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
+      logger = NullLogger.new if quiet
+      bundle = S3DataFileBundle.for_prefix(
+        bucket: bucket,
+        prefix: prefix,
+        format: format,
+        filter: filter,
+        logger: logger
+      )
+      if delete_cond and upsert_columns
+        raise ArgumentError, "delete_cond and upsert_columns are exclusive"
+      end
+      importer =
+        if delete_cond
+          Importer::InsertDelta.new(
+            dao: table.classify.constantize,
+            bundle: bundle,
+            columns: columns,
+            delete_cond: delete_cond,
+            logger: logger
+          )
+        elsif upsert_columns
+          Importer::Upsert.new(
+            dao: table.classify.constantize,
+            bundle: bundle,
+            columns: columns,
+            upsert_columns: upsert_columns,
+            logger: logger
+          )
+        else
+          raise ArgumentError, "either of delete_cond or upsert_columns is required for transport_delta"
+        end
+      importer
+    end
+
+    def Importer.transport_all_from_s3(
+        strategy: 'rename',
+        bucket: nil, prefix:, format:, filter: nil,
+        table:, columns:,
+        logger: RedshiftConnector.logger, quiet: false)
+      bucket = bucket ? S3Bucket.get(bucket) : S3Bucket.default
+      logger = NullLogger.new if quiet
+      bundle = S3DataFileBundle.for_prefix(
+        bucket: bucket,
+        prefix: prefix,
+        format: format,
+        filter: filter,
+        logger: logger
+      )
+      importer = get_rebuild_class(strategy).new(
+        dao: table.classify.constantize,
+        bundle: bundle,
+        columns: columns,
+        logger: logger
+      )
+      importer
+    end
+
+    def Importer.get_rebuild_class(strategy)
+      case strategy.to_s
+      when 'rename' then RebuildRename
+      when 'truncate' then RebuildTruncate
+      else
+        raise ArgumentError, "unsupported rebuild strategy: #{strategy.inspect}"
+      end
+    end
+  end
+end

data/lib/redshift-connector/importer/activerecord-import.rb

@@ -0,0 +1,2 @@
+require 'activerecord-import'
+require 'activerecord-import/base'

data/lib/redshift-connector/importer/insert_delta.rb

@@ -0,0 +1,32 @@
+require 'redshift-connector/importer/activerecord-import'
+require 'redshift-connector/logger'
+
+module RedshiftConnector
+  class Importer::InsertDelta
+    def initialize(dao:, bundle:, columns:, delete_cond:, logger: RedshiftConnector.logger)
+      @dao = dao
+      @bundle = bundle
+      @columns = columns
+      @delete_cond = delete_cond
+      @logger = logger
+    end
+
+    def execute
+      delete_rows(@delete_cond)
+      import
+    end
+
+    def delete_rows(cond_expr)
+      @logger.info "DELETE #{@dao.table_name} where (#{cond_expr})"
+      @dao.connection.execute("delete from #{@dao.table_name} where #{cond_expr}")
+      @logger.info "deleted."
+    end
+
+    def import
+      @logger.info "IMPORT #{@bundle.url}* -> #{@dao.table_name} (#{@columns.join(', ')})"
+      @bundle.each_batch do |rows|
+        @dao.import(@columns, rows)
+      end
+    end
+  end
+end

data/lib/redshift-connector/importer/rebuild_rename.rb

@@ -0,0 +1,41 @@
+require 'redshift-connector/importer/activerecord-import'
+require 'redshift-connector/logger'
+
+module RedshiftConnector
+  class Importer::RebuildRename
+    def initialize(dao:, bundle:, columns:, logger: RedshiftConnector.logger)
+      @dao = dao
+      @bundle = bundle
+      @columns = columns
+      @logger = logger
+    end
+
+    def execute
+      dest_table = @dao.table_name
+      tmp_table = "#{dest_table}_new"
+      old_table = "#{dest_table}_old"
+
+      tmp_dao = @dao.dup
+      tmp_dao.table_name = tmp_table
+
+      exec_update "drop table if exists #{tmp_table}"
+      exec_update "create table #{tmp_table} like #{dest_table}"
+      import(tmp_dao)
+      exec_update "drop table if exists #{old_table}"
+      # Atomic table exchange
+      exec_update "rename table #{dest_table} to #{old_table}, #{tmp_table} to #{dest_table}"
+    end
+
+    def exec_update(query)
+      @logger.info query
+      @dao.connection.execute(query)
+    end
+
+    def import(dao)
+      @logger.info "IMPORT #{@bundle.url}* -> #{dao.table_name} (#{@columns.join(', ')})"
+      @bundle.each_batch do |rows|
+        dao.import(@columns, rows)
+      end
+    end
+  end
+end