redshift-connector 5.6.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 353a12f1ed7ed90351c0ed041e703f12677adc96
-  data.tar.gz: 82c7026fe179f663b7aa23c6ee77ac9dcac0fea4
+  metadata.gz: f31b6eadc1161cb752be8f735f6ab3cdf2e87d32
+  data.tar.gz: edb474edc4bd93ee6ab5efc26d9b9770f05bd109
 SHA512:
-  metadata.gz: 94a7f7f64546b8d1007f036f904e06778bca6632ae2fe312c11d0a086b4b63085aa8d91464fd7bdcd47f138ae10360e9b8d81e5def7701a031a994128eadb9a1
-  data.tar.gz: 7c4da6fd4f9e320ec34d6fc4bc5cc0a2b9723322599ee4a94a795e49e3c1622efa5b16ea7d5f93c1f7cf081ef42eac75e0210381b9225f7661a8fa8567b15872
+  metadata.gz: 6d8c0229f3bc0a7beae47859ac7aee5b0df82d82e838ad33803ab9d9e2321e660cd6befd74fee7e800f9394efff425856f20bc0fb77c69dbac407315a65d8333
+  data.tar.gz: c7514f15467cf56f6c3b60c0670a4f338645d0b05b96e418500aa9172b696a3f22c3277d498a4e309a779c2ab3202cd4f5a8dd2ad9fcda3e1527390a6a36b74b

data/lib/redshift-connector/connector.rb

@@ -40,30 +40,12 @@ module RedshiftConnector
         condition: condition,
         logger: logger
       )
-      if delete_cond and upsert_columns
-        raise ArgumentError, "delete_cond and upsert_columns are exclusive"
-      end
-      dao = dest_table.classify.constantize
-      importer =
-        if delete_cond
-          Importer::InsertDelta.new(
-            dao: dao,
-            bundle: bundle,
-            columns: columns,
-            delete_cond: delete_cond,
-            logger: logger
-          )
-        elsif upsert_columns
-          Importer::Upsert.new(
-            dao: dao,
-            bundle: bundle,
-            columns: columns,
-            upsert_columns: upsert_columns,
-            logger: logger
-          )
-        else
-          raise ArgumentError, "either of delete_cond or upsert_columns is required for transport_delta"
-        end
+      importer = Importer.transport_delta_from_bundle(
+        bundle: bundle,
+        table: dest_table, columns: columns,
+        delete_cond: delete_cond, upsert_columns: upsert_columns,
+        logger: logger, quiet: quiet
+      )
       new(exporter: exporter, importer: importer, logger: logger)
     end
 
@@ -97,11 +79,11 @@ module RedshiftConnector
         columns: columns,
         logger: logger
       )
-      importer = Importer.get_rebuild_class(strategy).new(
-        dao: table.classify.constantize,
+      importer = Importer.transport_all_from_bundle(
+        strategy: strategy,
         bundle: bundle,
-        columns: columns,
-        logger: logger
+        table: table, columns: columns,
+        logger: logger, quiet: quiet
       )
       new(exporter: exporter, importer: importer, logger: logger)
     end

data/lib/redshift-connector/importer.rb

@@ -28,6 +28,20 @@ module RedshiftConnector
         filter: filter,
         logger: logger
       )
+      transport_delta_from_bundle(
+        bundle: bundle,
+        table: table, columns: columns,
+        delete_cond: delete_cond, upsert_columns: upsert_columns,
+        logger: logger, quiet: quiet
+      )
+    end
+
+    def Importer.transport_delta_from_bundle(
+      bundle:,
+      table:, columns:,
+      delete_cond: nil, upsert_columns: nil,
+      logger: RedshiftConnector.logger, quiet: false
+    )
       if delete_cond and upsert_columns
         raise ArgumentError, "delete_cond and upsert_columns are exclusive"
       end
@@ -68,6 +82,20 @@ module RedshiftConnector
         filter: filter,
         logger: logger
       )
+      transport_all_from_bundle(
+        strategy: strategy,
+        bundle: bundle,
+        table: table, columns: columns,
+        logger: logger, quiet: quiet
+      )
+    end
+
+    def Importer.transport_all_from_bundle(
+      strategy: 'rename',
+      bundle:,
+      table:, columns:,
+      logger: RedshiftConnector.logger, quiet: false
+    )
       importer = get_rebuild_class(strategy).new(
         dao: table.classify.constantize,
         bundle: bundle,

data/lib/redshift-connector/s3_data_file_bundle.rb

@@ -37,37 +37,6 @@ module RedshiftConnector
       @bucket.credential_string
     end
 
-    REPORT_SIZE = 10_0000
-
-    def each_batch(report: true)
-      @logger.info "reader: #{@reader_class}"
-      n = 0
-      reported = 0
-      do_each_batch(@batch_size) do |rows|
-        yield rows
-        n += rows.size
-        if n / REPORT_SIZE > reported
-          @logger.info "#{n} rows processed" if report
-          reported = n / REPORT_SIZE
-        end
-      end
-      @logger.info "total #{n} rows processed" if report
-    end
-
-    def do_each_batch(batch_size)
-      filter = @filter
-      buf = []
-      each_row do |row|
-        buf.push filter.(*row)
-        if buf.size == batch_size
-          yield buf
-          buf = []
-        end
-      end
-      yield buf unless buf.empty?
-    end
-    private :do_each_batch
-
     def data_files
       @bucket.objects(prefix: @prefix)
         .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }

data/lib/redshift-connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '5.6.0'
+  VERSION = '6.0.0'
 end

data/test/config.rb

@@ -1,13 +1,18 @@
-$TEST_SCHEMA = 'aamine'
-
 module RedshiftConnector
+  # For test only
+  $TEST_SCHEMA = 'hidekazukobayashi'
+
   Exporter.default_data_source = Redshift
 
   S3Bucket.add(
-    'redshift-copy-buffer',
-    bucket: 'redshift-copy-buffer',
-    prefix: 'development',
-    iam_role: 'arn:aws:iam::789035092620:role/RedshiftDevelopers',
+    'test',
+    bucket: 'misc-internal.ap-northeast-1',
+    prefix: 'development/hidekazu-kobayashi',
+    # When using IAM role
+    #iam_role: 'arn:aws:iam::NNNNNNNNNNNN:role/RRRRRRRRR',
+    # When using explicit access key
+    access_key_id: 'AKIAJJGEKUU2MXO3X4NA',
+    secret_access_key: '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',
     default: true
   )
 end

data/test/database.yml

@@ -1,7 +1,7 @@
 mysql:
   adapter: mysql2
   host: localhost
-  username: minero-aoki
+  username: test
   database: test
   encoding: utf8
 
@@ -10,6 +10,6 @@ redshift:
   host: dwh.ckpd.co
   port: 5439
   database: production
-  username: aamine
-  password: "3edCVfr$"
+  username: hidekazukobayashi
+  password: '?N6s3oH#jc9k3d+P'
   encoding: utf8

metadata

@@ -1,31 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 5.6.0
+  version: 6.0.0
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-17 00:00:00.000000000 Z
+date: 2017-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: redshift-connector-data_file
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: activerecord-redshift
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,33 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activerecord-redshift
+  name: redshift-connector-data_file
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.0
+        version: '0.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.18.0
+        version: '0.18'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -163,7 +163,6 @@ files:
 - test/config.rb.example
 - test/database.yml
 - test/database.yml.example
-- test/foreach.rb
 - test/helper.rb
 - test/item_pvs.ct.mysql
 - test/item_pvs.ct.redshift
@@ -191,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Redshift bulk data connector

data/test/foreach.rb

@@ -1,5 +0,0 @@
-require_relative 'helper'
-
-RedshiftConnector.foreach(schema: 'tabemiru', table: 'items', query: 'select id from tabemiru.items where id < 50 order by 1', enable_sort: true) do |row|
-  p row
-end