redshifter 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f5f06faed21081d4041a36d24b07891aa5dd5dc
-  data.tar.gz: 5b6d57bdf2a5a22a30a33af0455b97095d2006e4
+  metadata.gz: c151f3b6b5deb6045873d8a6cad78750678daf42
+  data.tar.gz: c109238dbc076bcb27076698c463ac5f21134fdc
 SHA512:
-  metadata.gz: b27b23941087fd98d1096d26f632757aab46425467885c39b5f11cab69601ca3a2dde796a4a6129b163cac409fcf40b27f8e1bbebc1f33961bf4f46e46b82b60
-  data.tar.gz: 614fe6a514775a18b723478e16adfadb48f9ea0dece1730a842a5579c8a770024418a7c9456dca3a7f677d3f0768ca9eae3b220a422dd2b8c55d98a8c7e6e54a
+  metadata.gz: 6c43734276fe0cd4229713ae6f97d8aa8a824d4472e7f076c2a71ad5ad37ab19d10923c3ecc5d1d71ddcf32fa02964e860d223d5b9e32d8dbcf25a106186c577
+  data.tar.gz: bc7974e4cc1a02d2593fc7ba3647489484a8bbd2834584230b07d1822a6cf85ba70515d66ac0d95fcc43b9aa3f650610d24fd108f7420572fd086eea8422c20e

data/README.md

@@ -19,6 +19,7 @@ Feature Roadmap:
   - 0.5.0 - Add functionality for source_table_filter to selectively export rows
   - 0.5.1 - Fix bug where there was an erroneous default value for source_table_filter
   - 0.6.0 - Add ReplaceRedshiftTableJob
+  - 0.6.1 - Introduce logger config option
 
 ## Installation
 
@@ -62,6 +63,7 @@ Redshifter.setup do |config|
   # start new dynos
   config.heroku_api_key = '<Heroku user api key>'
   config.heroku_app_name = 'name of the app on heroku'
+  config.logger = Rails.logger
 end
 ```
 

data/lib/redshifter/config.rb

@@ -4,6 +4,7 @@ module Redshifter
       yield config if block_given?
       load_export_table_definitions
       config_dynosaur
+      setup_default_logger
     end
 
     def config
@@ -21,7 +22,8 @@ module Redshifter
                              :heroku_api_key,
                              :heroku_app_name,
                              :temp_directory_path,
-                             :table_config_path).new
+                             :table_config_path,
+                             :logger).new
     end
 
     private
@@ -38,5 +40,11 @@ module Redshifter
         config.app_name = Redshifter.config.heroku_app_name
       end
     end
+
+    def setup_default_logger
+      require 'logger'
+      config.logger = Logger.new(STDOUT)
+    end
+
   end
 end

data/lib/redshifter/extract_and_replace_redshift_table.rb

@@ -6,24 +6,31 @@ module Redshifter
     end
 
     def run
+      Redshifter.config.logger.info "Extracting rows in batches ..."
       extracted_s3_urls = Util::ExtractAndTransformUpdates
         .new(table: table,
              since: Table::EPOCH_TIMESTAMP,
              s3_util: s3_util
         ).run
 
+      Redshifter.config.logger.info "Extracted #{extracted_s3_urls.size} batches. "
+
       if extracted_s3_urls.any?
+        Redshifter.config.logger.info "Writing manifest file to S3 ..."
         manifest_url = Util::S3ManifestWriter
           .new(file_name: "#{SecureRandom.uuid}.manifest",
                file_urls: extracted_s3_urls,
                s3_util: s3_util
           ).run
 
+        Redshifter.config.logger.info "Replacing Redshift table ..."
         Util::CreateOrReplaceTable
           .new(table: table,
                manifest_url: manifest_url
           ).run
       end
+
+      Redshifter.config.logger.info "Extract and replace completed successfully!"
     end
 
     private

data/lib/redshifter/extract_and_update_redshift_table.rb

@@ -6,6 +6,7 @@ module Redshifter
     end
 
     def run
+      Redshifter.config.logger.info "Extracting rows in batches ..."
       extracted_s3_urls = Util::ExtractAndTransformUpdates
                             .new(table: table,
                                  since: table.redshift_last_update,
@@ -13,17 +14,21 @@ module Redshifter
                             ).run
 
       if extracted_s3_urls.any?
+        Redshifter.config.logger.info "Writing manifest file to S3 ..."
         manifest_url = Util::S3ManifestWriter
                          .new(file_name: "#{SecureRandom.uuid}.manifest",
                               file_urls: extracted_s3_urls,
                               s3_util: s3_util
                          ).run
 
+        Redshifter.config.logger.info "Updating Redshift table ..."
         Util::UpdateTable
           .new(table: table,
                manifest_url: manifest_url
           ).run
       end
+
+      Redshifter.config.logger.info "Extract and update completed successfully!"
     end
 
     private

data/lib/redshifter/util/extract_and_transform_updates.rb

@@ -18,7 +18,7 @@ module Redshifter
 
       # Writes pipe delimited 'CSV' files to S3 of updated records.
       # Returns a list of internal s3 URLs created
-      def run(batch_size: 1000)
+      def run(batch_size: 10000)
         uploaded_s3_urls = []
         run_name = SecureRandom.uuid
 
@@ -65,6 +65,8 @@ module Redshifter
           # remove the injected id from the first column
           rows.map! { |row| row[1..-1] }
 
+          Redshifter.config.logger.info "Extracting batch #{batch_count} ..."
+
           yield rows, batch_count
 
           break if rows.size < batch_size

data/lib/redshifter/version.rb

@@ -1,3 +1,3 @@
 module Redshifter
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshifter
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Justin Richard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-30 00:00:00.000000000 Z
+date: 2016-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dynosaur