redshifter 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 09710d63e1dafe18da8216b0d4de3662ca660e62
-  data.tar.gz: de3081992b84e55b39ae4c84b4d81fb18db40a4f
+  metadata.gz: 904cec0aad5170e27712addd72009852c1b3a6d5
+  data.tar.gz: 6f5d606f0862a1522288987636928c1be4d0e6e1
 SHA512:
-  metadata.gz: fabe694521c2268d6b55cf634ca2ef62808695641a307eb60f12672e9e05049ccf0cdf3bf6a6ea1176e4e0a1b1d6d67a2e001465233ff025798c55d71553788f
-  data.tar.gz: 658c6ccf1c6dbcf9a21b14f658d4707f2af9f192d400050bc6d559d70064b787f8cef37b7cb4715b617b81e94376d702be9265fe6b7cff407029c20514c1ad4c
+  metadata.gz: 8cc099b110abcd0d0696219604388879215ebcac444d3f36dd098840a8b8b49cbd91d727383493a857c216633ce6f86970122d298f42b2ab02b2094b091a0c6d
+  data.tar.gz: 33a85ccf7d1b28bacb02d36354e0e43406f73edd0ef6d9f49600a9791e05c0c0a7caea1e5a798bffabf64f12bd02d71237f4de9e0936a7309f0181c6e8fd2707

data/README.md

@@ -16,6 +16,7 @@ Feature Roadmap:
   - 0.2.4 - New config format; update and replace rake tasks available
   - 0.3.0 - Public version
   - 0.4.0 - Make S3 region configurable
+  - 0.5.0 - Added functionality for source_table_filter to selectively export rows
 
 ## Installation
 
@@ -79,6 +80,8 @@ Redshifter.config.tables = {
     source_table_name: 'books',
     # [required] Prefixing your redshift table with its source is recommended
     redshift_table_name: 'app_name_books',
+    # [optional] Provide a conditional to specify which rows get exported to Redshift
+    source_table_filter: 'title IS NOT NULL',
     # [required] Columns with Redshift datatypes to create; may differ from source DB
     redshift_columns: {
       'id' => 'INTEGER',
@@ -120,7 +123,7 @@ Redshifter.config.tables = {
 ```
 $ rake redshifter:replace[books_with_export_at]
 ```
-   
+
 ### Schedule a Redshifter::Job::UpdateRedshiftTableJob resque job per each table you want to export updates for
 
 Then schedule this meta job to run in `resque_schedule.yml` to run once at 10:00pm
@@ -134,6 +137,36 @@ etl_books_to_redshift:
   description: 'Export the books table to Redshift'
 ```  
 
+### Monitoring Rake tasks with New Relic (optional)
+New Relic offers Rake task instrumentation as of version `3.13.0` of their `newrelic_rpm` agent.  Redshifter does not directly use or depend on `newrelic_rpm`.  You must use New Relic's rake instrumentation and explicitly identify the tasks in your app that you want to monitor.  See [New Relic docs](https://docs.newrelic.com/docs/agents/ruby-agent/background-jobs/rake-instrumentation).
+
+In addition, to setting `attributes.include` and `rake.tasks` as defined in their docs, it also seems to be necessary to manually start the agent synchonously in your Rakefile to assure that fast running tasks are reported.
+
+In summary, your `newrelic.yml` should include keys and values like this to monitor redshifter rake tasks:
+```yaml
+common: &default_settings
+  #...
+  attributes:
+    include: job.rake.*  # allows rake args reporting
+  rake:
+    # rake task monitoring must be white listed here AND not blacklisted via
+    #   autostart.blacklisted_* config values
+    tasks: ['redshifter:update', 'redshifter:replace']
+#...
+```
+
+and your `Rakefile` should end up looking something like this:
+```ruby
+# Rakefile
+# ...
+require 'redshifter/tasks'
+
+# Force agent start assuring rake.tasks listed in newrelic.yml are instrumented
+NewRelic::Agent.manual_start(sync_startup: true) if Rails.env.production?
+# ...
+```
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/redshifter/table.rb

@@ -14,9 +14,10 @@ module Redshifter
       @redshift_sort_keys = config[:redshift_sort_keys]
       @redshift_sort_style = config[:redshift_sort_style]
       @redshift_primary_key = config[:redshift_primary_key]
+      @source_table_filter = config[:source_table_filter] || 1
     end
 
-    attr_reader :source_table_name, :redshift_table_name
+    attr_reader :source_table_name, :redshift_table_name, :source_table_filter
 
     def redshift_column_names
       redshift_columns.keys

data/lib/redshifter/util/extract_and_transform_updates.rb

@@ -74,7 +74,7 @@ module Redshifter
       end
 
       def select_batch_sql(columns:, batch_size:, start_id:)
-        "select #{columns.join(', ')} from #{table.source_table_name} where updated_at >= '#{since}' AND id >= #{start_id} ORDER BY id ASC limit #{batch_size}"
+        "SELECT #{columns.join(', ')} FROM #{table.source_table_name} WHERE (#{table.source_table_filter}) AND updated_at >= '#{since}' AND id >= #{start_id} ORDER BY id ASC limit #{batch_size}"
       end
     end
   end

data/lib/redshifter/version.rb

@@ -1,3 +1,3 @@
 module Redshifter
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshifter
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Justin Richard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-04 00:00:00.000000000 Z
+date: 2016-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dynosaur
@@ -173,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: ETL processing jobs to exporting Rails model tables to Redshift