firehose_integration 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba2eded2ae69513146014504efed4d2aaa8bc666
-  data.tar.gz: 91994144647e8daeaab259af8f9e0dcfa7242b5a
+  metadata.gz: 6c5cb0ed37b314dbdc6f3cf1c4ace1f576a06885
+  data.tar.gz: 414b0fb0b90463232e45bb1dfc2391b52e9cea45
 SHA512:
-  metadata.gz: b5604bb6df7f159e31c5837603cce360fbf390a0d66f59e0a6eb62874d0c2643637bb2d047774e8eafd97851411b5205cc707d95efab9d5ad7975f6fbc481197
-  data.tar.gz: 0a75b00665f45736d94974a8ca275b4960eb8b3f554b3fa0c69472a5e3cfdfed02eef6760e18ae0ebf56f84af0af7f24c526467fcf61d9d65619bddfa5a18be2
+  metadata.gz: a87d42534fd683aaa5f1f3cf8708041d9e65d4015480123bcba6de84a506a374a101f9411263cfa81165778297f3d47c9a98498537c32aab79afe7d4aa5927ae
+  data.tar.gz: b6ebc256144c58320ea80da0dd136ff7c7a0b870b0477e8d8b2b06f355fe95c5390c776ab2c96d40d3ffcfa950d5e1ab4075386ff98019e10fe506ccf99db3f1

data/README.rdoc

@@ -1,3 +1,146 @@
 = FirehoseIntegration
 
-This project rocks and uses MIT-LICENSE.
+This gem will send data to Firehose every time your models are updated.
+The general idea is to use a separate stream for each model you want to serialize.
+This essentially simplifies data syncing with Amazon Redshift via Firehose.
+
+= Basic Usage
+
+Add the following to your Gemfile and bundle install
+
+  gem 'firehose-integration'
+
+Create and configure config/initializers/aws.rb
+
+  require 'aws-sdk'
+
+  Aws.config.update({
+    region: 'us-east-1',
+    credentials: Aws::Credentials.new('akid', 'secret')
+  })
+
+For each model you want to integrate create a kinesis serializer
+
+  # apps/kinesis_serializers/dummy_model_kinesis_serializer.rb
+  module DummyModelKinesisSerializer
+    extend ActiveSupport::Concern
+
+    included do
+      def to_kinesis
+        prepare_for_redshift [
+          id,
+          updated_at,
+          created_at
+        ]
+      end
+
+      def self.kinesis_stream_name
+        'dummy-stream'
+      end
+    end
+  end
+
+Add the following line to the model you want to integrate.
+
+  # app/models/dummy_model.rb
+  class DummyModel < ActiveRecord::Base
+    firehose_integratable
+  end
+
+You will need to make sure you have the streams you're using created in Amazon Firehose.
+For the DummyModel example here we'd setup a 'dummy-stream' in Firehose.
+
+= Advanced Usage
+
+There are cases where simple 1-to-1 serialization from your existing tables won't work with Redshift.
+One such case is if you use Postgres array datatypes. Redshift doesn't support that datatype so you will
+need to normalize the data again when putting it in Redshift. To handle these edge cases you create a
+'kinesis_extra_serialization' method which will be executed when your model is updated if present.
+
+  # apps/kinesis_serializers/dummy_model_kinesis_serializer.rb
+  module DummyModelKinesisSerializer
+    extend ActiveSupport::Concern
+
+    included do
+      def to_kinesis
+        prepare_for_redshift [
+          id,
+          updated_at,
+          created_at
+        ]
+      end
+
+      def self.kinesis_stream_name
+        'dummy-stream'
+      end
+
+      def kinesis_extra_serialization
+        self.book_ids.each do |book_id|
+          data = [id, book_id, created_at, updated_at].join("|")
+          FirehoseIntegration::KinesisJob.perform_later("dummy-books", data)
+        end
+      end
+    end
+  end
+
+Another common case you may encounter is the need to denormalize data into other tables. Lets assume
+you have a dummy_models table that you want but you also have a denormalized_dummy_models table you need
+to keep in sync when your model is updated.
+
+  # apps/kinesis_serializers/dummy_model_kinesis_serializer.rb
+  module DummyModelKinesisSerializer
+    extend ActiveSupport::Concern
+    include DenormalizedDummyModelKinesisSerializer
+
+    included do
+      def to_kinesis
+        prepare_for_redshift [
+          id,
+          updated_at,
+          created_at
+        ]
+      end
+
+      def self.kinesis_stream_name
+        'dummy-stream'
+      end
+
+      def kinesis_extra_serialization
+        serialize_denormalized_dummy_model(id)
+      end
+    end
+  end
+
+  # apps/kinesis_serializers/denormalized_dummy_model_kinesis_serializer.rb
+  module DenormalizedDummyModelKinesisSerializer
+    extend ActiveSupport::Concern
+
+    included do
+      def denormalized_dummy_model_to_kinesis(dummy_id)
+        prepare_for_redshift [ id,
+          created_at,
+          updated_at
+        ]
+      end
+
+      def self.denormalized_kinesis_stream_name
+        "denormalized-dummy-model"
+      end
+
+      def serialize_denormalized_dummy_model(dummy_id)
+        FirehoseIntegration::KinesisJob.perform_later(self.class.denormalized_kinesis_stream_name, self.denormalized_dummy_model_to_kinesis(dummy_id))
+      end
+    end
+  end
+
+If you'd like to turn the syncing off for some reason like when running on a staging server, just set the following ENV variable to true:
+
+SKIP_KINESIS_EVENTS=true
+
+= Versions
+
+0.0.2 - Adding ability to turn off job queueing with ENV variable
+0.0.1 - Initial release
+
+= License
+This project rocks and uses MIT-LICENSE.

data/lib/firehose_integration/models/concerns/kinesis_event.rb

@@ -6,7 +6,7 @@ module FirehoseIntegration
 
     module ClassMethods
       def firehose_integratable
-        after_commit :send_kinesis_event, unless: Proc.new { |instance| instance.try(:skip_kinesis_event) }
+        after_commit :send_kinesis_event, unless: Proc.new { |instance| instance.try(:skip_kinesis_event) || ENV['SKIP_KINESIS_EVENTS'] == 'true' }
 
         begin
           include "#{self.model_name.name}KinesisSerializer".constantize

data/lib/firehose_integration/version.rb

@@ -1,3 +1,3 @@
 module FirehoseIntegration
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/test/dummy/app/models/dummy_model.rb

@@ -1,3 +1,3 @@
-class DummyModel < ActiveRecord::Base # Inherit from any actual model that doesn't have Kinesis included
+class DummyModel < ActiveRecord::Base
   firehose_integratable
 end

data/test/dummy/app/models/stupid_model.rb

@@ -1,3 +1,3 @@
-class StupidModel < ActiveRecord::Base # Inherit from any actual model that doesn't have Kinesis included
+class StupidModel < ActiveRecord::Base
   firehose_integratable
 end