firehose_integration 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ba2eded2ae69513146014504efed4d2aaa8bc666
+  data.tar.gz: 91994144647e8daeaab259af8f9e0dcfa7242b5a
+SHA512:
+  metadata.gz: b5604bb6df7f159e31c5837603cce360fbf390a0d66f59e0a6eb62874d0c2643637bb2d047774e8eafd97851411b5205cc707d95efab9d5ad7975f6fbc481197
+  data.tar.gz: 0a75b00665f45736d94974a8ca275b4960eb8b3f554b3fa0c69472a5e3cfdfed02eef6760e18ae0ebf56f84af0af7f24c526467fcf61d9d65619bddfa5a18be2

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 onomojo
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= FirehoseIntegration
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,34 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'FirehoseIntegration'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/lib/firehose_integration.rb

@@ -0,0 +1,9 @@
+require "firehose_integration/active_record_relation"
+
+require "firehose_integration/jobs/kinesis_job"
+require "firehose_integration/jobs/kinesis_bulk_job"
+require "firehose_integration/jobs/kinesis_single_object_job"
+
+require "firehose_integration/models/concerns/kinesis_event"
+module FirehoseIntegration
+end

data/lib/firehose_integration/active_record_relation.rb

@@ -0,0 +1,6 @@
+class ActiveRecord::Relation
+  def update_all_with_kinesis(params)
+    self.update_all params
+    FirehoseIntegration::KinesisSingleObjectJob.perform_later(ancestors.first.to_s, self.pluck(:id))
+  end
+end

data/lib/firehose_integration/jobs/kinesis_bulk_job.rb

@@ -0,0 +1,22 @@
+module FirehoseIntegration
+  class KinesisBulkJob < ActiveJob::Base
+    queue_as :kinesis_events_bulk
+    def perform(stream, data)
+      client = Aws::Firehose::Client.new(region:'us-east-1')
+
+      records = []
+      data.each do |d|
+        records << {
+          data: "#{d}\n"
+        }
+      end
+
+      params = {
+        delivery_stream_name: stream,
+        records: records
+      }
+
+      client.put_record_batch params
+    end
+  end
+end

data/lib/firehose_integration/jobs/kinesis_job.rb

@@ -0,0 +1,16 @@
+module FirehoseIntegration
+  class KinesisJob < ActiveJob::Base
+    queue_as :kinesis_events
+    def perform(stream, data)
+      client = Aws::Firehose::Client.new(region:'us-east-1')
+
+      params = {
+        delivery_stream_name: stream,
+        record: {
+          data: "#{data}\n"
+        }
+      }
+      client.put_record params
+    end
+  end
+end

data/lib/firehose_integration/jobs/kinesis_single_object_job.rb

@@ -0,0 +1,24 @@
+module FirehoseIntegration
+  class KinesisSingleObjectJob < ActiveJob::Base
+    queue_as :kinesis_events_single
+
+    def perform(class_name, ids)
+      client = Aws::Firehose::Client.new(region:'us-east-1')
+      results = []
+      ids.each do |id|
+        object = class_name.constantize.find(id)
+        stream = object.class.kinesis_stream_name
+        data = object.to_kinesis
+
+        params = {
+          delivery_stream_name: stream,
+          record: {
+            data: "#{data}\n"
+          }
+        }
+        results << client.put_record(params)
+      end
+      results
+    end
+  end
+end

data/lib/firehose_integration/models/concerns/kinesis_event.rb

@@ -0,0 +1,71 @@
+module FirehoseIntegration
+  module KinesisEvent
+    extend ActiveSupport::Concern
+
+    MAX_REDSHIFT_STRING_SIZE = 65535
+
+    module ClassMethods
+      def firehose_integratable
+        after_commit :send_kinesis_event, unless: Proc.new { |instance| instance.try(:skip_kinesis_event) }
+
+        begin
+          include "#{self.model_name.name}KinesisSerializer".constantize
+        rescue
+        end
+      end
+
+      def kinesis_stream_name
+        raise(NoMethodError, "Model must define class method kinesis_stream_name")
+      end
+    end
+
+    included do
+
+      def to_kinesis
+        raise(NoMethodError, "Model must define instance method to_kinesis")
+      end
+
+      def prepare_for_redshift(field)
+        if field.present?
+          if field.is_a? Array
+            output = []
+            field.each do |f|
+              output << massage_data_for_redshift(f)
+            end
+            return output.join("|")
+          end
+        end
+      end
+
+      def massage_data_for_redshift field
+        if field.is_a? String
+          output = escape_string_for_redshift(field).truncate(MAX_REDSHIFT_STRING_SIZE)
+          Rails.logger.info "Redshift data has been truncated due to length: #{output.truncate(50)}" if output.size == MAX_REDSHIFT_STRING_SIZE
+          output
+        else
+          field
+        end
+      end
+
+      def escape_string_for_redshift field
+        return field unless field.is_a? String
+        output = field
+        {
+          '\n' => '\\n',
+          '\r' => '\\r',
+          '\|' => '&verbar;'
+        }.each do |pattern, replacement|
+          output = output.gsub(Regexp.new(pattern), replacement)
+        end
+        output
+      end
+
+
+      def send_kinesis_event
+        KinesisJob.perform_later(self.class.kinesis_stream_name, self.to_kinesis)
+        self.kinesis_extra_serialization if self.methods.include? :kinesis_extra_serialization
+      end
+    end
+  end
+end
+ActiveRecord::Base.send :include, FirehoseIntegration::KinesisEvent

data/lib/firehose_integration/version.rb

@@ -0,0 +1,3 @@
+module FirehoseIntegration
+  VERSION = "0.0.1"
+end

data/lib/tasks/firehose_integration_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :firehose_integration do
+#   # Task goes here
+# end

data/test/cassettes/kinesis_bulk_failure.yml

@@ -0,0 +1,55 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://firehose.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: '{"DeliveryStreamName":"invalid-stream-name","Records":[{"Data":"c29tZSBkYXRhCg=="}]}'
+    headers:
+      Content-Type:
+      - application/x-amz-json-1.1
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby2/2.3.8 ruby/2.2.1 x86_64-darwin15
+      X-Amz-Target:
+      - Firehose_20150804.PutRecordBatch
+      X-Amz-Date:
+      - 20160526T204248Z
+      Host:
+      - firehose.us-east-1.amazonaws.com
+      X-Amz-Content-Sha256:
+      - 51d39395cb7e526eb7bac24de83e3012137401a5d77dcaf4308119c1c69a8f33
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=akid/20160526/us-east-1/firehose/aws4_request,
+        SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target,
+        Signature=bb4def5cb89830d9f160f15e40b4327f556bd719d7b9cc1c06084bf43e342ef2
+      Content-Length:
+      - '84'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 400
+      message: Bad Request
+    headers:
+      X-Amzn-Requestid:
+      - 69cc5bea-2382-11e6-aaa0-171acafcc56d
+      X-Amz-Id-2:
+      - sJghDMvGF8Lt4ULOjnCoI0pj42bU/9gRZ+uLaCTZCDaRcKRyVHnr9GcucTdP8S8fgkWwPsVY3+o8tUzVXoMzHg==
+      Content-Type:
+      - application/x-amz-json-1.1
+      Content-Length:
+      - '117'
+      Date:
+      - Thu, 26 May 2016 20:42:50 GMT
+      Connection:
+      - close
+    body:
+      encoding: UTF-8
+      string: '{"__type":"ResourceNotFoundException","message":"Firehose invalid-stream-name
+        not found under account 000111222333."}'
+    http_version:
+  recorded_at: Thu, 26 May 2016 20:42:48 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kinesis_bulk_success.yml

@@ -0,0 +1,52 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://firehose.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: '{"DeliveryStreamName":"test-stream","Records":[{"Data":"c29tZSBkYXRhCg=="},{"Data":"bW9yZSBkYXRhCg=="}]}'
+    headers:
+      Content-Type:
+      - application/x-amz-json-1.1
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby2/2.3.8 ruby/2.2.1 x86_64-darwin15
+      X-Amz-Target:
+      - Firehose_20150804.PutRecordBatch
+      X-Amz-Date:
+      - 20160526T204248Z
+      Host:
+      - firehose.us-east-1.amazonaws.com
+      X-Amz-Content-Sha256:
+      - 46abc6a5b3638e3d8b76072597aade6f9ebad6eee1e4935c232a1be5e3842214
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=akid/20160526/us-east-1/firehose/aws4_request,
+        SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target,
+        Signature=bad19647525c88c8307912d614088b00d9c26a644934eac53c62530eacef1ef6
+      Content-Length:
+      - '104'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amzn-Requestid:
+      - 6a098c7c-2382-11e6-9ea8-514fc73df602
+      X-Amz-Id-2:
+      - 8ZperIhPiVSMSHSeCAxfkbqbqgyofdOwaTrcrhvi8UP/ocNmmlKYbSvsAoFKhjVO0qH0XHXF3SoaETk+QvZhjw==
+      Content-Type:
+      - application/x-amz-json-1.1
+      Content-Length:
+      - '90'
+      Date:
+      - Thu, 17 Mar 2016 15:43:58 GMT
+    body:
+      encoding: UTF-8
+      string: '{"FailedPutCount":0,"RequestResponses":[{"RecordId":"SOMEID"},{"RecordId":"SOMEOTHERID"}]}'
+    http_version:
+  recorded_at: Thu, 17 Mar 2016 15:43:58 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kinesis_failure.yml

@@ -0,0 +1,55 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://firehose.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: '{"DeliveryStreamName":"invalid-stream-name","Record":{"Data":"c29tZSBkYXRhCg=="}}'
+    headers:
+      Content-Type:
+      - application/x-amz-json-1.1
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby2/2.3.8 ruby/2.2.1 x86_64-darwin15
+      X-Amz-Target:
+      - Firehose_20150804.PutRecord
+      X-Amz-Date:
+      - 20160526T210311Z
+      Host:
+      - firehose.us-east-1.amazonaws.com
+      X-Amz-Content-Sha256:
+      - 174411b44d27135ef894b2bfc487dacf8753738bc6070405ece770a2c5acfd65
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=akid/20160526/us-east-1/firehose/aws4_request,
+        SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target,
+        Signature=ae6474afcfb57bd28f8540c6e43fb12bae6cd41fe2db9b1994ce47c514b87703
+      Content-Length:
+      - '81'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 400
+      message: Bad Request
+    headers:
+      X-Amzn-Requestid:
+      - 42fabd77-2385-11e6-be61-91f005091ebf
+      X-Amz-Id-2:
+      - XKgPSA6n5lN6sk6tvNz+5wUS46riOeVei6jvqCPucXpQEy2/EzjzpzwHn8hknKQg8dWvq9iVcAjfH66qN0C4JdkRjEdnQQFn
+      Content-Type:
+      - application/x-amz-json-1.1
+      Content-Length:
+      - '117'
+      Date:
+      - Thu, 26 May 2016 21:03:14 GMT
+      Connection:
+      - close
+    body:
+      encoding: UTF-8
+      string: '{"__type":"ResourceNotFoundException","message":"Firehose invalid-stream-name
+        not found under account 000111222333."}'
+    http_version:
+  recorded_at: Thu, 26 May 2016 21:03:12 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kinesis_success.yml

@@ -0,0 +1,52 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://firehose.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: '{"DeliveryStreamName":"some-stream","Record":{"Data":"c29tZSBkYXRhCg=="}}'
+    headers:
+      Content-Type:
+      - application/x-amz-json-1.1
+      Accept-Encoding:
+      - ''
+      User-Agent:
+      - aws-sdk-ruby2/2.3.8 ruby/2.2.1 x86_64-darwin15
+      X-Amz-Target:
+      - Firehose_20150804.PutRecord
+      X-Amz-Date:
+      - 20160526T205525Z
+      Host:
+      - firehose.us-east-1.amazonaws.com
+      X-Amz-Content-Sha256:
+      - b770027680e36df60472bda3598bc24ab1b28083f6c0bae0f858715d2830ed83
+      Authorization:
+      - AWS4-HMAC-SHA256 Credential=akid/20160526/us-east-1/firehose/aws4_request,
+        SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target,
+        Signature=5dd75f094a5b67f726c7ea3333102a3d9a870975e9357efa48d13c35ae5e26c6
+      Content-Length:
+      - '73'
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      X-Amzn-Requestid:
+      - 2cfe307f-2384-11e6-9868-69152542a0c3
+      X-Amz-Id-2:
+      - GWN297SaqQ9EdAgTzh9MzxSU3OFyemuVtiiKIz+idFwspK9imwSwhLzEM3jjVqTl8joR1DcvfFO4riIj0spnOfjRoBqHS5cC
+      Content-Type:
+      - application/x-amz-json-1.1
+      Content-Length:
+      - '21'
+      Date:
+      - Thu, 26 May 2016 20:55:27 GMT
+    body:
+      encoding: UTF-8
+      string: '{"RecordId":"SOMEID"}'
+    http_version:
+  recorded_at: Thu, 26 May 2016 20:55:25 GMT
+recorded_with: VCR 2.9.3