rails-pipeline 1.1.1

data/lib/rails-pipeline/emitter.rb

@@ -0,0 +1,121 @@
+# A Pipeline emitter is an active record model that, when changed,
+# will publish the changed fields to some sort of queue
+
+require "rails-pipeline/symmetric_encryptor"
+
+module RailsPipeline
+  module Emitter
+
+    def self.included(base)
+      RailsPipeline::SymmetricEncryptor.included(base)
+      base.send :include, InstanceMethods
+      base.extend ClassMethods
+      base.after_commit :emit_on_create, on: :create, if: :persisted?
+      base.after_commit :emit_on_update, on: :update
+      base.after_commit :emit_on_destroy, on: :destroy
+
+      if RailsPipeline::HAS_NEWRELIC
+        base.send :include, ::NewRelic::Agent::MethodTracer
+        base.add_method_tracer :emit, 'Pipeline/Emitter/emit'
+      end
+    end
+
+    module InstanceMethods
+      def emit_on_create
+        emit(:create)
+      end
+
+      def emit_on_update
+        emit(:update)
+      end
+
+      def emit_on_destroy
+        emit(:destroy)
+      end
+
+      def emit(event_type = RailsPipeline::EncryptedMessage::EventType::CREATED)
+        if ENV.has_key?("DISABLE_RAILS_PIPELINE") || ENV.has_key?("DISABLE_RAILS_PIPELINE_EMISSION")
+          RailsPipeline.logger.debug "Skipping outgoing pipeline messages (disabled by env vars)"
+          return
+        end
+        begin
+          self.class.pipeline_versions.each do |version|
+            enc_data = create_message(version, event_type)
+            self.publish(enc_data.topic, enc_data.to_s)
+          end
+        rescue Exception => e
+          RailsPipeline.logger.error("Error during emit(): #{e}")
+          puts e.backtrace.join("\n")
+          raise e
+        end
+      end
+
+      def create_message(version, event_type)
+        topic = self.class.topic_name(version)
+        RailsPipeline.logger.debug "Emitting to #{topic}"
+        data = self.send("to_pipeline_#{version}")
+        enc_data = self.class.encrypt(data.to_s, type_info: data.class.name, topic: topic, event_type: event_type)
+        return enc_data
+      end
+
+    end
+
+    module ClassMethods
+      # Get the list of versions to emit (all that are implemented, basically)
+      def pipeline_versions
+        if pipeline_method_cache.any?
+          return pipeline_method_cache.keys
+        end
+        versions = []
+        pipeline_methods = instance_methods.grep(/^to_pipeline/).sort
+        pipeline_methods.each do |pipeline_method|
+          version = pipeline_method.to_s.gsub("to_pipeline_", "")
+          if version.starts_with?("1_") && version != "1_0"
+            # Delete the default v1.0 emitter if a later v1 emitter is defined
+            i = versions.index("1_0")
+            versions.delete_at(i) if !i.nil?
+          end
+          pipeline_method_cache[version] = pipeline_method
+          versions << version
+        end
+        return versions
+      end
+
+      # Get pub/sub topic name to which changes to this model will be published
+      def topic_name(version="1_0")
+        return "harrys-#{Rails.env}-v#{major(version)}-#{table_name}"
+      end
+
+      # Get the major version number from a "1_1" style major/minor version string
+      def major(version)
+        if version.include? '_'
+          major, _ = version.split('_', 2)
+        else
+          major = version
+        end
+        return major
+      end
+
+      def _secret
+        ENV.fetch("PIPELINE_SECRET", Rails.application.config.secret_token)
+      end
+
+      def pipeline_method_cache
+        @pipeline_method_cache ||= {}
+      end
+
+      def pipeline_method_cache=(cache)
+        @pipeline_method_cache = cache
+      end
+
+    end
+  end
+
+
+  module RedisEmitter
+    def self.included(base)
+      RailsPipeline::Emitter.included(base)
+      RailsPipeline::RedisPublisher.included(base)
+    end
+  end
+end

data/lib/rails-pipeline/handlers/activerecord_crud.rb

@@ -0,0 +1,35 @@
+module RailsPipeline
+  module SubscriberHandler
+    class ActiveRecordCRUD < BaseHandler
+      def handle_payload
+        begin
+          case event_type
+          when RailsPipeline::EncryptedMessage::EventType::CREATED
+            return target_class.create!(_attributes(payload), without_protection: true)
+          when RailsPipeline::EncryptedMessage::EventType::UPDATED
+            # We might want to allow confiugration of the primary key field
+            object = target_class.find(payload.id)
+            object.update_attributes!(_attributes(payload), without_protection: true)
+            return object
+          when RailsPipeline::EncryptedMessage::EventType::DELETED
+            object = target_class.find(payload.id)
+            object.destroy
+            return object
+          end
+        rescue ActiveRecord::StatementInvalid, ActiveRecord::RecordNotFound => e
+          RailsPipeline.logger.error "Could not handle payload: #{payload.inspect}, event_type: #{event_type}"
+        end
+      end
+
+      def _attributes(payload)
+        attributes_hash = payload.to_hash
+        attributes_hash.each do |attribute_name, value|
+          if attribute_name.match /_at$/
+            attributes_hash[attribute_name] = Time.at(value).to_datetime
+          end
+        end
+        return attributes_hash
+      end
+    end
+  end
+end

data/lib/rails-pipeline/handlers/base_handler.rb

@@ -0,0 +1,19 @@
+module RailsPipeline
+  module SubscriberHandler
+    class BaseHandler
+      attr_reader :payload, :event_type, :target_class, :envelope
+
+      def initialize(payload, target_class: nil, envelope: nil, event_type: nil)
+        @payload = payload
+        @target_class = target_class
+        @envelope = envelope
+        if envelope
+          @event_type = envelope.event_type
+        else
+          @event_type = event_type
+        end
+      end
+
+    end
+  end
+end

data/lib/rails-pipeline/handlers/logger.rb

@@ -0,0 +1,13 @@
+module RailsPipeline
+  module SubscriberHandler
+    class Logger < BaseHandler
+      def handle_payload
+        # We'll need to GPG encrypt this
+        # Maybe it should encrypt using a public key set in environment variable
+        # Then the subscriber app would set it when installing the pipeline
+        # Would need to add configuration for that
+        RailsPipeline.logger.info envelope.to_s
+      end
+    end
+  end
+end

data/lib/rails-pipeline/ironmq_publisher.rb

@@ -0,0 +1,37 @@
+require 'base64'
+require 'iron_mq'
+
+# Backend for data pipeline that publishes to IronMQ
+#
+# Assumes the following env vars:
+# - IRON_TOKEN=MY_TOKEN
+# - IRON_PROJECT_ID=MY_PROJECT_ID
+
+module RailsPipeline::IronmqPublisher
+
+  def self.included(base)
+    base.send :include, InstanceMethods
+    base.extend ClassMethods
+  end
+
+  module InstanceMethods
+    def publish(topic_name, data)
+      t0 = Time.now
+      queue = _iron.queue(topic_name)
+      queue.post({payload: Base64.strict_encode64(data)}.to_json)
+      t1 = Time.now
+      ::NewRelic::Agent.record_metric('Pipeline/IronMQ/publish', t1-t0) if RailsPipeline::HAS_NEWRELIC
+      RailsPipeline.logger.debug "Publishing to IronMQ: #{topic_name} took #{t1-t0}s"
+    end
+
+    def _iron
+      @iron = IronMQ::Client.new if @iron.nil?
+      return @iron
+    end
+
+  end
+
+  module ClassMethods
+  end
+
+end

data/lib/rails-pipeline/ironmq_pulling_subscriber.rb

@@ -0,0 +1,96 @@
+require 'base64'
+require 'json'
+
+module RailsPipeline
+    class IronmqPullingSubscriber
+        include RailsPipeline::Subscriber
+
+        attr_reader :queue_name
+
+        def initialize(queue_name)
+            @queue_name  = queue_name
+            @subscription_status = false
+        end
+
+        # Valid Parameters at this time are 
+        # wait_time - An integer indicating how long in seconds we should long poll on empty queues
+        # halt_on_error - A boolean indicating if we should stop our queue subscription if an error occurs
+        def start_subscription(params={wait_time: 2, halt_on_error: true}, &block)
+            activate_subscription
+
+            while active_subscription?
+                pull_message(params[:wait_time]) do |message|
+                    process_message(message, params[:halt_on_error], block)
+                end
+            end
+        end
+
+
+        def process_message(message, halt_on_error, block)
+            begin
+                if message.nil? || JSON.parse(message.body).empty?
+                    deactivate_subscription
+                else
+                    payload = parse_ironmq_payload(message.body)
+                    envelope = generate_envelope(payload)
+
+                    process_envelope(envelope, message, block)
+                end
+            rescue Exception => e
+                if halt_on_error
+                    deactivate_subscription
+                end
+
+                RailsPipeline.logger.error "A message was unable to be processed as was not removed from the queue."
+                RailsPipeline.logger.error "The message: #{message.inspect}"
+                raise e
+            end
+        end
+
+        def active_subscription?
+            @subscription_status
+        end
+
+        def activate_subscription
+            @subscription_status = true
+        end
+
+        def deactivate_subscription
+            @subscription_status = false
+        end
+
+        def process_envelope(envelope, message, block)
+            callback_status = block.call(envelope)
+
+            if callback_status
+                message.delete
+            end
+        end
+
+
+        #the wait time on this may need to be changed
+        #haven't seen rate limit info on these calls but didnt look
+        #all that hard either.
+        def pull_message(wait_time)
+            queue = _iron.queue(queue_name)
+            yield queue.get(:wait => wait_time)
+        end
+
+        private
+
+        def _iron
+            @iron = IronMQ::Client.new if @iron.nil?
+            return @iron
+        end
+
+        def parse_ironmq_payload(message_body)
+            payload = JSON.parse(message_body)["payload"]
+            Base64.strict_decode64(payload)
+        end
+
+        def generate_envelope(payload)
+            RailsPipeline::EncryptedMessage.parse(payload)
+        end
+
+    end
+end

data/lib/rails-pipeline/ironmq_subscriber.rb

@@ -0,0 +1,21 @@
+# Sinatra endpoints for consuming IronMQ push queues for the rails-pipeline
+
+require 'sinatra'
+require 'base64'
+
+module RailsPipeline
+  class IronmqSubscriber < Sinatra::Base
+    include RailsPipeline::Subscriber
+
+    post '/' do
+      t0 = Time.now
+      data = request.body.read
+      payload = JSON.parse(Base64.strict_decode64(data))['payload']
+      envelope = RailsPipeline::EncryptedMessage.parse(payload)
+      handle_envelope(envelope)
+      t1 = Time.now
+      RailsPipeline.logger.debug "Consuming from IronMQ: #{envelope.topic} took #{t1-t0}s"
+    end
+
+  end
+end

data/lib/rails-pipeline/pipeline_version.rb

@@ -0,0 +1,40 @@
+module RailsPipeline
+  # A thin wrapper around our version object
+  # A version has the form X_Y where
+  #   - X is the major version
+  #   - Y is the minor version
+  #
+  # example: 1_0, 2_1, etc...
+  class PipelineVersion
+
+    include Comparable
+
+    attr_reader :major, :minor
+
+    def initialize(version_string)
+      # raise error?
+      @major, @minor = version_string.split('_').map(&:to_i)
+    end
+
+    def to_s
+      "#{major}_#{minor}"
+    end
+
+    def <=>(other)
+      if major == other.major
+        return minor <=> other.minor
+      else
+        return major <=> other.major
+      end
+    end
+
+    def eql?(other)
+      return to_s.eql?(other.to_s)
+    end
+
+    def hash
+      return to_s.hash
+    end
+
+  end
+end

data/lib/rails-pipeline/protobuf/encrypted_message.pb.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+require 'protocol_buffers'
+
+module RailsPipeline
+  # forward declarations
+  class EncryptedMessage < ::ProtocolBuffers::Message; end
+
+  class EncryptedMessage < ::ProtocolBuffers::Message
+    # forward declarations
+
+    # enums
+    module EventType
+      include ::ProtocolBuffers::Enum
+
+      set_fully_qualified_name "RailsPipeline.EncryptedMessage.EventType"
+
+      CREATED = 0
+      UPDATED = 1
+      DELETED = 2
+    end
+
+    set_fully_qualified_name "RailsPipeline.EncryptedMessage"
+
+    required :string, :uuid, 1
+    required :string, :salt, 2
+    required :string, :iv, 3
+    required :string, :ciphertext, 4
+    optional :string, :owner_info, 5
+    optional :string, :type_info, 6
+    optional :string, :topic, 7
+    optional ::RailsPipeline::EncryptedMessage::EventType, :event_type, 8, :default => ::RailsPipeline::EncryptedMessage::EventType::CREATED
+    required :string, :api_key, 9
+  end
+
+end

data/lib/rails-pipeline/protobuf/encrypted_message.proto

@@ -0,0 +1,18 @@
+package RailsPipeline;
+
+message EncryptedMessage {
+	required string uuid = 1;
+	required string salt = 2;
+	required string iv = 3;
+	required string ciphertext = 4;
+	optional string owner_info = 5; // e.g. user_id, if you store a key on the User
+	optional string type_info = 6;  // ruby class name
+	optional string topic = 7;      // useful for redis message forwarder
+	enum EventType {
+		CREATED = 0;
+		UPDATED = 1;
+		DELETED = 2;
+	}
+	optional EventType event_type = 8 [default = CREATED];    // indicates what kind of event this payload contains
+	required string api_key = 9;
+}