magic_pipe 0.1.0

data/lib/magic_pipe/loaders.rb

@@ -0,0 +1,22 @@
+module MagicPipe
+  module Loaders
+    def self.lookup(type)
+      case type
+      when :simple_active_record then SimpleActiveRecord
+      when Class then type
+      else
+        raise ConfigurationError, "Unknown MagicPipe::Loaders type: '#{type}'."
+      end
+    end
+  end
+end
+
+files = File.expand_path("../loaders/**/*.rb", __FILE__)
+Dir[files].each do |f|
+  begin
+    require f
+  rescue LoadError
+    # Some components have extra dependencies
+  end
+end
+

data/lib/magic_pipe/loaders/simple_active_record.rb

@@ -0,0 +1,54 @@
+module MagicPipe
+  module Loaders
+    class SimpleActiveRecord
+      def initialize(record, wrapper=nil)
+        @record = record
+        @wrapper = wrapper
+      end
+
+      attr_reader :record
+
+      def decompose
+        {
+          klass: @record.class.to_s,
+          id: @record.id,
+          wrapper: (@wrapper && @wrapper.to_s),
+        }
+      end
+
+
+      class << self
+        def load(decomposed_data)
+          input = symbolize_keys(decomposed_data)
+          record_klass_name = input[:klass]
+          record_id = input[:id]
+          wrapper_klass_name = input[:wrapper]
+
+          record_klass = load_constant!(record_klass_name, "ActiveRecord model")
+          record = record_klass.find(record_id) # let it raise ActiveRecord::RecordNotFound
+
+          if wrapper_klass_name
+            wrapper_klass = load_constant!(wrapper_klass_name, "object serializer")
+            wrapper_klass.new(record)
+          else
+            record
+          end
+        end
+
+        private
+
+        def load_constant!(name, context)
+          Object.const_get(name)
+        rescue NameError
+          raise MagicPipe::LoaderError.new(name, context)
+        end
+
+        def symbolize_keys(hash)
+          hash.map do |k, v|
+            [k.to_sym, v]
+          end.to_h
+        end
+      end
+    end
+  end
+end

data/lib/magic_pipe/metrics.rb

@@ -0,0 +1,54 @@
+module MagicPipe
+  class Metrics
+    def initialize(config)
+      @client = config.metrics_client
+      @default_tags = build_default_tags(config)
+    end
+
+    attr_reader :client
+
+    def increment(metric, tags: [])
+      @client.increment(metric, tags: all_tags(tags))
+    end
+
+
+    private
+
+
+    def all_tags(list)
+      @default_tags + list
+    end
+
+    def build_default_tags(config)
+      list = [
+        "producer:#{config.producer_name.to_s.gsub(" ", "_")}",
+        "pipe_instance:#{config.client_name.to_s}",
+        "loader:#{config.loader.to_s}",
+        "codec:#{config.codec.to_s}",
+        "transport:#{transport_tag(config)}",
+        "sender:#{config.sender.to_s}",
+      ]
+    end
+
+    def transport_tag(config)
+      t = config.transport
+      if t.is_a?(Array)
+        "multi_" + t.map { |s| sanitize_tag_string(s) }.join("-")
+      else
+        t.to_s
+      end
+    end
+
+    def sanitize_tag_string(value)
+      value.to_s.tr(":. /", "")
+    end
+
+    def method_missing(name, *args, &block)
+      client.public_send(name, *args, &block)
+    end
+
+    def respond_to_missing?(name, include_all)
+      client.respond_to?(name, include_all)
+    end
+  end
+end

data/lib/magic_pipe/senders.rb

@@ -0,0 +1,23 @@
+module MagicPipe
+  module Senders
+    def self.lookup(type)
+      case type
+      when :sync then Sync
+      when :async then Async
+      when Class then type
+      else
+        raise ConfigurationError, "Unknown MagicPipe::Senders type: '#{type}'."
+      end
+    end
+  end
+end
+
+files = File.expand_path("../senders/**/*.rb", __FILE__)
+Dir[files].each do |f|
+  begin
+    require f
+  rescue LoadError
+    # Some components have extra dependencies
+  end
+end
+

data/lib/magic_pipe/senders/async.rb

@@ -0,0 +1,76 @@
+require 'sidekiq'
+require "magic_pipe/senders/base"
+require "magic_pipe/senders/metrics_mixin"
+
+module MagicPipe
+  module Senders
+    class Async < Base
+      class Worker
+        include Sidekiq::Worker
+        include Senders::MetricsMixin
+
+        def perform(decomposed_object, topic, time, client_name)
+          client = MagicPipe.lookup_client(client_name)
+          object = client.loader.load(decomposed_object)
+          codec = client.codec
+
+          metadata = {
+            topic: topic,
+            producer: client.config.producer_name,
+            time: time.to_i,
+            mime: codec::TYPE
+          }
+
+          envelope = Envelope.new(
+            body: object,
+            **metadata
+          )
+
+          payload = codec.new(envelope).encode
+          client.transport.submit(payload, metadata)
+
+          track_success(client.metrics, topic)
+        rescue => e
+          track_failure(client.metrics, topic)
+          raise e
+        end
+      end
+
+
+      SETTINGS = {
+        "class" => Worker,
+        "retry" => true
+      }
+
+      def call
+        enqueue
+      end
+
+      def enqueue
+        options = SETTINGS.merge({
+          "queue" => queue_name,
+          "args" => [
+            decomposed_object,
+            @topic,
+            @time.to_i,
+            @config.client_name
+          ]
+        })
+        Sidekiq::Client.push(options)
+      end
+
+
+      private
+
+
+      def queue_name
+        @config.async_transport_options[:queue]
+      end
+
+      def decomposed_object
+        loader = MagicPipe::Loaders.lookup(@config.loader)
+        loader.new(@object, @wrapper).decompose
+      end
+    end
+  end
+end

data/lib/magic_pipe/senders/base.rb

@@ -0,0 +1,24 @@
+module MagicPipe
+  module Senders
+    class Base
+      # object should be something similar
+      # to an ActiveModel::Serializer or
+      # ActiveRecord object.
+      #
+      def initialize(object, topic, wrapper, time, codec, transport, config, metrics)
+        @object = object
+        @topic = topic
+        @wrapper = wrapper
+        @time = time
+        @codec = codec
+        @transport = transport
+        @config = config
+        @metrics = metrics
+      end
+
+      def call
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/magic_pipe/senders/metrics_mixin.rb

@@ -0,0 +1,20 @@
+module MagicPipe
+  module Senders
+    module MetricsMixin
+      def track_success(metrics, topic)
+        metrics.increment(
+          "magic_pipe.senders.mgs_sent",
+          tags: ["topic:#{topic}"]
+        )
+      end
+
+      def track_failure(metrics, topic)
+        metrics.increment(
+          "magic_pipe.senders.failure",
+          tags: ["topic:#{topic}"]
+        )
+      end
+
+    end
+  end
+end

data/lib/magic_pipe/senders/sync.rb

@@ -0,0 +1,38 @@
+require "magic_pipe/senders/base"
+require "magic_pipe/senders/metrics_mixin"
+
+module MagicPipe
+  module Senders
+    class Sync < Base
+      include MetricsMixin
+
+      def call
+        metadata = build_metadata
+        envelope = build_message(metadata)
+        payload = @codec.new(envelope).encode
+        @transport.submit(payload, metadata)
+        track_success(@metrics, @topic)
+      rescue => e
+        track_failure(@metrics, @topic)
+        raise e
+      end
+
+      def build_message(metadata)
+        Envelope.new(body: data, **metadata)
+      end
+
+      def build_metadata
+        {
+          topic: @topic,
+          producer: @config.producer_name,
+          time: @time.to_i,
+          mime: @codec::TYPE
+        }
+      end
+
+      def data
+        @wrapper ? @wrapper.new(@object) : @object
+      end
+    end
+  end
+end

data/lib/magic_pipe/transports.rb

@@ -0,0 +1,26 @@
+module MagicPipe
+  module Transports
+    def self.lookup(type)
+      case type
+      when :https then Https
+      when :sqs then Sqs
+      when :log then Log
+      when :debug then Debug
+      when Array then Multi
+      when Class then type
+      else
+        raise ConfigurationError, "Unknown MagicPipe::Transports type: '#{type}'."
+      end
+    end
+  end
+end
+
+files = File.expand_path("../transports/**/*.rb", __FILE__)
+Dir[files].each do |f|
+  begin
+    require f
+  rescue LoadError
+    # Some components have extra dependencies
+  end
+end
+

data/lib/magic_pipe/transports/base.rb

@@ -0,0 +1,17 @@
+module MagicPipe
+  module Transports
+    class Base
+      def initialize(config, metrics)
+        @config = config
+        @metrics = metrics
+        @logger = @config.logger
+      end
+
+      attr_reader :metrics, :logger
+
+      def submit(payload, metadata)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/magic_pipe/transports/debug.rb

@@ -0,0 +1,17 @@
+require "magic_pipe/transports/base"
+
+module MagicPipe
+  module Transports
+    class Debug < Base
+      def initialize(*)
+      end
+
+      def submit(payload, metadata)
+        $magic_pipe_out = {
+          payload: payload,
+          metadata: metadata
+        }
+      end
+    end
+  end
+end

data/lib/magic_pipe/transports/https.rb

@@ -0,0 +1,80 @@
+require "magic_pipe/transports/base"
+
+require "faraday"
+require "typhoeus"
+require "typhoeus/adapters/faraday"
+
+module MagicPipe
+  module Transports
+    class Https < Base
+      def initialize(config, metrics)
+        super(config, metrics)
+        @options = @config.https_transport_options
+        @conn = build_connection
+      end
+
+      attr_reader :conn
+
+
+      # TODO: should this raise an error on failure?
+      # So that it can be retried?
+      #
+      def submit(payload, metadata)
+        @conn.post do |r|
+          r.body = payload
+          r.headers["X-MagicPipe-Sent-At"] = metadata[:time]
+          r.headers["X-MagicPipe-Topic"] = metadata[:topic]
+          r.headers["X-MagicPipe-Producer"] = metadata[:producer]
+        end
+      end
+
+
+      private
+
+      def url
+        @options.fetch(:url)
+      end
+
+      def auth_token
+        @options.fetch(:auth_token)
+      end
+
+      def timeout
+        @options.fetch(:timeout)
+      end
+
+      def open_timeout
+        @options.fetch(:open_timeout)
+      end
+
+      def content_type
+        MagicPipe::Codecs.lookup(@config.codec)::TYPE
+      end
+
+      def user_agent
+        "MagicPipe v%s (Faraday v%s, Typhoeus v%s)" % [
+          MagicPipe::VERSION,
+          Faraday::VERSION,
+          Typhoeus::VERSION
+        ]
+      end
+
+      # For a single backend, can't this be cached as a read only global?
+      #
+      def build_connection
+        Faraday.new(url) do |f|
+          f.request :retry, max: 2, interval: 0.1, backoff_factor: 2
+          f.request :basic_auth, auth_token, 'x'
+
+          f.headers['Content-Type'] = content_type
+          f.headers['User-Agent'] = user_agent
+
+          f.options.timeout = timeout
+          f.options.open_timeout = open_timeout
+
+          f.adapter :typhoeus
+        end
+      end
+    end
+  end
+end