magic_pipe 0.1.0

data/lib/magic_pipe.rb

@@ -0,0 +1,45 @@
+require "magic_pipe/version"
+require "magic_pipe/errors"
+
+require "magic_pipe/config"
+require "magic_pipe/metrics"
+
+require "magic_pipe/envelope"
+
+require "magic_pipe/loaders"
+require "magic_pipe/codecs"
+require "magic_pipe/senders"
+require "magic_pipe/transports"
+
+require "magic_pipe/client"
+
+module MagicPipe
+  class << self
+    def lookup_client(name)
+      @store[name.to_sym]
+    end
+
+    # All this should be loaded before Sidekiq
+    # or Puma start forking threads.
+    #
+    def store_client(client)
+      @store ||= {}
+      @store[client.name.to_sym] = client
+    end
+
+    def clear_clients
+      @store = {}
+    end
+
+    def build(&block)
+      unless block_given?
+        raise ConfigurationError, "No configuration block provided."
+      end
+
+      config = Config.new(&block)
+      client = Client.new(config)
+      store_client(client)
+      client
+    end
+  end
+end

data/lib/magic_pipe/client.rb

@@ -0,0 +1,41 @@
+module MagicPipe
+  class Client
+    def initialize(config)
+      @config = config
+      @name = config.client_name
+
+      @metrics = Metrics.new(@config)
+
+      @transport = build_transport
+
+      @codec = Codecs.lookup(config.codec)
+      @sender = Senders.lookup(config.sender)
+
+      @loader = Loaders.lookup(config.loader)
+    end
+
+    attr_reader :name, :config, :codec, :transport, :sender, :loader, :metrics
+
+    def send_data(object:, topic:, wrapper: nil, time: Time.now.utc)
+      sender.new(
+        object,
+        topic,
+        wrapper,
+        time,
+        codec,
+        transport,
+        @config,
+        @metrics
+      ).call
+      true
+    end
+
+
+    private
+
+    def build_transport
+      klass = Transports.lookup(@config.transport)
+      klass.new(@config, @metrics)
+    end
+  end
+end

data/lib/magic_pipe/codecs.rb

@@ -0,0 +1,24 @@
+module MagicPipe
+  module Codecs
+    def self.lookup(type)
+      case type
+      when :json then Json
+      when :thrift then Thrift
+      when :msgpack, :message_pack then MessagePack
+      when :yaml then Yaml
+      when Class then type
+      else
+        raise ConfigurationError, "Unknown MagicPipe::Codecs type: '#{type}'."
+      end
+    end
+  end
+end
+
+files = File.expand_path("../codecs/**/*.rb", __FILE__)
+Dir[files].each do |f|
+  begin
+    require f
+  rescue LoadError
+    # Some components have extra dependencies
+  end
+end

data/lib/magic_pipe/codecs/base.rb

@@ -0,0 +1,26 @@
+module MagicPipe
+  module Codecs
+    class Base
+      TYPE = "none"
+
+      # object should be something similar
+      # to an ActiveModel::Serializer or
+      # ActiveRecord object.
+      #
+      def initialize(object)
+        @object = object
+      end
+
+      attr_reader :object
+      alias_method :o, :object
+
+      def encode
+        raise NotImplementedError
+      end
+
+      def type
+        self.class.const_get(:TYPE)
+      end
+    end
+  end
+end

data/lib/magic_pipe/codecs/json.rb

@@ -0,0 +1,38 @@
+require "magic_pipe/codecs/base"
+
+module MagicPipe
+  module Codecs
+    class Json < Base
+      TYPE = "application/json"
+
+      def encode
+        if o.respond_to?(:to_json)
+          o.to_json
+        elsif o.respond_to?(:as_json)
+          json_dump(o.as_json)
+        else
+          json_dump(o)
+        end
+      end
+
+
+      private
+
+
+      begin
+        require "oj"
+
+        def json_dump(data)
+          Oj.dump(data)
+        end
+      rescue LoadError
+        puts "[#{self.to_s}] The oj gem is not available. Using json from the stdlib."
+        require "json"
+
+        def json_dump(data)
+          JSON.dump(data)
+        end
+      end
+    end
+  end
+end

data/lib/magic_pipe/codecs/message_pack.rb

@@ -0,0 +1,81 @@
+require "magic_pipe/codecs/base"
+
+begin
+  require "msgpack"
+
+  module MagicPipe
+    module Codecs
+      class MessagePack < Base
+        TYPE = "application/x-msgpack"
+
+        def encode
+          case o
+          when Hash
+            o.to_msgpack
+          else
+            o.as_json.to_msgpack
+          end
+        end
+      end
+    end
+  end
+
+  # Extensions required to serialize time values
+
+  begin
+    # If used in Rails, most timestamps will be
+    # ActiveSupport::TimeWithZone instances.
+
+    require "active_support/time_with_zone"
+
+    ActiveSupport::TimeWithZone.class_eval do
+      def to_msgpack_ext
+        ("TWZ[" + self.to_s + "]").to_msgpack
+      end
+
+      def self.from_msgpack_ext(data)
+        Time.zone.parse(data)
+      end
+    end
+    MessagePack::DefaultFactory.register_type 0x42, ActiveSupport::TimeWithZone
+  rescue LoadError
+  end
+
+  require "time"
+  require "date"
+
+  Time.class_eval do
+    def to_msgpack_ext
+      to_i.to_msgpack
+    end
+
+    def self.from_msgpack_ext(data)
+      n = MessagePack.unpack(data)
+      Time.at(n)
+    end
+  end
+  MessagePack::DefaultFactory.register_type 0x43, Time
+
+  Date.class_eval do
+    def to_msgpack_ext
+      ("D[" + to_s + "]").to_msgpack
+    end
+
+    def self.from_msgpack_ext(data)
+      Date.parse(MessagePack.unpack(data))
+    end
+  end
+  MessagePack::DefaultFactory.register_type 0x44, Date
+
+  DateTime.class_eval do
+    def to_msgpack_ext
+      ("DT[" + to_s + "]").to_msgpack
+    end
+
+    def self.from_msgpack_ext(data)
+      DateTime.parse(MessagePack.unpack(data))
+    end
+  end
+  MessagePack::DefaultFactory.register_type 0x45, DateTime
+rescue LoadError
+end

data/lib/magic_pipe/codecs/thrift.rb

@@ -0,0 +1,17 @@
+require "magic_pipe/codecs/base"
+
+module MagicPipe
+  module Codecs
+    class Thrift < Base
+      # application/vnd.apache.thrift.binary
+      # application/vnd.apache.thrift.compact
+      # application/vnd.apache.thrift.json
+
+      TYPE = "application/vnd.apache.thrift.binary"
+
+      def encode
+        "not implemented"
+      end
+    end
+  end
+end

data/lib/magic_pipe/codecs/yaml.rb

@@ -0,0 +1,18 @@
+require "magic_pipe/codecs/base"
+require "yaml"
+
+module MagicPipe
+  module Codecs
+    class Yaml < Base
+      # text/vnd.yaml
+      # text/x-yaml
+      # application/x-yaml
+
+      TYPE = "application/x-yaml"
+
+      def encode
+        ::YAML.dump(o)
+      end
+    end
+  end
+end

data/lib/magic_pipe/config.rb

@@ -0,0 +1,103 @@
+require "logger"
+require "singleton"
+
+module MagicPipe
+  class Config
+    FIELDS = [
+      :client_name,      # the name of this client
+      :producer_name,
+      :logger,           # A Logger
+      :metrics_client,   # Statsd compatible object
+
+      :loader,
+      :codec,
+      :transport,
+      :sender,
+
+      :https_transport_options,
+      :sqs_transport_options,
+      :async_transport_options,
+    ]
+
+    attr_accessor *FIELDS
+    alias_method :transports=, :transport=
+    alias_method :transports, :transport
+
+
+    def initialize
+      yield self if block_given?
+      set_defaults
+    end
+
+
+    private
+
+
+    def set_defaults
+      @client_name ||= "magic_pipe"
+      @producer_name ||= "Anonymous Piper"
+      @logger ||= Logger.new($stdout)
+      @metrics_client ||= dummy_metrics_object
+
+      @loader ||= :simple_active_record
+      @sender ||= :sync
+      @codec ||= :yaml
+      @transport ||= :log
+
+      set_https_defaults
+      set_sqs_defaults
+      set_async_defaults
+    end
+
+
+    def set_https_defaults
+      return unless @https_transport_options
+
+      defaults = {
+        url: "https://localhost:8080/foo",
+        auth_token: "missing",
+        timeout: 2,
+        open_timeout: 3,
+      }
+      @https_transport_options = defaults.merge(@https_transport_options)
+    end
+
+
+    def set_sqs_defaults
+      @sqs_transport_options ||= {}
+      defaults = {
+        queue: "magic_pipe",
+      }
+      @sqs_transport_options = defaults.merge(@sqs_transport_options)
+    end
+
+
+    # Since Sidekiq is the go-to sender for production, this
+    # should always be defined.
+    #
+    def set_async_defaults
+      @async_transport_options ||= {}
+      defaults = {
+        queue: "magic_pipe"
+      }
+      @async_transport_options = defaults.merge(@async_transport_options)
+    end
+
+
+    def dummy_metrics_object
+      Class.new do
+        def initialize(logger)
+          @out = logger
+        end
+        def method_missing(name, *args, &block)
+          @out.debug("[metrics] #{name}: #{args}")
+          # Uncomment this to create a black hole
+          # self.class.new(@out)
+        end
+        def respond_to_missing?(*)
+          true
+        end
+      end.new(@logger)
+    end
+  end
+end

data/lib/magic_pipe/envelope.rb

@@ -0,0 +1,29 @@
+module MagicPipe
+  class Envelope
+    def initialize(body:, topic:, producer:, time:, mime:)
+      @body = body
+      @topic = topic
+      @producer = producer
+      @time = time.to_i
+      @mime = mime
+    end
+
+    attr_accessor :body
+
+
+    def as_json(*)
+      {
+        body: @body.as_json,
+        topic: @topic,
+        producer: @producer,
+        time: @time,
+        mime: @mime,
+      }
+    end
+
+
+    def ==(other)
+      as_json == other.as_json
+    end
+  end
+end

data/lib/magic_pipe/errors.rb

@@ -0,0 +1,14 @@
+module MagicPipe
+  class Error < StandardError
+  end
+
+  class ConfigurationError < Error
+  end
+
+  class LoaderError < Error
+    def initialize(class_name, context)
+      @message = "Can't resolve class name '#{class_name}' (#{context})"
+    end
+    attr_reader :message
+  end
+end