banter 0.4.0

data/lib/banter/db_logger.rb

@@ -0,0 +1,52 @@
+
+# Internal database logger that saves the publish message before it is send
+# and also acts as a sink for messages.
+module Banter
+  class DbLogger
+
+    def initialize
+      @config = Configuration.configuration[:mongo]
+      if @config.present?
+        @client = Mongo::MongoClient.new(@config[:host], @config[:port])
+      else
+        @client = Mongo::MongoClient.new
+      end
+
+      @db = @client.db("logger")
+
+    end
+
+    def start_subscribe
+      subscriber = ::Banter::Server::RabbitMQSubscriber.new("")
+      subscriber.start("logger") do |info, properties, contents|
+        # Write to mongo the contents and the routing_key, with the routing_key being indexed
+        routing_key = info[:routing_key]
+        chunks = routing_key.split(".")
+        # Logger will split the data into collections based upon the first parameter of the routing_key (initial roll out will only use 'honest')
+        collection_name = chunks[1] || chunks[0]
+        collection = @db.collection(collection_name)
+        # no need for safe writes.  need to write to a file log as well.
+        collection.insert({routing_key: routing_key, cts: contents, ts: properties[:timestamp]})
+        # make sure that we always index these two items in mongo
+        # it's a small price of a call, so no big deal
+        collection.ensure_index( {routing_key:1, ts:1} )
+      end
+    end
+
+    def log_publish(routing_key, message)
+      collection_name = "publisher"
+      collection = @db.collection(collection_name)
+
+      # no need for safe writes.  need to write to a file log as well.
+      collection.insert({routing_key: routing_key, cts: message[:payload], ts: message[:ts], pub: message[:pub]})
+      # make sure that we always index these two items in mongo
+      # it's a small price of a call, so no big deal
+      collection.ensure_index( {routing_key:1, ts:1} )
+
+    end
+
+    def teardown
+      @subscriber.teardown
+    end
+  end
+end

data/lib/banter/exceptions/payload_validation_error.rb

@@ -0,0 +1,4 @@
+module Banter
+  class PayloadValidationError < StandardError
+  end
+end

data/lib/banter/logger.rb

@@ -0,0 +1,49 @@
+# Internal log class that is used to log messages before sending, after receiving, and failure to send
+module Banter
+  class Logger
+    def initialize(enable_publish_logging = true)
+      @enabled = Banter::Configuration.configuration[:logger][:enabled]
+      @enabled = enable_publish_logging if @enabled.nil?
+    end
+
+    def log_publish(routing_key, message)
+      return unless @enabled
+      # generate tags needed
+      tags = ["PUBLISH", message[:ts], message[:pub], message[:v], routing_key]
+
+      # FIX!!! -thl
+      # Could logging like this be too slow?
+      # Or should it be threaded?
+      # We'll need to benchmark, as we don't want this to get too slow.
+      logger.tagged(tags) { logger.warn message[:payload].as_json }
+
+      # TODO: -thl
+      # Log it into mongo as well?
+    end
+
+    def failed_publish(routing_key, properties, message)
+      tags = ["FAILED_SEND", message[:ts], message[:pub], message[:v], routing_key]
+      logger.tagged(tags) { logger.warn message[:payload].as_json }
+    end
+
+    def log_receive(routing_key, message)
+      tags = ["RECEIVED", message[:ts], message[:pub], message[:v], routing_key, Process::pid]
+      logger.tagged(tags) { logger.warn message[:payload].as_json }
+    end
+
+    def log_service(service_name, log_level, message)
+      tags = ["SERVICE", service_name, Process::pid]
+      log_method = log_level.to_sym.to_proc
+      logger.tagged(tags) { log_method.call(logger) { message.as_json } }
+    end
+
+    def teardown
+      logger.close
+    end
+
+    private
+    def logger
+      @logger ||= ActiveSupport::TaggedLogging.new(Banter.logger)
+    end
+  end
+end

data/lib/banter/logging.rb

@@ -0,0 +1,42 @@
+require 'logger'
+
+# Manages a logger that can be used throughout the pubsub gem
+# The logger can be set to any object that quacks like a logger including the
+# Rails logger if so desired
+#
+# The logger can be set in the initializer of a program or anywhere throughout as
+# Banter.logger = Rails.logger for example
+
+module Banter
+  module Logging
+    # Sets the logger
+    #
+    # @param logger The logger to use throughout the pubsub gem
+    # @return The logger we use throughout the gem
+    def self.logger=(logger)
+      raise StandardError("Can't set logger to nil") unless logger.present?
+      @logger = logger
+    end
+
+    # Gets the logger
+    # If no logger is defined when this method is called it will return a standard
+    # ruby logger
+    #
+    # @return The logger we use throughout the gem
+    def self.logger 
+      config = Banter::Configuration.configuration[:logger]
+      @logger ||= create_logger( {}.
+        merge( config[:level].present? ? { log_level: ::Logger::Severity.const_get(config[:level].upcase) } : {} ).
+        merge( config[:file].present? ? { log_target: config[:file] } : {} ) )
+    end
+
+    # Builds a standard ruby Logger
+    #
+    # @return Returns the logger at the end of the method
+    def self.create_logger( options = {} )
+      @logger = ::Logger.new(options.fetch(:log_target){ STDOUT })
+      @logger.level = options.fetch(:log_level){ ::Logger::INFO }
+      @logger
+    end
+  end
+end

data/lib/banter/message.rb

@@ -0,0 +1,50 @@
+
+require "hashie"
+require "json"
+
+module Banter
+  class Message
+    @@message_version = "1"
+    attr_reader :ts
+    attr_reader :pub
+    attr_reader :version
+    attr_reader :payload
+    attr_reader :context
+
+    def initialize
+    end
+
+    # Context object should be passed into the call.
+    def serialize(context, routing_key, payload)
+      @ts = Time.now.to_i
+      @pub = "#{routing_key}:#{Socket.gethostname()}:#{Process::pid}"
+      @version = @@message_version
+      @payload = payload
+      @context = context
+      to_hash
+    end
+
+    def parse(envelope)
+      contents = Hashie::Mash.new(::JSON.parse(envelope))
+      @ts = contents[:ts]
+      @pub = contents[:pub]
+      # Version used for messaging can get updated if we need to add extra header information, and need
+      # to move each section of the library separately.
+      @version = contents[:v]
+      @payload = contents[:payload]
+      @context = ::Banter::Context.from_json(contents[:context])
+      to_hash
+    end
+
+    def to_hash
+      ::Hashie::Mash.new({
+        ts: @ts,
+        pub: @pub,
+        v: @@message_version,
+        payload: @payload,
+        context: @context.as_json
+      })
+    end
+
+  end
+end

data/lib/banter/middleware.rb

@@ -0,0 +1,14 @@
+# Middleware to ensure that we clear the context with every request
+
+module Banter
+  class Middleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      Banter::Context.clear!
+      @app.call(env)
+    end
+  end
+end

data/lib/banter/publisher.rb

@@ -0,0 +1,93 @@
+module Banter
+  class Publisher
+    attr_reader :publisher
+    attr_reader :channel
+
+    @@publisher = nil
+
+    def self.instance
+      if @@publisher.nil?
+        @@publisher = ::Banter::Publisher.new
+      end
+      @@publisher
+    end
+
+    def initialize(exchange="honest")
+      @exchange = exchange
+      @disabled = false
+      @logger = ::Banter::Logger.new()
+
+      @config = Configuration.configuration
+
+      self
+    end
+
+    def enable(value)
+      @disabled = !value
+      @disabled
+    end
+
+    def start
+      if !@config[:enabled].nil? && @config[:enabled] == false
+        @disabled = true
+        return
+      end
+
+      # grab server configuration from initialization file somewhere
+      begin
+        @connection = Bunny.new(Configuration.configuration[:connection])
+        @connection.start
+
+        @channel = @connection.create_channel
+        @publisher = @channel.topic(@exchange, :durable=>true, :auto_delete=>false)
+
+      rescue => e
+        Airbrake.notify(e, parameters: {message: e.message}, environment_name: ENV['RAILS_ENV'] )
+        return
+      end
+
+
+      @publisher.on_return do |return_info, properties, content|
+        # contents are already transformed into message that we want to send
+        @logger.failed_publish(return_info[:routing_key], properties, content)
+      end
+
+    end
+
+    def publish(context, key, payload, enabled = true)
+      routing_key = "#{@exchange}.#{key}"
+      envelope = ::Banter::Message.new.serialize(context, key, payload)
+
+      if @publisher.nil?
+        start
+      end
+
+      if @disabled || @publisher.nil? || !enabled
+        @logger.failed_publish(routing_key, {}, envelope)
+      else
+        tries = 2
+        begin
+          @publisher.publish(envelope.to_json, :persistent=>true, :mandatory=>true, :timestamp=>envelope[:ts], :content_type=>"application/json", :routing_key =>routing_key )
+          @logger.log_publish(routing_key, envelope)
+        rescue => e
+          tries -= 1
+          teardown
+          start
+          if tries > 0 && @publisher
+            retry
+          else
+            @logger.failed_publish(routing_key, {}, envelope)
+          end
+        end
+
+      end
+
+    end
+
+    def teardown
+      @connection.close
+      @publisher = nil
+    end
+
+  end
+end

data/lib/banter/railtie.rb

@@ -0,0 +1,27 @@
+# This railtie provides Rails engine hooks
+
+module Banter
+  class Railtie < ::Rails::Railtie
+    initializer "banter.insert_middleware" do |app|
+      app.config.middleware.use Banter::Middleware
+    end
+
+    console do
+      # Set up basic context to identify messages getting generated from console
+      Banter::Context.setup_context(
+        application: "#{Banter::Configuration.application_name}/console",
+        hostname: `hostname`.strip,
+        unique_id: Digest::SHA1.new.to_s
+      )
+    end
+
+    rake_tasks do
+      # Set up basic context to identify messages getting generated from rake tasks
+      Banter::Context.setup_context(
+        application: "#{Banter::Configuration.application_name}/rake",
+        hostname: `hostname`.strip,
+        unique_id: Digest::SHA1.new.to_s
+      )
+    end
+  end
+end

data/lib/banter/server.rb

@@ -0,0 +1,7 @@
+module Banter
+  module Server
+  end
+end
+
+require 'banter/server/client_queue_listener'
+require 'banter/server/subscriber_server'

data/lib/banter/server/client_queue_listener.rb

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'celluloid/autostart'
+require 'celluloid/io'
+require 'banter'
+
+module Banter
+  module Server
+    class ClientQueueListener
+      include Celluloid
+
+      attr_reader :worker_class
+
+      def initialize(worker_class, request_key, queue, durable = true, topic = "honest")
+        @topic        = topic
+        @request_key  = request_key
+        @worker_class = worker_class
+        @queue_name   = queue
+        @durable      = durable
+        @subscriber   = ::Banter::Server::RabbitMQSubscriber.new(@request_key, @durable, @topic)
+      end
+
+      def start
+        @subscriber.start(@queue_name, false) do |delivery_info, properties, envelope|
+          message_received(delivery_info, properties, envelope)
+          true
+        end
+      end
+
+      def shutdown
+        @subscriber.teardown if @subscriber.present?
+        # TODO -thl
+        # This the kosher thing to do in ruby?
+        teardown
+      end
+
+      def message_received(delivery_info, properties, envelope)
+        Banter.logger.debug("Message received by listener on #{worker_class.name} with payload: #{envelope[:payload]}")
+        worker = worker_class.new(delivery_info, properties, envelope[:context])
+        worker.perform!(envelope[:payload])
+      rescue => e
+        Banter.logger.error("Failed message for #{worker_class.name}")
+        Airbrake.notify("Failed perform for #{worker_class.name}",
+                        params: {
+                          info:       delivery_info,
+                          properties: properties,
+                          context:    envelope[:context],
+                          payload:    envelope[:payload] })
+
+      end
+
+      protected
+      def teardown
+      end
+    end
+  end
+end

data/lib/banter/server/rabbit_mq_subscriber.rb

@@ -0,0 +1,75 @@
+module Banter
+  module Server
+
+    class RabbitMQSubscriber
+      attr_reader :listener
+      attr_reader :exchange
+      attr_reader :channel
+
+      def initialize(routing_key, durable = true, topic="honest")
+        @initial_key = routing_key
+        @durable     = durable
+        @topic       = topic
+
+        if @initial_key.present?
+          @routing_key = "#{@topic}.#{@initial_key}.#"
+        else
+          @routing_key = "#{@topic}.#"
+        end
+        @logger = ::Banter::Logger.new
+
+        self
+      end
+
+      # name - used to ensure that certain consumers are actually listening to an exchange
+      # pass in a lambda for this method to work.  We might only want to expose the content instead of
+      # all 3 chunks.
+      def start(name, blocking=false)
+        @connection = Bunny.new(Configuration.configuration[:connection])
+        begin
+          @connection.start
+        rescue => e
+          Airbrake.notify("RabbitMQ unreachable!", params: { message: e.message }, environment_name: ENV['RAILS_ENV'])
+          raise e
+        end
+
+        @channel  = @connection.create_channel
+        @exchange = @channel.topic(@topic, :durable => @durable, :auto_delete => false)
+
+        # FIX!!! -thl
+        # Need to ensure that the ids for a server will be reproducible in case a server
+        # goes down and has to get restarted.
+        if @initial_key.present?
+          @queue = "#{@initial_key}.#{name}"
+        else
+          @queue = "#{name}"
+        end
+
+        queue_arguments                           = {}
+        queue_arguments["x-dead-letter-exchange"] = Configuration.configuration[:dead_letter] if Configuration.configuration[:dead_letter].present?
+        queue_arguments["x-message-ttl"]          = Configuration.default_queue_ttl
+        @listener                                 = @channel.queue(@queue, arguments: queue_arguments).bind(@exchange, routing_key: @routing_key, exclusive: false)
+
+        # Parameters for subscribe that might be useful:
+        # :block=>true - Used for long running consumer applications.  (backend servers?)
+        @consumer                                 = @listener.subscribe(consumer_tag: name, block: blocking)
+
+        @consumer.on_delivery do |delivery_info, properties, contents|
+          Banter.logger.debug("Message delivery with contents: #{contents}")
+          if delivery_info[:redelivered]
+            Airbrake.notify("PubSub Message redelivery", params: { info: delivery_info, props: properties, contents: contents }, environment_name: ENV['RAILS_ENV'])
+          end
+          message = ::Banter::Message.new.parse(contents)
+          @logger.log_receive(delivery_info[:routing_key], message)
+          yield delivery_info, properties, message
+          true
+        end
+      end
+
+      def teardown
+        @consumer.cancel if @consumer.present?
+        @connection.close if @connection.present?
+      end
+    end
+  end
+end