honest_pubsub 0.2.2

data/lib/honest_pubsub/publisher.rb

@@ -0,0 +1,93 @@
+module HonestPubsub
+  class Publisher
+    attr_reader :publisher
+    attr_reader :channel
+
+    @@publisher = nil
+
+    def self.instance
+      if @@publisher.nil?
+        @@publisher = ::HonestPubsub::Publisher.new
+      end
+      @@publisher
+    end
+
+    def initialize(exchange="honest")
+      @exchange = exchange
+      @disabled = false
+      @logger = ::HonestPubsub::Logger.new()
+
+      @config = Configuration.configuration
+
+      self
+    end
+
+    def enable(value)
+      @disabled = !value
+      @disabled
+    end
+
+    def start
+      if !@config[:enabled].nil? && @config[:enabled] == false
+        @disabled = true
+        return
+      end
+
+      # grab server configuration from initialization file somewhere
+      begin
+        @connection = Bunny.new(Configuration.configuration[:connection])
+        @connection.start
+
+        @channel = @connection.create_channel
+        @publisher = @channel.topic(@exchange, :durable=>true, :auto_delete=>false)
+
+      rescue => e
+        Airbrake.notify(e, parameters: {message: e.message}, environment_name: ENV['RAILS_ENV'] )
+        return
+      end
+
+
+      @publisher.on_return do |return_info, properties, content|
+        # contents are already transformed into message that we want to send
+        @logger.failed_publish(return_info[:routing_key], properties, content)
+      end
+
+    end
+
+    def publish(context, key, payload, enabled = true)
+      routing_key = "#{@exchange}.#{key}"
+      envelope = ::HonestPubsub::Message.new.serialize(context, key, payload)
+
+      if @publisher.nil?
+        start
+      end
+
+      if @disabled || @publisher.nil? || !enabled
+        @logger.failed_publish(routing_key, {}, envelope)
+      else
+        tries = 2
+        begin
+          @publisher.publish(envelope.to_json, :persistent=>true, :mandatory=>true, :timestamp=>envelope[:ts], :content_type=>"application/json", :routing_key =>routing_key )
+          @logger.log_publish(routing_key, envelope)
+        rescue => e
+          tries -= 1
+          teardown
+          start
+          if tries > 0 && @publisher
+            retry
+          else
+            @logger.failed_publish(routing_key, {}, envelope)
+          end
+        end
+
+      end
+
+    end
+
+    def teardown
+      @connection.close
+      @publisher = nil
+    end
+
+  end
+end

data/lib/honest_pubsub/railtie.rb

@@ -0,0 +1,27 @@
+# This railtie provides Rails engine hooks
+
+module HonestPubsub
+  class Railtie < ::Rails::Railtie
+    initializer "honest_pubsub.insert_middleware" do |app|
+      app.config.middleware.use HonestPubsub::Middleware
+    end
+
+    console do
+      # Set up basic context to identify messages getting generated from console
+      HonestPubsub::Context.setup_context(
+        application: "#{HonestPubsub::Configuration.application_name}/console",
+        hostname: `hostname`.strip,
+        unique_id: Digest::SHA1.new.to_s
+      )
+    end
+
+    rake_tasks do
+      # Set up basic context to identify messages getting generated from rake tasks
+      HonestPubsub::Context.setup_context(
+        application: "#{HonestPubsub::Configuration.application_name}/rake",
+        hostname: `hostname`.strip,
+        unique_id: Digest::SHA1.new.to_s
+      )
+    end
+  end
+end

data/lib/honest_pubsub/server/client_queue_listener.rb

@@ -0,0 +1,54 @@
+require 'rubygems'
+require 'celluloid/autostart'
+require 'celluloid/io'
+require 'honest_pubsub'
+
+module HonestPubsub
+  module Server
+    class ClientQueueListener
+      include Celluloid
+
+      attr_reader :worker_class
+
+      def initialize(worker_class, request_key, queue, durable = true, topic = "honest")
+        @topic = topic
+        @request_key = request_key
+        @worker_class = worker_class
+        @queue_name = queue
+        @durable = durable
+        @subscriber = ::HonestPubsub::Subscriber.new(@request_key, @durable, @topic)
+      end
+
+      def start
+        @subscriber.start(@queue_name, false) do |delivery_info, properties, envelope|
+          message_received(delivery_info, properties, envelope)
+          true
+        end
+      end
+
+      def shutdown
+        @subscriber.teardown if @subscriber.present?
+        # TODO -thl
+        # This the kosher thing to do in ruby?
+        teardown
+      end
+
+      def message_received(delivery_info, properties, envelope)
+        begin
+          HonestPubsub.logger.debug( "Message recieved by listener with payload: #{envelope[:payload]}" )
+          worker = "#{@worker_class}".constantize.new(delivery_info, properties)
+          raise PayloadError.new("Invalid Payload") unless worker.valid_payload?(envelope[:payload])
+          HonestPubsub.logger.debug( "Calling worker perform on message" )
+          worker.perform(envelope[:context], envelope[:payload])
+        rescue => e
+          #puts "Error in worker #{e.message}!!!"
+          Airbrake.notify("Failed perform for #{self.class.name}", params: {info: delivery_info, properties: properties, envelope: envelope.to_hash } )
+        end
+      end
+
+      protected
+      def teardown
+      end
+    end
+  end
+end

data/lib/honest_pubsub/server/client_worker.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'active_support/core_ext'
+
+module HonestPubsub
+  module Server
+    class ClientWorker
+      @@registered_subscribers = []
+      class_attribute :payload_validators, :error_handlers, :subscribed_key, :subscribed_queue
+      attr_accessor :delivery_routing_data, :delivery_properties
+
+      def self.inherited(klass)
+        @@registered_subscribers << klass
+      end
+
+      # Specify the routing key that the subscriber class should listen to.
+      # @param [String] routing_key_name The routing key to subscribe to. Must be characters only separated by periods (.)
+      # @param [Hash] options Allowed option is :on to optionally specify a queue. If not provided, queue name is generated from the routing key
+      def self.subscribe_to(routing_key_name, options = {})
+        options.assert_valid_keys(:on)
+        unless validate_routing_key_name(routing_key_name)
+          raise ArgumentError.new("#{routing_key_name} is not supported. Only lower case characters separated by periods are allowed.")
+        end
+        self.subscribed_key = routing_key_name
+        self.subscribed_queue = generated_queue_name(routing_key_name, options[:on])
+      end
+
+      # Sets the validator for payload
+      #
+      # @param validator The validator to use for validating the payload.
+      #                  Returns false if the payload is not valid.
+      #                  Proc must accept a payload as an argument.
+      def self.validates_payload_with(*validators)
+        self.payload_validators ||= []
+        self.payload_validators += validators
+        #
+        # executable = if validator.class <= Proc then
+        #                validator
+        #              elsif validator.class == Symbol
+        #                self.method(validator)
+        #              end
+        # self.payload_validators.push executable
+      end
+
+      # Sets an error handler for the class
+      def self.handle_errors_with(handler)
+        error_handler = handler
+      end
+
+      def initialize(delivery_routing_data, delivery_properties)
+        @delivery_routing_data = delivery_routing_data
+        @delivery_properties = delivery_properties
+      end
+
+      # Need to have a real perform method for the message to be processed.
+      def perform(payload)
+        raise "Need implementation for your worker."
+      end
+
+      def routing_key
+        delivery_routing_data[:routing_key]
+      end
+
+      # Iterates over all the payload validators and returns false if any of them are false
+      def valid_payload?(payload)
+        return true unless payload_validators.present?
+
+        payload_validators.inject(true) { |is_valid, validator|
+          is_valid && (validator.respond_to?(:call) ? validator.call(payload) : send(validator, payload) )
+        }
+      end
+
+      private
+
+      def self.validate_routing_key_name(key)
+        return true if key.blank?
+        key.match(/\A([a-z]+\.?)*([a-z]+)\Z/).present?
+      end
+
+      def self.generated_queue_name(routing_key, queue_name)
+        return queue_name if queue_name.present?
+        [ HonestPubsub::Configuration.application_name.to_s.gsub(/[^\w\_]/, ''), routing_key.gsub(".", '_') ].reject(&:blank?).join('_')
+      end
+
+    end
+  end
+end

data/lib/honest_pubsub/server/subscriber_server.rb

@@ -0,0 +1,84 @@
+require 'rubygems'
+require 'celluloid/autostart'
+require 'celluloid/io'
+require 'awesome_print'
+require 'active_support/all'
+require 'optparse'
+require 'fileutils'
+
+require_relative './client_queue_listener'
+
+# In order to use the server, the caller must be able to bring in the necessary
+# classes for require before instantiating the instance.
+
+module HonestPubsub
+  module Server
+
+    class SubscriberServer
+      include Celluloid
+
+      def initialize(subscribers)
+        @workers = subscribers.map { |subscriber| create_queue_listeners(subscriber) }
+      end
+
+      def start
+        @workers.each do |worker|
+          puts "Starting worker: #{worker.worker_class.name}"
+          worker.start
+        end
+
+        thread     = Thread.current
+        interrupts = ["HUP", "INT", "QUIT", "ABRT", "TERM"]
+        interrupts.each do |signal_name|
+          Signal.trap(signal_name) {
+            puts "Processing #{signal_name}"
+            thread.run
+          }
+        end
+
+        Thread.stop
+
+        ::HonestPubsub::Logger.new.log_service("all_services", :warn, "Starting shutdown of all services")
+
+        @workers.each do |worker|
+          ::HonestPubsub::Logger.new.log_service("all_services", :warn, "Tearing down worker: #{worker.worker_class.name}")
+          begin
+            STDOUT.puts "Tearing down subscriber for #{worker.worker_class.name}"
+            worker.shutdown
+          rescue => e
+            ::HonestPubsub::Logger.new.log_service("all_services", :warn, "#{worker.worker_class.name} - did not tear down correctly.  Error - #{e.message}")
+          end
+        end
+      ensure
+        HonestPubsub::CLI.instance.remove_pid
+      end
+
+      private
+
+      def warn(message)
+        old_behavior = ActiveSupport::Deprecation.behavior
+        ActiveSupport::Deprecation.behavior = [:stderr, :log]
+        ActiveSupport::Deprecation.warn(message)
+      ensure
+        ActiveSupport::Deprecation.behavior = old_behavior
+      end
+
+      def create_queue_listeners(subscriber)
+        routing_key           = subscriber.subscribed_key
+        subscribed_queue_name = subscriber.subscribed_queue
+
+        if routing_key.blank?
+          raise ArgumentError.new("Routing key must be provided in #{subscriber.name} using `subscribe_to routing_key`")
+        end
+
+        if subscribed_queue_name.blank?
+          raise ArgumentError.new("Queue Name must be provided in #{subscriber.name} using `subscribe_to routing_key, on: queue_name`")
+        end
+
+        STDOUT.puts "Setting up listener for request_key: #{routing_key} and queue:#{subscribed_queue_name}"
+        ClientQueueListener.new(subscriber, routing_key, subscribed_queue_name)
+      end
+    end
+  end
+end
+

data/lib/honest_pubsub/server.rb

@@ -0,0 +1,8 @@
+module HonestPubsub
+  module Server
+  end
+end
+
+require 'honest_pubsub/server/client_queue_listener'
+require 'honest_pubsub/server/client_worker'
+require 'honest_pubsub/server/subscriber_server'

data/lib/honest_pubsub/subscriber.rb

@@ -0,0 +1,69 @@
+module HonestPubsub
+  class Subscriber
+    attr_reader :listener
+    attr_reader :exchange
+    attr_reader :channel
+
+    def initialize(routing_key, durable = true, topic="honest")
+      @initial_key = routing_key
+      @durable = durable
+      @topic = topic
+
+      if @initial_key.present?
+        @routing_key = "#{@topic}.#{@initial_key}.#"
+      else
+        @routing_key = "#{@topic}.#"
+      end
+      @logger = ::HonestPubsub::Logger.new
+
+      self
+    end
+
+    # name - used to ensure that certain consumers are actually listening to an exchange
+    # pass in a lambda for this method to work.  We might only want to expose the content instead of
+    # all 3 chunks.
+    def start(name, blocking=false)
+      @connection = Bunny.new(Configuration.configuration[:connection])
+      begin
+        @connection.start
+      rescue => e
+        Airbrake.notify("RabbitMQ unreachable!", params: { message: e.message}, environment_name: ENV['RAILS_ENV'] )
+        raise e
+      end
+
+      @channel = @connection.create_channel
+      @exchange = @channel.topic(@topic, :durable=>@durable, :auto_delete=>false)
+
+      # FIX!!! -thl
+      # Need to ensure that the ids for a server will be reproducible in case a server
+      # goes down and has to get restarted.
+      if @initial_key.present?
+        @queue = "#{@initial_key}.#{name}"
+      else
+        @queue = "#{name}"
+      end
+
+      queue_arguments = {}
+      queue_arguments["x-dead-letter-exchange"] = Configuration.configuration[:dead_letter] if Configuration.configuration[:dead_letter].present?
+      @listener = @channel.queue(@queue, :arguments=>queue_arguments ).bind(@exchange, :routing_key => @routing_key, :exclusive=>false)
+      # Parameters for subscribe that might be useful:
+      # :block=>true - Used for long running consumer applications.  (backend servers?)
+      @consumer = @listener.subscribe(:consumer_tag=>name, :block=>blocking)
+      @consumer.on_delivery do |delivery_info, properties, contents|
+        HonestPubsub.logger.debug( "Message delivery with contents: #{contents}")
+        if delivery_info[:redelivered]
+          Airbrake.notify("PubSub Message redelivery", params: {info: delivery_info, props: properties, contents: contents}, environment_name: ENV['RAILS_ENV'] )
+        end
+        message = ::HonestPubsub::Message.new.parse(contents)
+        @logger.log_receive(delivery_info[:routing_key], message)
+        yield delivery_info, properties, message
+        true
+      end
+    end
+
+    def teardown
+      @consumer.cancel if @consumer.present?
+      @connection.close if @connection.present?
+    end
+  end
+end

data/lib/honest_pubsub/version.rb

@@ -0,0 +1,3 @@
+module HonestPubsub
+  VERSION = "0.2.2"
+end

data/lib/honest_pubsub.rb

@@ -0,0 +1,46 @@
+require "active_support/all"
+require "bunny"
+require "yaml"
+require "airbrake"
+
+require "honest_pubsub/configuration"
+require "honest_pubsub/context"
+require "honest_pubsub/db_logger"
+require "honest_pubsub/logger"
+require "honest_pubsub/logging"
+require "honest_pubsub/message"
+require "honest_pubsub/publisher"
+require "honest_pubsub/subscriber"
+require "honest_pubsub/version"
+require "honest_pubsub/exceptions/payload_error"
+require 'honest_pubsub/server'
+
+if defined?(Rails::Railtie)
+  require "honest_pubsub/middleware"
+  require "honest_pubsub/railtie"
+end
+
+module HonestPubsub
+
+  def self.root
+    File.expand_path '../..', __FILE__
+  end
+
+# This method publishes payload to rabbitmq.  All listeners with appropriate
+# routing keys will receive the payload.
+# @param [String] routing_key Identifier of the message type
+# @param [Hash] payload The data that will be passed to the subscriber
+
+  def self.publish(routing_key, payload)
+    Publisher.instance.publish(HonestPubsub::Context.instance, routing_key, payload)
+  end
+
+  def self.logger
+    HonestPubsub::Logging.logger
+  end
+
+  # @param [Logger] logger Logger used by HonestPubsub
+  def self.logger=(logger)
+    HonestPubsub::Logging.logger = logger
+  end
+end

data/spec/config.yml

@@ -0,0 +1,20 @@
+test:
+  # Documentation for parameters for rabbit and bunny is located here: http://rubybunny.info/articles/connecting.html
+  connection:
+    host: localhost
+    port: 5672
+    # username: rabbit
+    # password: rabbit
+    heartbeat: 60 # in seconds
+    log_level: 0
+    log_file: test_rabbit.log
+    network_recovery_interval: 10 # in seconds
+    continuation_timeout: 4000 # in milliseconds
+
+  logger:
+    enabled: true
+    level: warn
+    file: test_pubsub.log
+
+
+  pid: pids/honest_pubsub

data/spec/honest_pubsub/#subscriber_spec.rb#

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe HonestPubsub::Subscriber do
+  subject { HonestPubsub::Subscriber.new( routing_key ) }
+
+  describe "#start" do
+    
+  end
+end

data/spec/honest_pubsub/cli_spec.rb

@@ -0,0 +1,145 @@
+require 'spec_helper'
+require 'honest_pubsub/cli'
+
+describe HonestPubsub::CLI do
+  let(:cli) { HonestPubsub::CLI.instance }
+  after(:each) do
+    cli.instance_variable_set(:@subscribers, nil)
+    cli.instance_variable_set(:@pidfile, nil)
+    cli.instance_variable_set(:@require_path, nil)
+  end
+
+  describe '#parse' do
+    let(:options) { [] }
+    before { cli.parse(options) }
+
+    context "pidfile" do
+      it "works when nothing is passed" do
+        expect(cli.pidfile).to be_nil
+      end
+
+      context "passed in as -P" do
+        let(:options) { [ "-P", "mypidfile.pid"]}
+        it "parses" do
+          expect(cli.pidfile).to eq('mypidfile.pid')
+        end
+      end
+
+      context "passed in as --pidfile" do
+        let(:options) { [ "--pidfile", "mypidfile.pid"]}
+        it "parses" do
+          expect(cli.pidfile).to eq('mypidfile.pid')
+        end
+      end
+    end
+
+    context "only" do
+      it "works when nothing is passed" do
+        expect(cli.subscribers).to be_nil
+      end
+
+      context "passed in as -o" do
+        let(:options) { [ "-o", "Foobar,Doobar"]}
+        it "parses" do
+          expect(cli.subscribers).to eq([ 'Foobar', 'Doobar'])
+        end
+      end
+
+      context "passed in as --only" do
+        let(:options) { [ "--only", "Foobar,Doobar"]}
+        it "parses" do
+          expect(cli.subscribers).to eq([ 'Foobar', 'Doobar'])
+        end
+      end
+    end
+
+    context "require" do
+      it "works when nothing is passed" do
+        expect(cli.require_path).to eq(".")
+      end
+
+      context "passed in as -r" do
+        let(:options) { [ "-r", "/path/to/my/file"]}
+        it "parses" do
+          expect(cli.require_path).to eq("/path/to/my/file")
+        end
+      end
+
+      context "passed in as --require" do
+        let(:options) { [ "--require", "/path/to/my/file"]}
+        it "parses" do
+          expect(cli.require_path).to eq("/path/to/my/file")
+        end
+      end
+    end
+  end
+
+  describe '#run' do
+    describe "high-level steps to be completed" do
+      before {
+        allow(cli).to receive(:load_environment)
+        allow(cli).to receive(:write_pidfile)
+        allow(cli).to receive(:load_subscribers)
+        cli.run
+      }
+
+      it "runs" do
+        expect(cli).to have_received(:load_environment)
+        expect(cli).to have_received(:write_pidfile)
+        expect(cli).to have_received(:load_subscribers)
+      end
+    end
+  end
+
+  describe '#write_pidfile' do
+    before {
+      cli.pidfile = pidfile
+      allow(File).to receive(:open)
+      cli.send(:write_pidfile)
+    }
+    context "no pidfile passed" do
+      let(:pidfile) { nil }
+      it "doesn't create a pidfile" do
+        expect(File).to_not have_received(:open)
+      end
+    end
+
+    context "pidfile path is supplied" do
+      let(:pidfile) { "/path/to/pidfile" }
+      it 'creates file' do
+        expect(File).to have_received(:open).with(pidfile, 'w')
+      end
+    end
+  end
+
+  describe '#load_subscribers' do
+    let(:subscribers) { nil }
+    let(:subscriber_server) { double('subscriber_server')}
+    before {
+      cli.subscribers = subscribers
+      allow(subscriber_server).to receive(:start)
+      allow(HonestPubsub::Server::SubscriberServer).to receive(:new).and_return(subscriber_server)
+      cli.send(:load_subscribers)
+    }
+
+    context "No subscribers passed through CLI" do
+      let(:all_subscribers) { HonestPubsub::Server::ClientWorker.class_variable_get(:@@registered_subscribers) }
+      it "loads all subscribers" do
+        expect(all_subscribers).to include(MyTestSubscriber1)
+        expect(all_subscribers).to include(MyTestSubscriber2)
+        expect(all_subscribers).to include(MyTestSubscriber3)
+        expect(HonestPubsub::Server::SubscriberServer).to have_received(:new).with(all_subscribers)
+        expect(subscriber_server).to have_received(:start)
+      end
+    end
+
+    context "Subscribers passed through CLI" do
+      let(:subscribers) { ['MyTestSubscriber1', 'MyTestSubscriber2'] }
+      it "loads all subscribers" do
+        expect(HonestPubsub::Server::SubscriberServer).to have_received(:new).with([MyTestSubscriber1, MyTestSubscriber2])
+        expect(subscriber_server).to have_received(:start)
+      end
+    end
+  end
+
+end