modern_times 0.1.0

data/lib/modern_times/exception.rb

@@ -0,0 +1,4 @@
+module ModernTimes
+  class Exception < ::Exception
+  end
+end

data/lib/modern_times/hornetq/client.rb

@@ -0,0 +1,53 @@
+require 'hornetq'
+
+# Handle Messaging and Queuing
+module ModernTimes
+  module HornetQ
+    module Client
+      # Singleton-ize
+      extend self
+
+      # Initialize the messaging system and connection pool for this VM
+      def init(config)
+        @config = config
+        @connection = ::HornetQ::Client::Connection.new(@config[:connection])
+        # Let's not create a session_pool unless we're going to use it
+        @session_pool_mutex = Mutex.new
+
+        at_exit do
+          close
+        end
+      end
+
+      def invm?
+        @connection.invm?
+      end
+
+      # Create a session targeted for a consumer (producers should use the session_pool)
+      def create_consumer_session
+        @connection.create_session(config[:session])
+      end
+
+      def session_pool
+        # Don't use the mutex unless we have to!
+        return @session_pool if @session_pool
+        @session_pool_mutex.synchronize do
+          # if it's been created in between the above call and now, return it
+          return @session_pool if @session_pool
+          return @session_pool = @connection.create_session_pool(config[:session])
+        end
+      end
+
+      def close
+        ModernTimes.logger.info "Closing #{self.name}"
+        @session_pool.close if @session_pool
+        @connection.close if @connection
+      end
+
+      def config
+        raise "#{self.name} never had it's init method called" unless @config
+        @config
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/marshal_strategy/json.rb

@@ -0,0 +1,17 @@
+module ModernTimes
+  module HornetQ
+    module MarshalStrategy
+      module JSON
+        def marshal(session, object, durable)
+          message = session.create_message(::HornetQ::Client::Message::TEXT_TYPE, durable)
+          message.body = object.to_json
+          message
+        end
+
+        def unmarshal(msg)
+          JSON::Parser.new(msg.body).parse
+        end
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/marshal_strategy/ruby.rb

@@ -0,0 +1,17 @@
+module ModernTimes
+  module HornetQ
+    module MarshalStrategy
+      module Ruby
+        def marshal(session, object, durable)
+          message = session.create_message(::HornetQ::Client::Message::BYTES_TYPE, durable)
+          message.body = ::Marshal.dump(object)
+          message
+        end
+
+        def unmarshal(msg)
+          ::Marshal.load(msg.body)
+        end
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/marshal_strategy/string.rb

@@ -0,0 +1,17 @@
+module ModernTimes
+  module HornetQ
+    module MarshalStrategy
+      module String
+        def marshal(session, object, durable)
+          message = session.create_message(::HornetQ::Client::Message::TEXT_TYPE, durable)
+          message.body = object.to_s
+          message
+        end
+
+        def unmarshal(msg)
+          msg.body
+        end
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/marshal_strategy.rb

@@ -0,0 +1,3 @@
+require 'modern_times/hornetq/marshal_strategy/json'
+require 'modern_times/hornetq/marshal_strategy/ruby'
+require 'modern_times/hornetq/marshal_strategy/string'

data/lib/modern_times/hornetq/publisher.rb

@@ -0,0 +1,72 @@
+require 'hornetq'
+
+# Protocol independent class to handle Publishing
+module ModernTimes
+  module HornetQ
+    class Publisher
+
+      # TODO: Possible performance enhancements on producer
+      # setDisableMessageID()
+      # setDisableMessageTimeStamp()
+      # See http://hornetq.sourceforge.net/docs/hornetq-2.1.2.Final/user-manual/en/html/perf-tuning.html
+      # Create producer pool as per above section 46.6?
+      def initialize(address, options={})
+        @address = address
+        @durable = !!options[:durable]
+        if options[:marshal].nil?
+          marshal_module = MarshalStrategy::Ruby
+        elsif options[:marshal].kind_of? Symbol
+          marshal_module = case options[:marshal]
+                             when :ruby   then MarshalStrategy::Ruby
+                             when :string then MarshalStrategy::String
+                             when :json   then MarshalStrategy::JSON
+                             else raise "Invalid marshal strategy: #{options[:marshal]}"
+                           end
+        elsif options[:marshal].kind_of? Module
+          marshal_module = options[:marshal]
+        else
+          raise "Invalid marshal strategy: #{options[:marshal]}"
+        end
+        self.extend marshal_module
+      end
+
+      # Publish the given object to the address.
+      def publish(object)
+        Client.session_pool.producer(@address) do |session, producer|
+          message = marshal(session, object, @durable)
+          first_time = true
+          begin
+            producer.send(message)
+          rescue Java::org.hornetq.api.core.HornetQException => e
+            ModernTimes.logger.warn "Received producer exception: #{e.message} with code=#{e.cause.code}"
+            if first_time && e.cause.code == Java::org.hornetq.api.core.HornetQException::UNBLOCKED
+              ModernTimes.logger.info "Retrying the send"
+              first_time = false
+              retry
+            else
+              raise
+            end
+          end
+        end
+      end
+
+      # For non-configured Rails projects, The above publish method will be overridden to
+      # call this publish method instead which calls all the HornetQ workers that
+      # operate on the given address.
+      def dummy_publish(object)
+        @@worker_instances.each do |worker|
+          if worker.kind_of?(Worker) && worker.address_name == @address
+            ModernTimes.logger.debug "Dummy publishing #{object} to #{worker}"
+            worker.perform(object)
+          end
+        end
+      end
+
+      def self.setup_dummy_publishing(workers)
+        @@worker_instances = workers.map {|worker| worker.new}
+        alias_method :real_publish, :publish
+        alias_method :publish, :dummy_publish
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/supervisor.rb

@@ -0,0 +1,19 @@
+module ModernTimes
+  module HornetQ
+    class Supervisor < ModernTimes::Base::Supervisor
+      # Make HornetQ::SupervisorMBean our mbean
+      mbean SupervisorMBean
+
+      attr_reader :message_count
+
+      def initialize(manager, worker_name, opts={})
+        super(manager, worker_name, opts)
+        @message_count = 0
+      end
+
+      def incr_message_count
+        @message_count += 1
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/supervisor_mbean.rb

@@ -0,0 +1,12 @@
+module ModernTimes
+  module HornetQ
+    class SupervisorMBean < ModernTimes::Base::SupervisorMBean
+
+      operation 'Total message count'
+      returns :int
+      def message_count
+        supervisor.message_count
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq/worker.rb

@@ -0,0 +1,121 @@
+module ModernTimes
+  module HornetQ
+  
+    # Base Worker Class for any class that will be processing messages from queues
+    class Worker < ModernTimes::Base::Worker
+      # Default to ruby marshaling, but extenders can override as necessary
+      include MarshalStrategy::Ruby
+
+      # Make HornetQ::Supervisor our supervisor
+      supervisor Supervisor
+      
+      attr_reader :session, :message_count
+
+      def initialize
+        @status = 'initialized'
+        @message_count = 0
+      end
+
+      def setup
+        session = Client.create_consumer_session
+        session.create_queue_ignore_exists(address_name, queue_name, false)
+        session.close
+      end
+
+      def status
+        @status || "Processing message #{message_count}"
+      end
+
+      def on_message(message)
+        object = unmarshal(message)
+        ModernTimes.logger.debug "#{self}: Received Object: #{object}" if ModernTimes.logger.debug?
+        perform(object)
+        ModernTimes.logger.debug "#{self}: Finished processing message" if ModernTimes.logger.debug?
+        ModernTimes.logger.flush if ModernTimes.logger.respond_to?(:flush)
+      rescue Exception => e
+        ModernTimes.logger.error "#{self}: Messaging Exception: #{e.inspect}\n#{e.backtrace.inspect}"
+      rescue java.lang.Exception => e
+        ModernTimes.logger.error "#{self}: Java Messaging Exception: #{e.inspect}\n#{e.backtrace.inspect}"
+      end
+
+      def perform(object)
+        raise "#{self}: Need to override perform method in #{self.class.name} in order to act on #{object}"
+      end
+
+      def self.address_name
+        @address_name ||= default_name
+      end
+
+      def self.queue_name
+        @queue_name ||= default_name
+      end
+
+      def address_name
+        self.class.address_name
+      end
+
+      def queue_name
+        self.class.queue_name
+      end
+
+      def to_s
+        "#{address_name}:#{queue_name}:#{index}"
+      end
+
+      # Start the event loop for handling messages off the queue
+      def start
+        @session = Client.create_consumer_session
+        @consumer = @session.create_consumer(queue_name)
+        @session.start
+        ModernTimes.logger.debug "#{self}: Starting receive loop"
+        @status = nil
+        while msg = @consumer.receive
+          @message_count += 1
+          supervisor.incr_message_count
+          on_message(msg)
+          msg.acknowledge
+        end
+        @status = 'Exited'
+        ModernTimes.logger.info "Exiting #{self}"
+      rescue Java::org.hornetq.api.core.HornetQException => e
+        if e.cause.code == Java::org.hornetq.api.core.HornetQException::OBJECT_CLOSED
+          # Normal exit
+          @status = 'Exited'
+          ModernTimes.logger.info "#{self}: Exiting due to close"
+        else
+          @status = "Exited with HornetQ exception #{e.message}"
+          ModernTImes.logger.error "#{self} HornetQException: #{e.message}\n#{e.backtrace.join("\n")}"
+        end
+      rescue Exception => e
+        @status = "Exited with exception #{e.message}"
+        ModernTimes.logger.error "#{self}: Exception, thread terminating: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+      rescue java.lang.Exception => e
+        @status = "Exited with java exception #{e.message}"
+        ModernTimes.logger.error "#{self}: Java exception, thread terminating: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+      end
+
+      def stop
+        @session.close if @session
+      end
+
+      #########
+      protected
+      #########
+
+      # Create a queue, assigned to the specified address
+      # Every time a message arrives, the perform instance method
+      # will be called. The parameter to the method is the Ruby
+      # object supplied when the message was sent
+      def self.address(address_name, opts={})
+        @address_name = address_name.to_s
+        #Messaging::Client.on_message(address, queue_name) do |object|
+        #  self.send(method.to_sym, object)
+        #end
+      end
+
+      def self.queue(queue_name, opts={})
+        @queue_name = queue_name.to_s
+      end
+    end
+  end
+end

data/lib/modern_times/hornetq.rb

@@ -0,0 +1,11 @@
+require 'modern_times/hornetq/client'
+require 'modern_times/hornetq/marshal_strategy'
+require 'modern_times/hornetq/publisher'
+require 'modern_times/hornetq/supervisor_mbean'
+require 'modern_times/hornetq/supervisor'
+require 'modern_times/hornetq/worker'
+
+module ModernTimes
+  module HornetQ
+  end
+end

data/lib/modern_times/loggable.rb

@@ -0,0 +1,23 @@
+module ModernTimes
+  module Loggable
+    def logger
+      @logger ||= (rails_logger || default_logger)
+    end
+
+    def rails_logger
+      (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) ||
+      (defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER.respond_to?(:debug) && RAILS_DEFAULT_LOGGER)
+    end
+
+    def default_logger
+      require 'logger'
+      l = Logger.new($stdout)
+      l.level = Logger::INFO
+      l
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+  end
+end

data/lib/modern_times/manager.rb

@@ -0,0 +1,92 @@
+module ModernTimes
+  class Manager
+    attr_accessor :allowed_workers
+
+    def initialize(config={})
+      @config = config
+      @domain = config[:domain] || 'ModernTimes'
+      @supervisors = []
+      @jmx_server = JMX::MBeanServer.new
+      bean = ManagerMBean.new("#{@domain}.Manager", "Manager", self)
+      @jmx_server.register_mbean(bean, "#{@domain}:type=Manager")
+      persist_file = config[:persist_file]
+    end
+
+    def add(worker_klass, num_workers, worker_options)
+      ModernTimes.logger.info "Starting #{worker_klass} with #{num_workers} workers with options #{worker_options.inspect}"
+      unless worker_klass.kind_of?(Class)
+        begin
+          worker_klass = Object.const_get(worker_klass.to_s)
+        rescue
+          raise ModernTimes::Exception.new("Invalid class: #{worker_klass}")
+        end
+      end
+      if @allowed_workers && !@allowed_workers.include?(worker_klass)
+        raise ModernTimes::Exception.new("Error: #{worker_klass.name} is not an allowed worker")
+      end
+      supervisor = worker_klass.create_supervisor(self)
+      mbean = supervisor.create_mbean(@domain)
+      @supervisors << supervisor
+      supervisor.worker_count = num_workers
+      @jmx_server.register_mbean(mbean, "#{@domain}:worker=#{worker_klass.name},type=Worker")
+      ModernTimes.logger.info "Started #{worker_klass.name} with #{num_workers} workers"
+    rescue Exception => e
+      ModernTimes.logger.error "Exception trying to add #{worker_klass.name}: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+    rescue java.lang.Exception => e
+      ModernTimes.logger.error "Java exception trying to add #{worker_klass.name}: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+    end
+
+    def start
+      return if @started
+      @started = true
+
+    end
+
+    def stop
+      @stopped = true
+      @supervisors.each { |supervisor| supervisor.stop }
+    end
+
+    def join
+      while !@stopped
+        sleep 1
+      end
+      @supervisors.each { |supervisor| supervisor.join }
+    end
+
+    def stop_on_signal
+      ['HUP', 'INT', 'TERM'].each do |signal_name|
+        Signal.trap(signal_name) do
+          ModernTimes.logger.info "Caught #{signal_name}"
+          stop
+        end
+      end
+    end
+
+    def persist_file=(file)
+      @persist_file = file
+      return unless file
+      @persist_file = file
+      if File.exist?(file)
+        hash = YAML.load_file(file)
+        hash.each do |worker_klass, count|
+          add(worker_klass, count)
+        end
+      end
+    end
+
+    def save_persist_state
+      return unless @persist_file
+      hash = {}
+      @supervisors.each do |supervisor|
+        hash[supervisor.worker_name] = {
+          :worker_count => supervisor.worker_count,
+          :options      => supervisor.worker_options
+        }
+      end
+      File.open(@persist_file, 'w') do |out|
+        YAML.dump(hash, out )
+      end
+    end
+  end
+end

data/lib/modern_times/manager_mbean.rb

@@ -0,0 +1,36 @@
+require 'jmx'
+
+module ModernTimes
+  class ManagerMBean < RubyDynamicMBean
+    attr_reader :manager
+    rw_attribute :foobar, :int, "Number of workers"
+
+    def initialize(name, description, manager)
+      super(name, description)
+      @manager = manager
+    end
+
+    operation 'Allowed workers'
+    returns :list
+    def allowed_workers
+      all = manager.allowed_workers || ['No Restrictions']
+      all.map {|worker_klass| worker_klass.name }
+    end
+
+    operation 'Start worker'
+    parameter :string, "worker", "The worker class to start"
+    parameter :int, "count", "Number of workers"
+    returns :string
+    def start_worker(worker, count)
+      ModernTimes.logger.debug "Attempting to start #{worker} with count=#{count}"
+      manager.add(worker, count)
+      return 'Successfuly started'
+    rescue Exception => e
+      ModernTimes.logger.error "Exception starting worker #{worker}: {e.message}\n\t#{e.backtrace.join("\n\t")}"
+      return "Exception starting worker #{worker}: {e.message}"
+    rescue java.lang.Exception => e
+      ModernTimes.logger.error "Java exception starting worker #{worker}: {e.message}\n\t#{e.backtrace.join("\n\t")}"
+      return "Java exception starting worker #{worker}: {e.message}"
+    end
+  end
+end

data/lib/modern_times/railsable.rb

@@ -0,0 +1,132 @@
+module ModernTimes
+  module Railsable
+    def init_rails
+      if cfg = YAML.load_file(File.join(Rails.root, "config", "hornetq.yml"))[Rails.env]
+        ModernTimes.logger.info "Messaging Enabled"
+        ModernTimes::HornetQ::Client.init(cfg)
+        @is_hornetq_enabled = true
+
+        # Need to start the HornetQ Server in this VM
+        if ModernTimes::HornetQ::Client.invm?
+          @server = ::HornetQ::Server.create_server('hornetq://invm')
+          @server.start
+
+          # Handle messages within this process
+          @manager = ModernTimes::Manager.new
+          if worker_cfg = cfg[:workers]
+            worker_cfg.each do |klass, count|
+              @manager.add(klass, count)
+            end
+          else
+            rails_workers.each do |klass|
+              @manager.add(klass, 1)
+            end
+          end
+
+          at_exit do
+            @manager.stop if @manager
+            @server.stop
+          end
+        end
+
+        # Create Async Queue and handle requests
+        #self.async_queue_name = self.async_address = "Messaging::Client.async"
+        #self.on_message(self.async_address, self.async_queue_name) do |request|
+        #  self.async_on_message(request)
+        #end
+
+      else
+        Rails.logger.info "Messaging disabled"
+        @is_hornetq_enabled = false
+        ModernTimes::HornetQ::Publisher.setup_dummy_publishing(rails_workers)
+      end
+    end
+
+    def create_rails_manager
+      cfg = YAML.load_file(File.join(Rails.root, "config", "hornetq.yml"))[Rails.env]
+      raise "No valid configuration" unless cfg
+      ModernTimes::HornetQ::Client.init(cfg)
+
+      manager = ModernTimes::Manager.new
+      manager.stop_on_signal
+      manager.allowed_workers = rails_workers
+      manager.persist_file = cfg[:persist_file] || File.join(Rails.root, "log", "modern_times.persist")
+      return manager
+    end
+
+    def rails_workers
+      @rails_workers ||= begin
+        workers = []
+        Dir["#{Rails.root}/app/workers/*_worker.rb"].each do |file|
+          require file
+          workers << File.basename(file).sub(/\.rb$/, '').classify.constantize
+        end
+        workers
+      end
+      #file = "#{Rails.root}/config/workers.yml"
+      #raise "No worker config file #{file}" unless File.exist?(file)
+    end
+
+    def hornetq_enabled?
+      @is_hornetq_enabled
+    end
+  end
+end
+
+
+## Protocol independent class to handle Messaging and Queuing
+#module Messaging
+#  class Client
+#
+#    # Publish to the specified address
+#    #   If the supplied object is kind_of? String, then a string is published
+#    #   Otherwise the Ruby Object is unmarshaled and sent as a Binary message
+#
+#    # Asynchronously invoke the supplied method
+#    #
+#    # Example:
+#    #   Messaging::Client.async(Dashboard, :update_dashboard_for_inquiry, xml_response)
+#    def self.async(klass, method, *param)
+#      @@session_pool.producer(self.async_address) do |session, producer|
+#        request = AsyncRequest.new
+#        request.klass = if klass.kind_of?(String)
+#          klass
+#        elsif klass.kind_of?(Symbol)
+#          klass.to_s
+#        else
+#          klass.name.to_s
+#        end
+#        request.method = method
+#        request.params = *param
+#        message = session.create_message(4,false) #HornetQ::Client::Message::BYTES_TYPE
+#        message['format'] = 'ruby'
+#        message.body = Marshal.dump(request)
+#        producer.send(message)
+#      end
+#    end
+#
+#    private
+#    # Call the specified class passing in the required parameters
+#    # If the method matches a class method, it is called, otherwise
+#    # an instance of the class is created and the method is called
+#    # on the new instance
+#    #
+#    # Note: Instance methods are more expensive because the class is instantiated
+#    #       for every call
+#    def self.async_on_message(request)
+#      klass = request.klass.constantize
+#      method = request.method.to_sym
+#      if klass.respond_to?(method, false)
+#        klass.send(method, *request.params)
+#      else
+#        klass.new.send(method, *request.params)
+#      end
+#    end
+#
+#    # Passed as the request message, used to hold all required parameters
+#    class AsyncRequest
+#      attr_accessor :klass, :method, :params
+#    end
+#
+#  end
+#end

data/lib/modern_times/thread.rb

@@ -0,0 +1,16 @@
+module ModernTimes
+  class Thread < ::Thread
+    def initialize(&block)
+      super() do
+        begin
+          yield
+        rescue => e
+          ModernTimes.logger.fatal("Thread #{self} died due to exception #{e.message}\n#{e.backtrace.join("\n")}")
+        ensure
+          ActiveRecord::Base.clear_active_connections!() if Module.const_get('ActiveRecord') rescue nil
+          ModernTimes.logger.flush if ModernTimes.logger.respond_to?(:flush)
+        end
+      end
+    end
+  end
+end

data/lib/modern_times.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+require 'modern_times/exception'
+require 'modern_times/base'
+require 'modern_times/hornetq'
+require 'modern_times/manager_mbean'
+require 'modern_times/manager'
+require 'modern_times/loggable'
+require 'modern_times/railsable'
+
+module ModernTimes
+  extend ModernTimes::Loggable
+  extend ModernTimes::Railsable
+end