emissary 1.3.18 → 1.3.19

data/lib/emissary/logger.rb

@@ -1,130 +0,0 @@
-#   Copyright 2010 The New York Times
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-#
-# $Id $
-# author: Carl P. corliss
-# Copyright 2009, The New York Times
-#
-require 'syslog'
-require 'fastthread'
-
-module Emissary
-	class Logger
-		
-		LOG_SYSLOG = 0x01
-		LOG_STDOUT = 0x02
-    LOG_STDERR = 0x03
-
-    EMERGENCY = Syslog::Constants::LOG_EMERG
-    EMERG     = Syslog::Constants::LOG_EMERG
-    ALERT     = Syslog::Constants::LOG_ALERT
-    FATAL     = Syslog::Constants::LOG_CRIT
-    CRITICAL  = Syslog::Constants::LOG_CRIT
-    CRIT      = Syslog::Constants::LOG_CRIT
-    ERROR     = Syslog::Constants::LOG_ERR
-    ERR       = Syslog::Constants::LOG_ERR
-    WARNING   = Syslog::Constants::LOG_WARNING
-    WARN      = Syslog::Constants::LOG_WARNING
-    NOTICE    = Syslog::Constants::LOG_NOTICE
-    INFO      = Syslog::Constants::LOG_INFO
-    DEBUG     = Syslog::Constants::LOG_DEBUG
-    
-    CONSTANT_ID_MAP = {
-      EMERGENCY => [ :emergency, :emerg ],
-      ALERT     => [ :alert ],
-      CRITICAL  => [ :fatal, :critical, :crit ],
-      ERROR     => [ :error, :err ],
-      WARNING   => [ :warning, :warn ],
-      NOTICE    => [ :notice ],
-      INFO      => [ :info ],
-      DEBUG     => [ :debug ]
-    }
-
-    CONSTANT_NAME_MAP = CONSTANT_ID_MAP.inject({}) do |mappings,(id,names)|
-      names.each { |name| mappings.merge!({ name => id, name.to_s.upcase => id, name.to_s.downcase => id }) }
-      mappings
-    end
-
-    attr_accessor :mode, :level, :name
-    
-		def self.instance(mode = LOG_STDERR, log_level = NOTICE, name = nil)
-			@@logger ||= Emissary::Logger.new mode, log_level
-		end
-
-		private :initialize
-		
-		def initialize mode, log_level, name = nil
-			@mode  = mode || LOG_STDERR
-			@level = log_level || INFO
-      @name  = name
-      @mutex = Mutex.new
-		end
-
-    def name=(p) @name = n || File.basename($0); end
-    
-    def normalize log_level
-      return log_level unless not log_level.kind_of? Fixnum
-      return CONSTANT_NAME_MAP[log_level.to_s.downcase] rescue INFO
-      return INFO
-    end
-    
-    def level_to_sym
-      CONSTANT_ID_MAP[@level].first
-    end
-    
-    def level_to_s
-      level_to_sym.to_s.upcase
-    end
-    
-		def loggable?(log_level)
-			log_level <= normalize(@level)
-		end
-
-    def syslogger
-      @mutex.synchronize {
-        Syslog.open(name, Syslog::LOG_PID | Syslog::LOG_NDELAY, Syslog::LOG_DAEMON) do |s|
-          s.LOG_UPTO(level)
-          yield s
-        end
-      }
-    end      
-		
-		def log(log_level, message, *args)
-			case @mode
-				when Logger::LOG_SYSLOG
-          messages = "#{CONSTANT_ID_MAP[log_level].first.to_s.upcase}: #{message}".gsub(/\t/, ' '*8).split(/\n/)
-          # some syslog daemons have a hard time with newlines, so split into multiple lines
-					messages.each do |message|
-            syslogger { |s| s.log(log_level, message, *args) }
-					end
-				when Logger::LOG_STDOUT
-					$stdout << sprintf(message << "\n", *args) unless not loggable? log_level
-				when Logger::LOG_STDERR
-					$stderr << sprintf(message << "\n", *args) unless not loggable? log_level
-			end
-			
-			self
-		end
-
-		CONSTANT_ID_MAP.values.flatten.each do |log_level|
-			class_eval %(
-				def #{log_level}(message, *args)
-					log(#{'::Emissary::Logger::' + log_level.to_s.upcase}, message, *args)
-				end
-			)
-		end
-	end
-end

data/lib/emissary/message.rb

@@ -1,217 +0,0 @@
-#   Copyright 2010 The New York Times
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-require 'uuid'
-require 'bert'
-
-module Emissary
-  class Message
-
-    attr_accessor :sender, :recipient, :replyto, :errors
-    attr_accessor :status, :operation, :thread, :time
-    attr_accessor :account, :agent, :method, :args
-    attr_reader   :uuid, :originator
-    
-    def initialize(payload = {})
-      @recipient  = ''
-      @sender     = Emissary.identity.queue_name
-      @replyto    = Emissary.identity.queue_name
-      @originator = @sender.dup
-      
-      @status    = [ :ok, '' ] # tuple of (status type, status message)
-      @operation = -1
-      @thread    = -1
-      @uuid      = UUID.new.generate
-
-      @errors  = []
-      
-      @agent   = @method = nil
-      @account = Emissary.identity.account_id
-      @args    = []
-
-      @time = {
-        :received => nil,
-        :sent     => nil,
-      }
-      
-      payload = {
-        :headers => {
-          :recipient  => @recipient,
-          :sender     => @sender,
-          :replyto    => @replyto,
-          :originator => @originator,
-          :status     => @status, 
-          :operation  => @operation,
-          :thread     => @thread,
-          :uuid       => @uuid,
-          :time       => @time.merge((payload[:time].symbolize rescue {})),
-        }.merge((payload[:headers].symbolize rescue {})),
-        :data   => {
-          :account => @account,
-          :agent   => @agent,
-          :method  => @method,
-          :args    => @args
-        }.merge((payload[:data].symbolize! rescue {})),
-        :errors => [ ] + (payload[:errors].symbolize rescue [])
-      }
-
-      payload[:headers].merge(payload[:data]).each do |k,v|
-        send("#{k}=".to_sym, v) rescue nil
-      end
-      
-      payload[:errors].each do |e|
-        exception = ::Emissary.klass_const(e[:type]).new(e[:message]) rescue StandardError.new("#{e[:type]}: #{e[:message]}")
-        exception.set_backtrace(e[:backtrace])
-        errors << exception
-      end
-      
-      @agent   = @agent.to_sym rescue nil
-      @method  = @method.to_sym rescue nil
-      @args    = @args || [] rescue []
-    end
-
-    def headers()
-      {
-        :recipient  => recipient,
-        :sender     => sender,
-        :replyto    => replyto,
-        :originator => originator,
-        :status     => status,
-        :operation  => operation,
-        :thread     => thread,
-        :time       => time,
-        :uuid       => uuid
-      }
-    end
-    
-    def data()
-      return { :account => account, :agent => agent, :method => method, :args => args }
-    end
-
-    def errors type = :default
-      case type
-        when :hashes
-          @errors.collect do |e|
-            { :type => e.class.name, :message => e.message, :backtrace => e.backtrace }
-          end
-      else
-        @errors
-      end
-    end
-    
-    def status_type=(t) status[0] = t.to_sym; end
-    def status_type() status[0] || :ok rescue :ok; end
-
-    def status_note=(n) status[1] = n; end
-    def status_note() status[1] || ''  rescue '' ; end
-
-    def route who = :recipient
-      headers[who].split(':') || [] rescue []
-    end
-    
-    def routing_key who = :recipient
-      route(who)[0] || nil rescue nil
-    end
-
-    def exchange who = :recipient
-      [ exchange_type(who), exchange_name(who) ]
-    end
-    
-    def exchange_type who = :recipient
-      route(who)[1].to_sym || :direct rescue :direct
-    end
-    
-    def exchange_name who = :recipient
-      key, type, name = route(who) rescue [ nil, :direct, 'amq.direct' ]
-      name || case type.to_sym
-        when :fanout, :topic, :matches, :headers
-          "amq.#{type}"
-        else
-          'amq.direct'
-      end
-    rescue
-      'amq.direct'
-    end
-    
-    def canonical_route who = :recipient
-      "#{routing_key(who)}:#{exchange(who).join(':')}"
-    end
-    
-    def will_loop?
-      canonical_route(:recipient) == canonical_route(:originator)
-    end
-    
-    def encode
-      BERT.encode({ :headers => headers, :data => data, :errors => errors(:hashes) })
-    end
-
-    def self.decode payload
-      begin
-        self.new BERT.decode(payload)
-      rescue StandardError => e
-        raise e unless e.message =~ /bad magic/i
-        Emissary.logger.error "Unable to decode message - maybe it wasn't encoded with BERT..?"
-        raise ::Emissary::Error::InvalidMessageFormat, "Unable to decode message - maybe it wasn't encoded with BERT? Message: #{payload.inspect}"
-      end
-    end
-
-    def stamp_sent!
-      time[:sent] = Time.now.to_f
-      self
-    end
-    
-    def stamp_received!
-      time[:received] = Time.now.to_f
-      self
-    end
-    
-    def trip_time
-      (time[:sent].to_f - time[:received].to_f rescue 0.0) || 0.0
-    end
-    
-    def response
-      header = {
-        :recipient => replyto || sender,
-        :status    => [ :ok, '' ],
-        :operation => operation,
-        :thread    => thread,
-        :time      => {
-          :received => nil,
-          :sent     => nil
-        }
-      }
-      return Message.new({ :headers => header, :data => data, :errors => errors(:hashes) })
-    end
-
-    def bounce(message = nil)
-      message ||= 'Message failed due to missing handler.'
-      bounced = self.response
-      bounced.status = [ :bounced, message ] 
-      return bounced
-    end
-
-    def error(message = nil)
-      message ||= 'Message failed due to unspecified error.'
-      error = self.response
-      error.status = [ :errored, message.to_s ]
-      if message.is_a? Exception
-        error.errors << message
-      else
-        ::Emissary.logger.warning "#{message.class.name} is not an exception..."
-      end
-      return error
-    end
-  end
-end

data/lib/emissary/operator.rb

@@ -1,275 +0,0 @@
-#   Copyright 2010 The New York Times
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-require 'monitor'
-require 'work_queue'
-require 'fastthread'
-
-module Emissary
-  module OperatorStatistics
-    RX_COUNT_MUTEX = Mutex.new
-    TX_COUNT_MUTEX = Mutex.new
-    
-    attr_reader :rx_count, :tx_count
-    def increment_tx_count
-      TX_COUNT_MUTEX.synchronize {
-        @tx_count = (@tx_count + 1 rescue 1)
-      }
-    end
-    
-    def tx_count
-      count = 0
-      TX_COUNT_MUTEX.synchronize {
-        count = @tx_count
-        @tx_count = 0
-      }
-      count
-    end
-    
-    def increment_rx_count
-      RX_COUNT_MUTEX.synchronize {
-        @rx_count = (@rx_count + 1 rescue 1)
-      }
-    end
-    
-    def rx_count
-      count = 0
-      RX_COUNT_MUTEX.synchronize {
-        count = @rx_count
-        @rx_count = 0
-      }
-      count
-    end
-  end
-  
-  class Operator
-    include Emissary::OperatorStatistics
-    
-    DEFAULT_STATUS_INTERVAL = 3600
-    DEFAULT_MAX_WORKERS     = 50
-    MAX_WORKER_TTL          = 60
-    
-    attr_reader   :config, :shutting_down, :signature 
-
-    # Override .new so subclasses don't have to call super and can ignore
-    # connection-specific arguments
-    #
-    def self.new(config, *args)
-      allocate.instance_eval do
-        # Store signature
-        @signature = config[:signature]
-
-        # Call a superclass's #initialize if it has one
-        initialize(config, *args)
-
-        # post initialize callback
-        post_init
-    
-        # set signature nil
-        @signature ||= Digest::MD5.hexdigest(config.to_s)
-        
-        self
-      end
-    end
-
-    def initialize(config, *args)
-      @config    = config
-      @workers   = (args[0][:max_workers] || DEFAULT_MAX_WORKERS rescue DEFAULT_MAX_WORKERS)
-
-      @agents    = WorkQueue.new(@workers, nil, MAX_WORKER_TTL)
-      @publisher = WorkQueue.new(@workers, nil, MAX_WORKER_TTL)
-
-      @timer     = nil
-      @stats     = WorkQueue.new(1, nil, MAX_WORKER_TTL)
-
-      @rx_count  = 0
-      @tx_count  = 0
-
-      @shutting_down = false
-      @connected = false
-    end
-
-    def connected?() @connected; end
-
-    def post_init
-    end
-
-    def connect
-      raise NotImplementedError, 'The connect method must be defined by the operator module'
-    end
-
-    def subscribe
-      raise NotImplementedError, 'The subscrie method must be defined by the operator module'
-    end
-
-    def unsubscribe
-      raise NotImplementedError, 'The unsubscribe method must be defined by the operator module'
-    end
-
-    def acknowledge message
-    end
-    
-    def reject message, requeue = true
-    end
-    
-    def send_data
-      raise NotImplementedError, 'The send_data method must be defined by the operator module'
-    end
-    
-    def close
-      raise NotImplementedError, 'The close method must be defined by the operator module'
-    end
-
-    def run
-      @connected = !!connect
-      subscribe 
-      schedule_statistics_gatherer
-      notify :startup
-      connected?
-    end
-
-    def disconnect
-      close
-      @connected = false
-    end
-
-    def shutting_down?() @shutting_down; end
-    
-    def shutdown!
-      unless shutting_down?
-        @shutting_down = true
-  
-        Emissary.logger.info "Cancelling periodic timer for statistics gatherer..."
-        @timer.cancel
-        
-        Emissary.logger.notice "Shutting down..."
-        notify :shutdown
-  
-        Emissary.logger.info "Shutting down agent workqueue..."
-        @agents.join
-  
-        Emissary.logger.info "Shutting down publisher workqueue..."
-        @publisher.join
-  
-        Emissary.logger.info "Disconnecting..."
-        disconnect
-      end
-    end
-
-    def enabled? what
-      unless [ :startup, :shutdown, :stats ].include? what.to_sym
-        Emissary.logger.debug "Testing '#{what}' - it's disabled. Not a valid option."
-        return false
-      end
-      
-      unless config[what]
-        Emissary.logger.debug "Testing '#{what}' - it's disabled. Missing from configuration."
-        return false
-      end
-      
-      if (config[:disable]||[]).include? what.to_s
-        Emissary.logger.debug "Testing '#{what}' - it's disabled. Listed in 'disable' configuration option."
-        return false
-      end
-      
-      Emissary.logger.debug "Testing '#{what}' - it's enabled.."
-      return true
-    end
-    
-    def received message
-      acknowledge message
-    end
-
-    def rejected message, opts = { :requeue => true }
-      reject message, opts
-    end
-    
-    def receive message
-      @agents.enqueue_b {
-        begin
-          raise message.errors.first unless message.errors.empty? or not message.errors.first.is_a? Exception
-          Emissary.logger.debug " ---> [DISPATCHER] Dispatching new message ... "
-          Emissary.dispatch(message, config, self).activate
-          # ack message if need be (operator dependant)
-          received message
-        rescue ::Emissary::Error::InvalidMessageFormat => e
-          Emissary.logger.warning e.message
-          rejected message, :requeue => true
-          # if it was an encoding error, then we are done - nothing more we can do
-        rescue Exception => e
-          Emissary.logger.error "AgentThread Error: #{e.class.name}: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
-          send message.error(e)
-          rejected message, :requeue => true
-        else
-          increment_rx_count
-        end
-        Emissary.logger.debug " ---> [DISPATCHER] tasks/workers: #{@agents.cur_tasks}/#{@agents.cur_threads}"
-      }
-    end
-
-    def send message
-      @publisher.enqueue_b {
-        Emissary.logger.debug " ---> [PUBLISHER]  Sending new message ... "
-        begin
-          unless message.will_loop?
-            Emissary.logger.debug "[PUBLISHER] -- Sending message..."
-            send_data message
-            increment_tx_count
-          else
-            Emissary.logger.notice "Not sending message destined for myself - would loop."
-          end
-        rescue Exception => e
-          Emissary.logger.error "PublisherThread Error: #{e.class.name}: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
-          @shutting_down = true
-        end
-        Emissary.logger.debug " ---> [PUBLISHER]  tasks/workers: #{@publisher.cur_tasks}/#{@publisher.cur_threads}"
-      }
-    end
-
-    def notify type
-      return unless enabled? type and EM.reactor_running?
-      
-      message = Emissary::Message.new(:data => { :agent => :emissary, :method => type })
-      case type
-        when :startup, :shutdown
-          message.recipient = config[type]
-        when :stats
-          message.agent = :stats
-          message.method = :gather
-      end
-
-      Emissary.logger.notice "Running #{type.to_s.capitalize} Notifier"
-      receive message
-    end
-    
-    def schedule_statistics_gatherer
-      stats_interval = enabled?(:stats) && config[:stats][:interval] ? config[:stats][:interval].to_i : DEFAULT_STATUS_INTERVAL
-      
-      # setup agent to process sending of messages
-      @timer = EM.add_periodic_timer(stats_interval) do
-        rx = rx_count; tx = tx_count
-        rx_throughput = sprintf "%0.4f", (rx.to_f / stats_interval.to_f)
-        tx_throughput = sprintf "%0.4f", (tx.to_f / stats_interval.to_f)
-        
-        Emissary.logger.notice "[statistics] publisher tasks/workers: #{@publisher.cur_tasks}/#{@publisher.cur_threads}"
-        Emissary.logger.notice "[statistics] dispatcher tasks/workers: #{@agents.cur_tasks}/#{@agents.cur_threads}"
-        Emissary.logger.notice "[statistics] #{tx} in #{stats_interval} seconds - tx rate: #{tx_throughput}/sec"
-        Emissary.logger.notice "[statistics] #{rx} in #{stats_interval} seconds - rx rate: #{rx_throughput}/sec"
-        
-        notify :stats
-      end
-    end
-  end
-end