batsir 0.1.0 → 0.3.7

data/lib/batsir/config.rb

@@ -1,34 +1,218 @@
 module Batsir
   class Config
-    def initialize(data={})
-      @data = {}
-      update!(data)
-    end
+    class << self
 
-    def update!(data)
-      data.each do |key, value|
-        self[key] = value
+      # Adapted from Merb::Config class
+
+      attr_accessor :config
+
+      def defaults
+        @defaults ||= {
+          :amqp_host => 'localhost',
+          :amqp_port => 5672,
+          :amqp_user => 'guest',
+          :amqp_pass => 'guest',
+          :amqp_vhost => '/',
+          :amqp_durable => true,
+          :ampq_pool_size => 5,
+          :redis_host => 'localhost',
+          :redis_port => 6379,
+          :redis_database => 0,
+          :redis_namespace => 'batsir',
+          :sidekiq_queue => 'batsir',
+          :log_name => 'batsir'
+        }
+      end
+
+      # Returns the current configuration or sets it up
+      def configuration
+        @config ||= setup
+      end
+
+      # Yields the configuration.
+      #
+      # ==== Block parameters
+      # c<Hash>:: The configuration parameters.
+      #
+      # ==== Examples
+      #   Batsir::Config.use do |config|
+      #     config[:exception_details] = false
+      #     config[:log_stream]        = STDOUT
+      #   end
+      #
+      # ==== Returns
+      # nil
+      #
+      # :api: public
+      def use
+        yield configuration
+        nil
+      end
+
+      # Detects whether the provided key is in the config.
+      #
+      # ==== Parameters
+      # key<Object>:: The key to check.
+      #
+      # ==== Returns
+      # Boolean:: True if the key exists in the config.
+      #
+      # :api: public
+      def key?(key)
+        configuration.key?(key)
+      end
+
+      # Retrieve the value of a config entry.
+      #
+      # ==== Parameters
+      # key<Object>:: The key to retrieve the parameter for.
+      #
+      # ==== Returns
+      # Object:: The value of the configuration parameter.
+      #
+      # :api: public
+      def [](key)
+        configuration[key]
+      end
+
+      # Set the value of a config entry.
+      #
+      # ==== Parameters
+      # key<Object>:: The key to set the parameter for.
+      # val<Object>:: The value of the parameter.
+      #
+      # :api: public
+      def []=(key, val)
+        configuration[key] = val
+      end
+
+      # Remove the value of a config entry.
+      #
+      # ==== Parameters
+      # key<Object>:: The key of the parameter to delete.
+      #
+      # ==== Returns
+      # Object:: The value of the removed entry.
+      #
+      # :api: public
+      def delete(key)
+        configuration.delete(key)
       end
-    end
 
-    def [](key)
-      @data[key.to_sym]
-    end
+      # Resets the configuration to its default state
+      #
+      def reset
+        setup
+      end
+
+      # Retrieve the value of a config entry, returning the provided default if the key is not present
+      #
+      # ==== Parameters
+      # key<Object>:: The key to retrieve the parameter for.
+      # default<Object>::
+      #   The default value to return if the parameter is not set.
+      #
+      # ==== Returns
+      # Object:: The value of the configuration parameter or the default.
+      #
+      # :api: public
+      def fetch(key, default = nil)
+        configuration.fetch(key, default)
+      end
+
+      # Returns the configuration as a hash.
+      #
+      # ==== Returns
+      # Hash:: The config as a hash.
+      #
+      # :api: public
+      def to_hash
+        configuration
+      end
+
+      # Sets up the configuration by storing the given settings.
+      #
+      # ==== Parameters
+      # settings<Hash>::
+      #   Configuration settings to use. These are merged with the defaults.
+      #
+      # ==== Returns
+      # The configuration as a hash.
+      #
+      # :api: private
+      def setup(settings = {})
+        @config = defaults.merge(settings)
+      end
 
-    def []=(key, value)
-      if value.class == Hash
-        @data[key.to_sym] = Config.new(value)
-      else
-        @data[key.to_sym] = value
+      # Set configuration parameters from a code block, where each method
+      # evaluates to a config parameter.
+      #
+      # ==== Parameters
+      # &block:: Configuration parameter block.
+      #
+      # ==== Examples
+      #   # Set environment and log level.
+      #   Batsir::Config.configure do
+      #     environment "development"
+      #     log_level   "debug"
+      #   end
+      #
+      # ==== Returns
+      # nil
+      #
+      # :api: public
+      def configure(&block)
+        ConfigBlock.new(self, &block) if block_given?
+        nil
       end
-    end
 
-    def method_missing(sym, *args)
-      if sym.to_s =~ /(.+)=$/
-        self[$1] = args.first
-      else
-        self[sym]
+      # Allows retrieval of single key config values via Batsir::Config.<key>
+      # Allows single key assignment via Merb.config.<key> = ...
+      #
+      # ==== Parameters
+      # method<~to_s>:: Method name as hash key value.
+      # *args:: Value to set the configuration parameter to.
+      #
+      # ==== Returns
+      # The value of the entry fetched or assigned to.
+      #
+      # :api: public
+      def method_missing(method, *args)
+        if method.to_s[-1,1] == '='
+          self[method.to_s.tr('=','').to_sym] = args.first
+        else
+          self[method]
+        end
       end
-    end
-  end
+
+    end # class << self
+
+    class ConfigBlock
+
+      # Evaluates the provided block, where any call to a method causes
+      # #[]= to be called on klass with the method name as the key and the arguments
+      # as the value.
+      #
+      # ==== Parameters
+      # klass<Object~[]=>:: The object on which to assign values.
+      # &block:: The block which specifies the config values to set.
+      #
+      # ==== Returns
+      # nil
+      #
+      # :api: private
+      def initialize(klass, &block)
+        @klass = klass
+        instance_eval(&block)
+      end
+
+      # Assign args as the value of the entry keyed by method.
+      #
+      # :api: private
+      def method_missing(method, *args)
+        @klass[method] = args.first
+      end
+
+    end # ConfigBlock
+  end # Config
 end

data/lib/batsir/dsl/conditional_notifier_declaration.rb

@@ -0,0 +1,31 @@
+module Batsir
+  module DSL
+    class ConditionalNotifierDeclaration
+      attr_reader :notifier_declarations
+
+      NotifierConditionDeclaration = Struct.new(:condition, :notifier, :options)
+
+      def initialize
+        @notifier_declarations = []
+      end
+
+      def add_conditional(condition, notifier_class, options)
+        @notifier_declarations << NotifierConditionDeclaration.new(condition, notifier_class, options)
+      end
+
+      def compile(output, stage_worker)
+        output << <<-EOF
+              conditional_notifier = Batsir::Notifiers::ConditionalNotifier.new
+        EOF
+        notifier_declarations.each do |notifier_declaration|
+          output << <<-EOF
+              condition = ->(message){#{notifier_declaration.condition}}
+              conditional_notifier.add_notifier condition, #{notifier_declaration.notifier}.new(#{notifier_declaration.options.to_s})
+          EOF
+        end
+        stage_worker.add_transformers_to_notifier("conditional_notifier", output)
+        stage_worker.add_notifier( "conditional_notifier", output)
+      end
+    end
+  end
+end

data/lib/batsir/dsl/dsl_mappings.rb

@@ -32,8 +32,8 @@ module Batsir
         @stage
       end
 
-      def filter(operation)
-        @stage.add_filter(operation)
+      def filter(filter_class, options = {})
+        @stage.add_filter(filter_class, options)
       end
 
       def inbound(&block)
@@ -43,6 +43,7 @@ module Batsir
       def outbound(&block)
         ::Blockenspiel.invoke(block, Batsir::DSL::OutboundMapping.new(@stage))
       end
+
     end
 
     class InboundMapping < ::Blockenspiel::Base
@@ -78,6 +79,14 @@ module Batsir
         ::Blockenspiel.invoke(block, Batsir::DSL::OutboundTransformerMapping.new(@stage))
       end
 
+      def conditional(&block)
+        new_block = ::Proc.new do
+          conditional &block
+        end
+        conditional = ::Blockenspiel.invoke(new_block, Batsir::DSL::ConditionalNotifierMapping.new)
+        @stage.add_conditional_notifier(conditional)
+      end
+
       def notifier(notifier_class, options = {})
         @stage.add_notifier(notifier_class, options)
       end
@@ -92,5 +101,21 @@ module Batsir
         @stage.add_notifier_transformer(transformer, options)
       end
     end
+
+    class ConditionalNotifierMapping < ::Blockenspiel::Base
+      def initialize
+        @notifier = nil
+      end
+
+      def conditional(&block)
+        @notifier = Batsir::DSL::ConditionalNotifierDeclaration.new
+        ::Blockenspiel.invoke(block, self)
+        @notifier
+      end
+
+      def notify_if(condition, notifier, options = {})
+        @notifier.add_conditional(condition, notifier, options)
+      end
+    end
   end
 end

data/lib/batsir/errors.rb

@@ -0,0 +1,18 @@
+module Batsir
+  module Errors
+
+    class BatsirError < RuntimeError; end
+    class NotifierError < BatsirError; end
+    class TransformError < BatsirError; end
+    class StrategyError < BatsirError; end
+
+    class ExecuteMethodNotImplementedError < BatsirError; end
+
+    class NotifierConnectionError < NotifierError; end
+
+    class JSONInputTransformError < TransformError; end
+    class JSONOutputTransformError < TransformError; end
+
+    class RetryStrategyFailed < StrategyError; end
+  end
+end

data/lib/batsir/filter.rb

@@ -6,7 +6,12 @@ module Batsir
       end
     end
 
+    def filter(message)
+      execute(message)
+    end
+
     def execute(message)
+      raise NotImplementedError.new
     end
   end
 end

data/lib/batsir/log.rb

@@ -0,0 +1,7 @@
+module Batsir
+  module Log
+    def log
+      Batsir::Logger
+    end
+  end
+end

data/lib/batsir/logger.rb

@@ -0,0 +1,37 @@
+require 'log4r'
+
+module Batsir
+  module Logger
+    DEBUG  = 1
+    INFO   = 2
+    WARN   = 3
+    ERROR  = 4
+    FATAL  = 5
+
+    class << self
+
+      DEFAULT_OUTPUT = 'stdout'
+
+      def log
+        @logger ||= setup
+      end
+
+      def setup
+        logger = Log4r::Logger.new(Batsir::Config.fetch(:log_name, "batsir"))
+        logger.level      = Batsir::Config.fetch(:log_level, WARN)
+        logger.outputters = Batsir::Config.fetch(:log_outputter, DEFAULT_OUTPUT)
+        logger
+      end
+
+      def reset
+        @logger = nil
+      end
+
+      # makes this respond like a Log4r::Logger
+      def method_missing(sym, *args, &block)
+        log.send sym, *args, &block
+      end
+
+    end
+  end
+end

data/lib/batsir/logo.rb

@@ -2,10 +2,6 @@ module Batsir
   def self.logo
   <<EOF
 
-
-
-
-
                                    E        ,.
                    ,,,agB8@f      ,8v      ,8L      J88&gg,,,
               .,a88888888@^       |8&,,,,,,88k       ,88888888&g,,
@@ -26,10 +22,9 @@ module Batsir
               `?9@888&,               j88f               ,d88$@9l'
                    `|?9t               |f               `9T"`
 
-
-
-
-
+                  ______  _______ _______ _______ _____ ______
+                  |_____] |_____|    |    |______   |   |_____/
+                  |_____] |     |    |    ______| __|__ |    \\_
 
 EOF
   end

data/lib/batsir/notifiers/amqp_notifier.rb

@@ -5,11 +5,21 @@ module Batsir
     class AMQPNotifier < Notifier
       include Batsir::AMQP
 
+      attr_reader :error_strategy
+
+      def initialize(options = {}, error_strategy = Batsir::Strategies::RetryStrategy)
+        super(options)
+        @error_strategy = error_strategy.new(self)
+        @bunny = Bunny.new(bunny_options).start
+        @x = @bunny.exchange( exchange )
+      end
+
       def execute(message)
-        Bunny.run( bunny_options )do |bunny|
-          exc = bunny.exchange( exchange )
-          exc.publish( message, :key => queue )
-        end
+        @x.publish(message, :routing_key => queue)
+      end
+
+      def handle_error(message, error)
+        @error_strategy.execute(message, error)
       end
     end
   end

data/lib/batsir/notifiers/conditional_notifier.rb

@@ -0,0 +1,29 @@
+module Batsir
+  module Notifiers
+    class ConditionalNotifier < Notifier
+      attr_accessor :notifiers
+
+      NotifierCondition = Struct.new(:condition, :notifier, :options)
+
+      def initialize(options = {})
+        super
+        @notifiers = []
+      end
+
+      def add_notifier( condition, notifier_class, options = {})
+        self.notifiers << NotifierCondition.new(condition, notifier_class, options)
+      end
+
+      def execute(message)
+        self.notifiers.each do |notifier_condition|
+          if notifier_condition.condition.call(message)
+            notifier = notifier_condition.notifier
+            options  = notifier_condition.options
+            notifier.notify(message)
+          end
+        end
+        message
+      end
+    end
+  end
+end

data/lib/batsir/notifiers/notifier.rb

@@ -21,11 +21,8 @@ module Batsir
       end
 
       def notify(message)
-        execute(transform(message))
-      end
-
-      def execute(message)
-
+        execute(transform(message.clone))
+        message
       end
 
       def transform(message)
@@ -34,6 +31,10 @@ module Batsir
         end
         message
       end
+
+      def execute(message)
+        raise NotImplementedError.new
+      end
     end
   end
 end

data/lib/batsir/registry.rb

@@ -5,11 +5,15 @@ module Batsir
     end
 
     def self.registry
-      @registry ||= {}
+      @registry || reset
     end
 
     def self.get(name)
-      registry.fetch(name)
+      registry.fetch(name, nil)
+    end
+
+    def self.reset
+      @registry = {}
     end
   end
 end

data/lib/batsir/stage.rb

@@ -10,6 +10,7 @@ module Batsir
     attr_accessor :cancellators
     attr_reader   :filter_declarations
     attr_reader   :notifiers
+    attr_reader   :conditional_notifiers
     attr_reader   :acceptors
     attr_reader   :running_acceptors
     attr_reader   :notifier_transformers
@@ -25,6 +26,7 @@ module Batsir
       @acceptors              = {}
       @filter_declarations    = []
       @notifiers              = {}
+      @conditional_notifiers  = []
       @notifier_transformers  = []
       @built                  = false
     end
@@ -50,6 +52,10 @@ module Batsir
       @filter_declarations.map{ |filter_declaration| filter_declaration.filter }
     end
 
+    def add_conditional_notifier(notifier_declaration)
+      @conditional_notifiers << notifier_declaration
+    end
+
     def add_notifier(notifier, options = {})
       @notifiers[notifier] ||= Set.new
       @notifiers[notifier] << options
@@ -72,10 +78,9 @@ module Batsir
     def start
       acceptors.each do |acceptor_class, options|
         options.each do |acceptor_options|
-          cancellator_reader, cancellator_writer = ::IO.pipe
-          acceptor_options.merge!(:stage_name => self.name, :cancellator => cancellator_reader)
-          @cancellators << cancellator_writer
+          acceptor_options.merge!(:stage_name => self.name)
           acceptor = acceptor_class.new(acceptor_options)
+
           if acceptor_transformers.any?
             acceptor_transformers.each do |transformer_declaration|
               transformer = transformer_declaration.transformer.new(transformer_declaration.options)
@@ -85,7 +90,7 @@ module Batsir
             acceptor.add_transformer(Batsir::Transformers::JSONInputTransformer.new)
           end
           @running_acceptors << acceptor
-          acceptor.start!
+          acceptor.async.start
         end
       end
       true

data/lib/batsir/stage_worker.rb

@@ -14,9 +14,10 @@ module Batsir
     end
 
     def execute(message)
+      return false if message.nil?
       return false unless @filter_queue
       @filter_queue.filters.each do |filter|
-        message = filter.execute(message)
+        message = filter.filter(message)
         return false if message.nil?
       end
       @filter_queue.notifiers.each do |notifier|
@@ -26,61 +27,7 @@ module Batsir
     end
 
     def self.compile_from(stage)
-      code = <<-EOF
-        class #{stage.name.capitalize.gsub(' ','')}Worker
-          def self.stage_name
-            "#{stage.name}"
-          end
-
-          def initialize
-            @filter_queue = self.class.filter_queue
-          end
-
-          def self.filter_queue
-            @filter_queue
-          end
-
-          def self.initialize_filter_queue
-            @filter_queue = Batsir::FilterQueue.new
-      EOF
-
-      stage.filter_declarations.each do |filter_declaration|
-        code << <<-EOF
-            @filter_queue.add #{filter_declaration.filter.to_s}.new(#{filter_declaration.options.to_s})
-        EOF
-      end
-
-      stage.notifiers.each do |notifier, options_set|
-        options_set.each do |options|
-          code << <<-EOF
-            notifier = #{notifier.to_s}.new(#{options.to_s})
-          EOF
-
-          if stage.notifier_transformers.any?
-            stage.notifier_transformers.each do |transformer_declaration|
-              code << <<-EOF
-            notifier.add_transformer #{transformer_declaration.transformer}.new(#{transformer_declaration.options.to_s})
-              EOF
-            end
-          else
-            code << <<-EOF
-            notifier.add_transformer Batsir::Transformers::JSONOutputTransformer.new
-            EOF
-          end
-          code << <<-EOF
-            @filter_queue.add_notifier notifier
-          EOF
-        end
-      end
-
-      code << <<-EOF
-          end
-
-          include Sidekiq::Worker
-          include Batsir::StageWorker
-        end
-      EOF
-      code
+      Batsir::Compiler::StageWorkerCompiler.new(stage).compile
     end
   end
 end

data/lib/batsir/strategies/retry_strategy.rb

@@ -0,0 +1,35 @@
+module Batsir
+  module Strategies
+    class RetryStrategy < Strategy
+      include Batsir::Log
+
+      attr_reader :retries, :attempts
+
+      def initialize(context, retries = 3)
+        super(context)
+        @retries  = retries
+        @attempts = {}
+      end
+
+      def execute(message, error)
+        @attempts[message] ? @attempts[message] += 1 : @attempts[message] = 0
+
+        if @attempts[message] >= @retries
+          error_msg = "Tried to send '#{message}' #{@attempts[message]} times and failed"
+          reset_attempts(message)
+          log.error error_msg
+          raise Batsir::Errors::RetryStrategyFailed.new error_msg
+        else
+          log.warn "Recovering from #{error}. Making another attempt (##{@attempts[message]+1})"
+          result = @context.execute(message)
+          reset_attempts(message)
+          return result
+        end
+      end
+
+      def reset_attempts(message)
+        @attempts.delete(message)
+      end
+    end
+  end
+end