startback 0.11.3 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83733eb966d9d9679a15840dfa76f717aacaacc5c3eb417c6bf518a3eea1e3cd
-  data.tar.gz: f13815553606f4567e136beb86fc19a066720a3bcdf520c3781c12a732808f47
+  metadata.gz: 67ffd7abceaf1b4e048cc8f02f668b682ea81f62fc7de5228a9f0d12abab1df8
+  data.tar.gz: 1eb075bdec30a569726bc3f2a74c1ce427aef8ff0c3cbcee401a61c034e71ca6
 SHA512:
-  metadata.gz: 0a2c80c7c65fbe845784a6ee1326bed2a1d6f89eed591580800e1ee9f403fea24b61e92b1f1ad13356f612308a30c6ecc17f6c6a16fadb9c82ae6dec61b68d31
-  data.tar.gz: 18dc6c715ba6ed1715c6d838421476959942611a120ba04eed7a72d40d505e53c60b64ecbf4e877d2a99d2c46300cfa8e83f6cb70960bb72ad599d7d5802509f
+  metadata.gz: 8cb4d6f1302a34cab267c836c1e0d81c54267607de7f24d350183ff05a79af7cc216a6f34bc96c0e1204f71cca1de744fde448590e99bd96a1ab45c15c8ceddf
+  data.tar.gz: fecf573e39eb04317d756414349d94092f24a07210629a55347191ef74ec94fcc8079f8a4f38b6c58bf94be5c59fbd79a90a168a4925819ce771269ccae6e3da

data/lib/startback/event/agent.rb

@@ -0,0 +1,71 @@
+module Startback
+  class Event
+    #
+    # An agent listen to specific events and react with its
+    # `call` method.
+    #
+    # This class is intended to be subclasses and the following
+    # methods overriden:
+    #
+    #   - install_listeners that installs sync and async listeners
+    #   - call to create a context and implement reaction behavior
+    #
+    class Agent
+      include Support::OperationRunner
+      include Support::Robustness
+
+      def initialize(engine)
+        @engine = engine
+        install_listeners
+      end
+      attr_reader :engine
+
+    protected
+
+      # Installs the various event handlers by calling `sync`
+      # and `async` methods.
+      #
+      # This method is intended to be overriden.
+      def install_listeners
+      end
+
+      # Returns the underlying bus
+      def bus
+        engine.bus
+      end
+
+      # Asynchronously listen to a specific event.
+      #
+      # See Bus#listen
+      def async(exchange, queue)
+        bus.async.listen(exchange, queue) do |event|
+          dup.call(event)
+        end
+      end
+
+      # Synchronously listen to a specific event.
+      #
+      # See Bus#listen
+      def sync(exchange, queue)
+        bus.listen(exchange, queue) do |event|
+          dup.call(event)
+        end
+      end
+
+      # Reacts to a specific event.
+      #
+      # This method must be implemented by subclasses and raises
+      # an error by default.
+      def call(event = nil)
+        log(:fatal, {
+          op: self.class,
+          op_data: event,
+          error: %Q{Unexpected call to Startback::Event::Agent#call},
+          backtrace: caller
+        })
+        raise NotImplementedError
+      end
+
+    end # class Agent
+  end # class Event
+end # module Starback

data/lib/startback/event/bus/bunny/async.rb

@@ -0,0 +1,165 @@
+require 'bunny'
+module Startback
+  class Event
+    class Bus
+      module Bunny
+        #
+        # Asynchronous implementation of the bus abstraction, on top of RabbitMQ
+        # and using the 'bunny' gem (you need to include it in your Gemfile
+        # yourself: it is NOT a startback official dependency).
+        #
+        # This bus implementation emits events by dumping them to RabbitMQ using
+        # the event type as exchange name. Listeners may use the `processor`
+        # parameter to specify the queue name ; otherwise a default "main" queue
+        # is used.
+        #
+        # Examples:
+        #
+        #     # Connects to RabbitMQ using all default options
+        #     #
+        #     # Uses the STARTBACK_BUS_BUNNY_ASYNC_URL environment variable for
+        #     # connection URL if present.
+        #     Startback::Bus::Bunny::Async.new
+        #
+        #     # Connects to RabbitMQ using a specific URL
+        #     Startback::Bus::Bunny::Async.new("amqp://rabbituser:rabbitpass@192.168.17.17")
+        #     Startback::Bus::Bunny::Async.new(url: "amqp://rabbituser:rabbitpass@192.168.17.17")
+        #
+        #     # Connects to RabbitMQ using specific connection options. See Bunny's own
+        #     # documentation
+        #     Startback::Bus::Bunny::Async.new({
+        #       connection_options: {
+        #         host: "192.168.17.17"
+        #       }
+        #     })
+        #
+        class Async
+          include Support::Robustness
+
+          CHANNEL_KEY = 'Startback::Bus::Bunny::Async::ChannelKey'
+
+          DEFAULT_OPTIONS = {
+            # (optional) The URL to use for connecting to RabbitMQ.
+            url: ENV['STARTBACK_BUS_BUNNY_ASYNC_URL'],
+
+            # (optional) The options has to pass to ::Bunny constructor
+            connection_options: nil,
+
+            # (optional) The options to use for the emitter/listener fanout
+            fanout_options: {},
+
+            # (optional) The options to use for the listener queue
+            queue_options: {},
+
+            # (optional) Default event factory to use, if any
+            event_factory: nil,
+
+            # (optional) A default context to use for general logging
+            context: nil,
+
+            # (optional) Size of consumer pool
+            consumer_pool_size: 1,
+
+            # (optional) Whether the program must be aborted on consumption
+            # error
+            abort_on_exception: true,
+
+            # (optional) Whether connection occurs immediately,
+            # or on demand later
+            autoconnect: false
+          }
+
+          # Creates a bus instance, using the various options provided to
+          # fine-tune behavior.
+          def initialize(options = {})
+            options = { url: options } if options.is_a?(String)
+            @options = DEFAULT_OPTIONS.merge(options)
+            connect if @options[:autoconnect]
+          end
+          attr_reader :options
+
+          def connect
+            disconnect
+            conn = options[:connection_options] || options[:url]
+            try_max_times(10) do
+              @bunny = ::Bunny.new(conn)
+              @bunny.start
+              channel # make sure we already create the channel
+              log(:info, {op: "#{self.class.name}#connect", op_data: conn}, options[:context])
+            end
+          end
+
+          def disconnect
+            if channel = Thread.current[CHANNEL_KEY]
+              channel.close
+              Thread.current[CHANNEL_KEY] = nil
+            end
+            @bunny.close if @bunny
+          end
+
+          def channel
+            unless @bunny
+              raise Startback::Errors::Error, "Please connect your bus first, or use autoconnect: true"
+            end
+
+            Thread.current[CHANNEL_KEY] ||= @bunny.create_channel(
+              nil,
+              consumer_pool_size, # consumer_pool_size
+              abort_on_exception? # consumer_pool_abort_on_exception
+            )
+          end
+
+          def emit(event)
+            stop_errors(self, "emit", event.context) do
+              fanout = channel.fanout(event.type.to_s, fanout_options)
+              fanout.publish(event.to_json)
+            end
+          end
+
+          def listen(type, processor = nil, listener = nil, &bl)
+            raise ArgumentError, "A listener must be provided" unless listener || bl
+
+            fanout = channel.fanout(type.to_s, fanout_options)
+            queue = channel.queue((processor || "main").to_s, queue_options)
+            queue.bind(fanout)
+            queue.subscribe do |delivery_info, properties, body|
+              event = stop_errors(self, "listen") do
+                factor_event(body)
+              end
+              stop_errors(self, "listen", event.context) do
+                (listener || bl).call(event)
+              end
+            end
+          end
+
+        protected
+
+          def consumer_pool_size
+            options[:consumer_pool_size]
+          end
+
+          def abort_on_exception?
+            options[:abort_on_exception]
+          end
+
+          def fanout_options
+            options[:fanout_options]
+          end
+
+          def queue_options
+            options[:queue_options]
+          end
+
+          def factor_event(body)
+            if options[:event_factory]
+              options[:event_factory].call(body)
+            else
+              Event.json(body, options)
+            end
+          end
+
+        end # class Async
+      end # module Bunny
+    end # class Bus
+  end # class Event
+end # module Startback

data/lib/startback/event/bus/memory/async.rb

@@ -0,0 +1,45 @@
+module Startback
+  class Event
+    class Bus
+      module Memory
+        #
+        # Asynchronous implementation of the Bus abstraction, for use between
+        # components sharing the same process.
+        #
+        # This implementation actually calls listeners synchronously (it mays)
+        # but hides error raised by them. See Bus::Bunny::Async for another
+        # implementation that is truly asynchronous and relies on RabbitMQ.
+        #
+        class Async
+          include Support::Robustness
+
+          DEFAULT_OPTIONS = {
+          }
+
+          def initialize(options = {})
+            @options = DEFAULT_OPTIONS.merge(options)
+            @listeners = {}
+          end
+
+          def connect
+          end
+
+          def emit(event)
+            (@listeners[event.type.to_s] || []).each do |l|
+              stop_errors(self, "emit", event) {
+                l.call(event)
+              }
+            end
+          end
+
+          def listen(type, processor = nil, listener = nil, &bl)
+            raise ArgumentError, "A listener must be provided" unless listener || bl
+            @listeners[type.to_s] ||= []
+            @listeners[type.to_s] << (listener || bl)
+          end
+
+        end # class Sync
+      end # module Memory
+    end # class Bus
+  end # class Event
+end # module Startback

data/lib/startback/event/bus/memory/sync.rb

@@ -0,0 +1,35 @@
+module Startback
+  class Event
+    class Bus
+      module Memory
+        #
+        # Synchronous implementation of the Bus abstraction, for use between
+        # components sharing the same process.
+        #
+        class Sync
+          include Support::Robustness
+
+          def initialize
+            @listeners = {}
+          end
+
+          def connect
+          end
+
+          def emit(event)
+            (@listeners[event.type.to_s] || []).each do |l|
+              l.call(event)
+            end
+          end
+
+          def listen(type, processor = nil, listener = nil, &bl)
+            raise ArgumentError, "A listener must be provided" unless listener || bl
+            @listeners[type.to_s] ||= []
+            @listeners[type.to_s] << (listener || bl)
+          end
+
+        end # class Sync
+      end # module Memory
+    end # class Bus
+  end # class Event
+end # module Startback

data/lib/startback/event/bus.rb

@@ -0,0 +1,100 @@
+module Startback
+  class Event
+    #
+    # Sync and async bus abstraction allowing to register listeners and
+    # emitting events towards them.
+    #
+    # This bus actually decorates two busses, one in synchronous and the
+    # other one is asynchronous (optional).
+    #
+    # * A synchronous bus MUST call the listeners as part of emitting
+    #   process, and MUST re-raise any error occuring during that process.
+    #   See, e.g. Startback::Bus::Memory::Sync
+    #
+    # * An asynchronous bus MAY call the listeners later, but MUST hide
+    #   errors to the emitter.
+    #   See, e.g. Startback::Bus::Memory::Async
+    #
+    # This bus facade emits events to both sync and async busses (if any),
+    # and listen on the sync one by default.
+    #
+    # For emitters:
+    #
+    #     # This will synchronously call every listeners who `listen`
+    #     # on the synchronous bus (& reraise exceptions) then call
+    #     # (possibly later) all listeners who `listen` on the
+    #     # asynchronous bus if any (& hide exceptions).
+    #     bus.emit(event)
+    #
+    #     # This only reaches sync listeners
+    #     bus.sync.emit(event)
+    #
+    #     # This only reaches async listeners (an async bus must be set)
+    #     bus.async.emit(event)
+    #
+    # Please note that there is currently no way to reach sync listeners
+    # without having to implement error handling on the emitter side.
+    #
+    # For listeners:
+    #
+    #     # This will listen synchronously and make the emitter fail if
+    #     # anything goes wrong with the callback:
+    #     bus.listen(event_type) do |event|
+    #       ...
+    #     end
+    #
+    #     # It is a shortcut for:
+    #     bus.sync.listen(event_type) do |event| ... end
+    #
+    #     This will listen asynchronously and could not make the emitter
+    #     fail if something goes wrong with the callback.
+    #     bus.async.listen(event_type) do |event|
+    #       ...
+    #     end
+    #
+    # Feel free to access the sync and async busses directly for specific
+    # cases though.
+    #
+    class Bus
+      include Support::Robustness
+
+      def initialize(sync = Memory::Sync.new, async = nil)
+        @sync = sync
+        @async = async
+      end
+      attr_reader :sync, :async
+
+      def connect
+        sync.connect if sync
+        async.connect if async
+      end
+
+      # Emits a particular event to the listeners.
+      #
+      # @arg event an event, should be an Event instance (through duck
+      #      typing is allowed)
+      def emit(event)
+        monitor({
+          op: "Startback::Bus#emit",
+          op_data: {
+            event: { type: event.type }
+          }
+        }, event.context) do
+          sync.emit(event)
+          async.emit(event) if async
+        end
+      end
+
+      # Registers `listener` as being interested in receiving events of
+      # a specific type.
+      #
+      # @arg type: Symbol, the type of event the listener is interested in.
+      # @arg listener: Proc, the listener itself.
+      def listen(type, processor = nil, listener = nil, &bl)
+        sync.listen(type, processor, listener, &bl)
+      end
+
+    end # class Bus
+  end # class Event
+end # module Startback
+require_relative 'bus/memory'

data/lib/startback/event/engine.rb

@@ -0,0 +1,171 @@
+require 'rack'
+require 'webrick'
+require 'startback'
+module Startback
+  class Event
+    #
+    # This class is the starting point of event handling in
+    # Startback. It holds a Bus instance to which emitters
+    # and listeners can connect, and the possibility for the
+    # the listening part to start an infinite loop (ServerEngine).
+    #
+    # The Engine automatically runs a Webrick small webapp
+    # with a /healthcheck webservice. The class can be extended
+    # and method `on_health_check` overriden to run specific
+    # checks.
+    #
+    # This class goes hand in hand with the `startback:engine`
+    # docker image. It can be extended by subclasses to override
+    # the following methods:
+    #
+    #   - bus to use something else than a simple memory bus
+    #   - on_health_check to check specific health conditions
+    #   - create_agents to instantiate all listening agents
+    #     (unless auto_create_agents is used)
+    #
+    class Engine
+      include Support::Robustness
+
+      DEFAULT_OPTIONS = {
+
+        # To be passed to ServerEngine
+        server_engine: {}
+
+      }
+
+      def initialize(options = {}, context = Context.new)
+        @options = DEFAULT_OPTIONS.merge(options)
+        @context = context
+      end
+      attr_reader :options, :context
+
+      class << self
+        def auto_create_agents?
+          !!@auto_create_agents
+        end
+
+        # Register a base class which will be used to discover
+        # the agents to start when the engine is ran.
+        def auto_create_agents(base_class = nil)
+          @auto_create_agents ||= base_class
+          @auto_create_agents
+        end
+      end
+
+      # This method is executed on health check and can be
+      # overriden by subclasses to perform specific checks.
+      def on_health_check
+        "Ok"
+      end
+
+      def bus
+        ::Startback::Event::Bus.new
+      end
+
+      def connect
+        log(:info, self, "Connecting to the bus now!")
+        bus.connect
+      end
+
+      def run(options = {})
+        connect
+
+        log(:info, self, "Running agents and server engine!")
+        create_agents
+        Runner.new(self, options[:server_engine] || {}).run
+      end
+
+      def create_agents
+        return unless parent = self.class.auto_create_agents
+
+        ObjectSpace
+          .each_object(Class)
+          .select { |klass| klass < parent }
+          .each { |klass| klass.new(self) }
+      end
+
+      class Runner
+
+        DEFAULT_SERVER_ENGINE_OPTIONS = {
+          daemonize: false,
+          worker_type: 'process',
+          workers: 1
+        }
+
+        def initialize(engine, options = {})
+          raise ArgumentError if engine.nil?
+
+          @engine = engine
+          @options = DEFAULT_SERVER_ENGINE_OPTIONS.merge(options)
+          require 'serverengine'
+        end
+        attr_reader :engine, :options
+
+        def run(options = {})
+          health = self.class.build_health_check(engine)
+          worker = self.class.build_worker(engine, health)
+          se = ServerEngine.create(nil, worker, options)
+          se.run
+          se
+        end
+
+        class << self
+          def run(*args, &bl)
+            new.run(*args, &bl)
+          end
+
+          def build_health_check(engine)
+            Rack::Builder.new do
+              map '/health-check' do
+                health = Startback::Web::HealthCheck.new {
+                  engine.on_health_check
+                }
+                run(health)
+              end
+            end
+          end
+
+          def build_worker(engine, health)
+            Module.new do
+              include Support::Env
+
+              def initialize
+                @stop_flag = ServerEngine::BlockingFlag.new
+              end
+
+              define_method(:health) do
+                health
+              end
+
+              define_method(:engine) do
+                engine
+              end
+
+              def run
+                ran = false
+                until @stop_flag.set?
+                  if ran
+                    engine.send(:log, :warn, engine, "Restarting internal loop")
+                  else
+                    engine.send(:log, :info, engine, "Starting internal loop")
+                  end
+                  Rack::Handler::WEBrick.run(health, {
+                    :Port => env('STARTBACK_ENGINE_PORT', '3000').to_i,
+                    :Host => env('STARTBACK_ENGINE_LISTEN', '0.0.0.0')
+                  })
+                  ran = true
+                end
+              end
+
+              def stop
+                engine.send(:log, :info, engine, "Stopping internal loop")
+                @stop_flag.set!
+                Rack::Handler::WEBrick.shutdown
+              end
+            end
+          end
+        end # class << self
+      end # class Runner
+    end # class Engine
+  end # class Event
+end # module Startback

data/lib/startback/event.rb

@@ -41,3 +41,6 @@ module Startback
 
   end # class Event
 end # module Startback
+require_relative 'event/agent'
+require_relative 'event/bus'
+require_relative 'event/engine'

data/lib/startback/support/env.rb

@@ -0,0 +1,41 @@
+module Startback
+  module Support
+    # This method provides the `env` and `env!` methods that
+    # help querying environment variables easily.
+    module Env
+
+      # Returns an environment variable or raise an error if
+      # not set.
+      #
+      # The result is always a String with no leading/trailing
+      # spaces.
+      #
+      # If a block is given, the environment variable is yield
+      # and the result of the block returned.
+      def env!(key, default = nil, &bl)
+        v = ENV[key].to_s.strip
+        raise Startback::Error, "Missing ENV var `#{key}`" if v.empty?
+
+        env(key, default, &bl)
+      end
+      module_function :env!
+
+      # Returns an environment variable or the default value
+      # passed as second argument.
+      #
+      # The result is always a String with no leading/trailing
+      # spaces.
+      #
+      # If a block is given, the environment variable is yield
+      # and the result of the block returned.
+      def env(key, default = nil, &bl)
+        v = ENV[key].to_s.strip
+        v = v.empty? ? default : v
+        v = bl.call(v) if bl && v
+        v
+      end
+      module_function :env
+
+    end # module Env
+  end # module Support
+end # module Startback

data/lib/startback/support/log_formatter.rb

@@ -3,6 +3,8 @@ module Startback
     class LogFormatter
 
       def call(severity, time, progname, msg)
+        msg = { message: msg } if msg.is_a?(String)
+        msg = { error: msg } if msg.is_a?(Exception)
         {
           severity: severity,
           time: time

data/lib/startback/support/robustness.rb

@@ -43,7 +43,7 @@ module Startback
           @@default_logger ||= begin
             l = ::Logger.new(STDOUT)
             l.formatter = LogFormatter.new
-            l.warn(op: "#{self}", op_data: { msg: "Using default logger", trace: caller })
+            l.warn(op: "#{self}", op_data: { msg: "Using default logger to STDOUT" })
             @@default_logger = l
           end
           @@default_logger
@@ -67,6 +67,8 @@ module Startback
             log_msg.dup
           elsif log_msg.is_a?(String)
             log_msg = { op: "#{log_msg}#{method.nil? ? '' : '#'+method.to_s}" }
+          elsif log_msg.is_a?(Exception)
+            log_msg = { error: log_msg }
           else
             log_msg = log_msg.class unless log_msg.is_a?(Module)
             log_msg = { op: "#{log_msg.name}##{method}" }
@@ -105,8 +107,6 @@ module Startback
         }
         Tools.info(args, op_took: took)
         result
-      rescue => ex
-        raise
       end
 
       # Executes the block without letting errors propagate.