startback 0.4.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08fb9451c77df7efeee3bbbee9d1c540093d083cd5128d088cfd2c52e48169e9'
-  data.tar.gz: ef62fa3c7e42f4ada0c35e9482fea0e81c23e9483c6b66b87586d439e26995e0
+  metadata.gz: 492399cde5cbfe2861575824a32c7788fc16137902d5a4194b3662e5306a5968
+  data.tar.gz: 192d23a34d668bb3ce495c0bb3c724ef58c4e7857bf8e0ed63d5948a76b43f49
 SHA512:
-  metadata.gz: 53604f5706cfdf4f1ae4022f423e0ab9ae9f8582fc8fb92b7379243b16f0afb108209b9509db89e94056c9a4912b2f86827495462746ebf88db1077e656d2528
-  data.tar.gz: 6a56f9fcf818278dd7646105363fb791b31bee1d3ab9afbbd65314eef28f01f34af20545df564846230f45c6add5cd04cf283965656caa4713a86941fba31df4
+  metadata.gz: 54ff69ead9f9b90eccbc9700a46229621cdfe31eb7b887207f6c69c0f755735ae80fc6257ff9ca6e3430d9fc75a2e04fd7b9ea4ecf6744557cb79d418e1ac3b4
+  data.tar.gz: 71de967d2ebee0cd0596b51532d91a1abc1197cd69d1b6db528fcf6bcd00977c484067a83ba266e37e8d807c3c2bba8efc3b67f822b0540eef9fea3c9578fc5b

data/lib/startback/audit/trailer.rb

@@ -0,0 +1,145 @@
+require 'forwardable'
+module Startback
+  module Audit
+    #
+    # Log & Audit trail abstraction, that can be registered as an around
+    # hook on OperationRunner and as an actual logger on Context instances.
+    #
+    # The trail is outputted as JSON lines, using a Logger on the "device"
+    # passed at construction. The following JSON entries are dumped:
+    #
+    # - severity  : INFO or ERROR
+    # - time      : ISO8601 Datetime of operation execution
+    # - op        : class name of the operation executed
+    # - op_took   : Execution duration of the operation
+    # - op_data   : Dump of operation input data
+    # - context   : Execution context, through its `h` information contract (IC)
+    #
+    # Dumping of operation data follows the following duck typing conventions:
+    #
+    # - If the operation instance responds to `to_trail`, this data is taken
+    # - If the operation instance responds to `input`, this data is taken
+    # - If the operation instance responds to `request`, this data is taken
+    # - Otherwise op_data is a JSON null
+    #
+    # By contributing to the Context's `h` IC, users can easily dump information that
+    # makes sense (such as the operation execution requester).
+    #
+    # The class implements a sanitization process when dumping the context and
+    # operation data. Blacklisted words taken in construction options are used to
+    # prevent dumping hash keys that match them (insentively). Default stop words
+    # are equivalent to:
+    #
+    #     Trailer.new("/var/log/trail.log", {
+    #       blacklist: "token password secret credential"
+    #     })
+    #
+    # Please note that the sanitization process does not apply recursively if
+    # the operation data is hierarchic. It only applies to the top object of
+    # Hash and [Hash]. Use `Operation#to_trail` to fine-tune your audit trail.
+    #
+    # Given that this Trailer is intended to be used as around hook on an
+    # `OperationRunner`, operations that fail at construction time will not be
+    # trailed at all, since they can't be ran in the first place. This may lead
+    # to trails not containing important errors cases if operations check their
+    # input at construction time.
+    #
+    class Trailer
+      extend Forwardable
+      def_delegators :@logger, :debug, :info, :warn, :error, :fatal
+
+      class Formatter
+
+        def call(severity, time, progname, msg)
+          if msg[:error] && msg[:error].respond_to?(:message, true)
+            msg[:error] = msg[:error].message
+          end
+          {
+            severity: severity,
+            time: time,
+          }.merge(msg).to_json << "\n"
+        end
+
+      end # class Formatter
+
+      DEFAULT_OPTIONS = {
+
+        # Words used to stop dumping for, e.g., security reasons
+        blacklist: "token password secret credential"
+
+      }
+
+      def initialize(device, options = {})
+        @options = DEFAULT_OPTIONS.merge(options)
+        @logger = ::Logger.new(device, 'daily')
+        @logger.formatter = Formatter.new
+      end
+      attr_reader :logger, :options
+
+      def call(runner, op)
+        result = nil
+        time = Benchmark.realtime{ result = yield }
+        logger.info(op_to_trail(op, time))
+        result
+      rescue => ex
+        logger.error(op_to_trail(op, time, ex))
+        raise
+      end
+
+    protected
+
+      def op_to_trail(op, time, ex = nil)
+        log_msg = {
+          op_took: time ? time.round(8) : nil,
+          op: op_name(op),
+          context: op_context(op),
+          op_data: op_data(op)
+        }
+        log_msg[:error] = ex if ex
+        log_msg
+      end
+
+      def op_name(op)
+        case op
+        when String then op
+        when Class  then op.name
+        else op.class.name
+        end
+      end
+
+      def op_context(op)
+        sanitize(op.respond_to?(:context, false) ? op.context.to_h : {})
+      end
+
+      def op_data(op)
+        data = if op.respond_to?(:to_trail, false)
+          op.to_trail
+        elsif op.respond_to?(:input, false)
+          op.input
+        elsif op.respond_to?(:request, false)
+          op.request
+        end
+        sanitize(data)
+      end
+
+      def sanitize(data)
+        case data
+        when Hash, OpenStruct
+          data.dup.delete_if{|k| k.to_s =~ blacklist_rx }
+        when Enumerable
+          data.map{|elm| sanitize(elm) }.compact
+        else
+          data
+        end
+      end
+
+      def blacklist_rx
+        @blacklist_rx ||= Regexp.new(
+          options[:blacklist].split(/\s+/).join("|"),
+          Regexp::IGNORECASE
+        )
+      end
+
+    end # class Trailer
+  end # module Audit
+end # module Startback

data/lib/startback/audit.rb

@@ -0,0 +1 @@
+require_relative 'audit/trailer'

data/lib/startback/bus/bunny/async.rb

@@ -0,0 +1,117 @@
+require 'bunny'
+module Startback
+  class Bus
+    module Bunny
+      #
+      # Asynchronous implementation of the bus abstraction, on top of RabbitMQ
+      # and using the 'bunny' gem (you need to include it in your Gemfile
+      # yourself: it is NOT a startback official dependency).
+      #
+      # This bus implementation emits events by dumping them to RabbitMQ using
+      # the event type as exchange name. Listeners may use the `processor`
+      # parameter to specify the queue name ; otherwise a default "main" queue
+      # is used.
+      #
+      # Examples:
+      #
+      #     # Connects to RabbitMQ using all default options
+      #     #
+      #     # Uses the STARTBACK_BUS_BUNNY_ASYNC_URL environment variable for
+      #     # connection URL if present.
+      #     Startback::Bus::Bunny::Async.new
+      #
+      #     # Connects to RabbitMQ using a specific URL
+      #     Startback::Bus::Bunny::Async.new("amqp://rabbituser:rabbitpass@192.168.17.17")
+      #     Startback::Bus::Bunny::Async.new(url: "amqp://rabbituser:rabbitpass@192.168.17.17")
+      #
+      #     # Connects to RabbitMQ using specific connection options. See Bunny's own
+      #     # documentation
+      #     Startback::Bus::Bunny::Async.new({
+      #       connection_options: {
+      #         host: "192.168.17.17"
+      #       }
+      #     })
+      #
+      class Async
+        include Support::Robustness
+
+        DEFAULT_OPTIONS = {
+          # (optional) The URL to use for connecting to RabbitMQ.
+          url: ENV['STARTBACK_BUS_BUNNY_ASYNC_URL'],
+
+          # (optional) The options has to pass to ::Bunny constructor
+          connection_options: nil,
+
+          # (optional) The options to use for the emitter/listener fanout
+          fanout_options: {},
+
+          # (optional) The options to use for the listener queue
+          queue_options: {},
+
+          # (optional) Default event factory to use, if any
+          event_factory: nil,
+
+          # (optional) A default context to use for general logging
+          context: nil
+        }
+
+        # Creates a bus instance, using the various options provided to
+        # fine-tune behavior.
+        def initialize(options = {})
+          options = { url: options } if options.is_a?(String)
+          @options = DEFAULT_OPTIONS.merge(options)
+          retried = 0
+          conn = options[:connection_options] || options[:url]
+          try_max_times(10) do
+            @bunny = ::Bunny.new(conn)
+            @bunny.start
+            @channel = @bunny.create_channel
+            log(:info, {op: "#{self.class.name}#connect", op_data: conn}, options[:context])
+          end
+        end
+        attr_reader :channel, :options
+
+        def emit(event)
+          stop_errors(self, "emit", event.context) do
+            fanout = channel.fanout(event.type.to_s, fanout_options)
+            fanout.publish(event.to_json)
+          end
+        end
+
+        def listen(type, processor = nil, listener = nil, &bl)
+          raise ArgumentError, "A listener must be provided" unless listener || bl
+          fanout = channel.fanout(type.to_s, fanout_options)
+          queue = channel.queue((processor || "main").to_s, queue_options)
+          queue.bind(fanout)
+          queue.subscribe do |delivery_info, properties, body|
+            event = stop_errors(self, "listen") do
+              factor_event(body)
+            end
+            stop_errors(self, "listen", event.context) do
+              (listener || bl).call(event)
+            end
+          end
+        end
+
+      protected
+
+        def fanout_options
+          options[:fanout_options]
+        end
+
+        def queue_options
+          options[:queue_options]
+        end
+
+        def factor_event(body)
+          if options[:event_factory]
+            options[:event_factory].call(body)
+          else
+            Event.json(body, options)
+          end
+        end
+
+      end # class Async
+    end # module Bunny
+  end # class Bus
+end # module Klaro

data/lib/startback/bus/bunny.rb

@@ -0,0 +1 @@
+require_relative 'bunny/async'

data/lib/startback/bus/memory/async.rb

@@ -0,0 +1,40 @@
+module Startback
+  class Bus
+    module Memory
+      #
+      # Asynchronous implementation of the Bus abstraction, for use between
+      # components sharing the same process.
+      #
+      # This implementation actually calls listeners synchronously (it mays)
+      # but hides error raised by them. See Bus::Bunny::Async for another
+      # implementation that is truly asynchronous and relies on RabbitMQ.
+      #
+      class Async
+        include Support::Robustness
+
+        DEFAULT_OPTIONS = {
+        }
+
+        def initialize(options = {})
+          @options = DEFAULT_OPTIONS.merge(options)
+          @listeners = {}
+        end
+
+        def emit(event)
+          (@listeners[event.type.to_s] || []).each do |l|
+            stop_errors(self, "emit", event) {
+              l.call(event)
+            }
+          end
+        end
+
+        def listen(type, processor = nil, listener = nil, &bl)
+          raise ArgumentError, "A listener must be provided" unless listener || bl
+          @listeners[type.to_s] ||= []
+          @listeners[type.to_s] << (listener || bl)
+        end
+
+      end # class Sync
+    end # module Memory
+  end # class Bus
+end # module Klaro

data/lib/startback/bus/memory/sync.rb

@@ -0,0 +1,30 @@
+module Startback
+  class Bus
+    module Memory
+      #
+      # Synchronous implementation of the Bus abstraction, for use between
+      # components sharing the same process.
+      #
+      class Sync
+        include Support::Robustness
+
+        def initialize
+          @listeners = {}
+        end
+
+        def emit(event)
+          (@listeners[event.type.to_s] || []).each do |l|
+            l.call(event)
+          end
+        end
+
+        def listen(type, processor = nil, listener = nil, &bl)
+          raise ArgumentError, "A listener must be provided" unless listener || bl
+          @listeners[type.to_s] ||= []
+          @listeners[type.to_s] << (listener || bl)
+        end
+
+      end # class Sync
+    end # module Memory
+  end # class Bus
+end # module Klaro

data/lib/startback/bus/memory.rb

@@ -0,0 +1,2 @@
+require_relative 'memory/sync'
+require_relative 'memory/async'

data/lib/startback/bus.rb

@@ -0,0 +1,94 @@
+require 'startback/event'
+module Startback
+  #
+  # Sync and async bus abstraction allowing to register listeners and
+  # emitting events towards them.
+  #
+  # This bus actually decorates two busses, one in synchronous and the
+  # other one is asynchronous (optional).
+  #
+  # * A synchronous bus MUST call the listeners as part of emitting
+  #   process, and MUST re-raise any error occuring during that process.
+  #   See, e.g. Startback::Bus::Memory::Sync
+  #  
+  # * An asynchronous bus MAY call the listeners later, but MUST hide
+  #   errors to the emitter.
+  #   See, e.g. Startback::Bus::Memory::Async
+  #
+  # This bus facade emits events to both sync and async busses (if any),
+  # and listen on the sync one by default.
+  #
+  # For emitters:
+  #
+  #     # This will synchronously call every listeners who `listen`
+  #     # on the synchronous bus (& reraise exceptions) then call
+  #     # (possibly later) all listeners who `listen` on the
+  #     # asynchronous bus if any (& hide exceptions).
+  #     bus.emit(event)
+  #
+  #     # This only reaches sync listeners
+  #     bus.sync.emit(event) 
+  #
+  #     # This only reaches async listeners (an async bus must be set)
+  #     bus.async.emit(event)
+  #
+  # Please note that there is currently no way to reach sync listeners
+  # without having to implement error handling on the emitter side. 
+  #
+  # For listeners:
+  #
+  #     # This will listen synchronously and make the emitter fail if
+  #     # anything goes wrong with the callback:
+  #     bus.listen(event_type) do |event|
+  #       ...
+  #     end
+  #
+  #     # It is a shortcut for:
+  #     bus.sync.listen(event_type) do |event| ... end 
+  #
+  #     This will listen asynchronously and could not make the emitter
+  #     fail if something goes wrong with the callback.
+  #     bus.async.listen(event_type) do |event|
+  #       ...
+  #     end
+  #
+  # Feel free to access the sync and async busses directly for specific
+  # cases though.
+  #
+  class Bus
+    include Support::Robustness
+
+    def initialize(sync = Memory::Sync.new, async = nil)
+      @sync = sync
+      @async = async
+    end
+    attr_reader :sync, :async
+
+    # Emits a particular event to the listeners.
+    #
+    # @arg event an event, should be an Event instance (through duck
+    #      typing is allowed)
+    def emit(event)
+      monitor({
+        op: "Startback::Bus#emit",
+        op_data: {
+          event: { type: event.type }
+        }
+      }, event.context) do
+        sync.emit(event)
+        async.emit(event) if async
+      end
+    end
+
+    # Registers `listener` as being interested in receiving events of
+    # a specific type.
+    #
+    # @arg type: Symbol, the type of event the listener is interested in.
+    # @arg listener: Proc, the listener itself. 
+    def listen(type, processor = nil, listener = nil, &bl)
+      sync.listen(type, processor, listener, &bl)
+    end
+
+  end # class Bus
+end # module Klaro
+require_relative 'bus/memory'

data/lib/startback/caching/entity_cache.rb

@@ -0,0 +1,80 @@
+module Startback
+  module Caching
+    #
+    # A overriable caching abstraction aiming at making Entity-based caching easy.
+    #
+    # This class MUST be overriden:
+    #
+    # * the `load_raw_data` protected method MUST be implemented.
+    # * the `full_key` protected method MAY be overriden to provide specific caching
+    #   keys, e.g. by using the context.
+    # * the `valid?` protected method MAY be overriden to check validity of data
+    #   extracted from the cache.
+    #
+    # An EntityCache takes an actual store at construction. The object must meet the
+    # specification writtern in Store. The 'cache' ruby gem can be used in practice.
+    #
+    class EntityCache
+
+      class << self
+
+        # Default time to live, in seconds
+        attr_writer :default_ttl
+
+        def default_ttl
+          @default_ttl || (superclass.respond_to?(:default_ttl, true) && superclass.default_ttl) || 3600
+        end
+
+      end # class DSL
+
+      def initialize(store, context = nil)
+        @store = store
+        @context = context
+      end
+      attr_reader :store, :context
+
+      # Returns the entity corresponding to a given key.
+      #
+      # If the entity is not in cache, loads it and puts it in cache using
+      # the caching options passed as second parameter.
+      def get(short_key, caching_options = default_caching_options)
+        cache_key = encode_key(full_key(short_key))
+        if store.exist?(cache_key)
+          cached = store.get(cache_key)
+          return cached if valid?(cache_key, cached)
+        end
+        load_raw_data(short_key).tap{|to_cache|
+          store.set(cache_key, to_cache, caching_options)
+        }
+      end
+
+      # Invalidates the cache under a given key.
+      def invalidate(key)
+        store.delete(encode_key(full_key(key)))
+      end
+
+    protected
+
+      def encode_key(key)
+        JSON.fast_generate(key)
+      end
+
+      def default_caching_options
+        { ttl: self.class.default_ttl }
+      end
+
+      def valid?(cache_key, cached)
+        true
+      end
+
+      def full_key(key)
+        key
+      end
+
+      def load_raw_data(short_key)
+        raise NotImplementedError, "#{self.class.name}#load_raw_data"
+      end
+
+    end # class EntityCache
+  end # module Caching
+end # module Startback

data/lib/startback/caching/store.rb

@@ -0,0 +1,34 @@
+module Startback
+  module Caching
+    #
+    # Caching store specification & dummy implementation.
+    #
+    # This class should not be used in real project, as it implements
+    # See the 'cache' gem that provides conforming implementations.
+    #
+    class Store
+
+      def initialize
+        @saved = {}
+      end
+      attr_reader :saved
+
+      def exist?(key)
+        saved.has_key?(key)
+      end
+
+      def get(key)
+        saved[key]
+      end
+
+      def set(key, value, ttl)
+        saved[key] = value
+      end
+
+      def delete(key)
+        saved.delete(key)
+      end
+
+    end # class Store
+  end # module Caching
+end # module Startback

data/lib/startback/context/middleware.rb

@@ -42,7 +42,7 @@ module Startback
       attr_reader :options
 
       def call(env)
-        env[RACK_ENV_KEY] ||= options[:context_class].new.tap{|c|
+        env[RACK_ENV_KEY] ||= options[:context_class].h({}).tap{|c|
           c.original_rack_env = env.dup
         }
         @app.call(env)

data/lib/startback/context.rb

@@ -1,13 +1,35 @@
 module Startback
   #
-  # Defines an execution context for Startback applications.
-  #
-  # This class is aimed at being subclassed for application required
-  # extension.
+  # Defines an execution context for Startback applications, and provides
+  # a cached factory for related abstractions (see `factor`).
   #
   # In web application, an instance of a context can be set on the Rack
   # environment, using Context::Middleware.
   #
+  # This class SHOULD be subclassed for application required extensions
+  # to prevent touching the global Startback state itself.
+  #
+  # Also, for event handling in distributed architectures, a Context should
+  # be dumpable and reloadable to JSON. An `h` information contract if provided
+  # for that. Subclasses may contribute to the dumping and reloading process
+  # through the `h_dump` and `h_factory` methods
+  #
+  #     module MyApp
+  #       class Context < Startback::Context
+  #
+  #         attr_accessor :foo
+  #
+  #         h_dump do |h|
+  #           h.merge!("foo" => foo)
+  #         end
+  #
+  #         h_factor do |c,h|
+  #           c.foo = h["foo"]
+  #         end
+  #
+  #       end
+  #     end
+  #
   class Context
     attr_accessor :original_rack_env
 
@@ -16,8 +38,75 @@ module Startback
     # instance, simply.
     #
     # Fatal errors catched by Web::CatchAll are sent on `error_handler#fatal`
+    #
+    # Deprecated, use the logger below instead.
     attr_accessor :error_handler
 
+    # A logger can be provided on the context, and will be used for everything
+    # related to logging, audit trailing and robustness. The logger receives
+    # object following the log & trail conventions of Startback, and must
+    # convert them to wathever log format is necessary.
+    attr_accessor :logger
+
+    # Implementation of the `h` information contract
+    class << self
+
+      def h(hash)
+        h_factor!(self.new, hash)
+      end
+
+      def h_factor!(context, hash)
+        h_factories.each do |f|
+          f.call(context, hash)
+        end
+        context
+      end
+
+      def h_factories
+        @h_factories ||= []
+      end
+
+      def h_factory(&factory)
+        h_factories << factory
+      end
+
+      ###
+
+      def h_dump!(context, hash = {})
+        h_dumpers.each do |d|
+          context.instance_exec(hash, &d)
+        end
+        hash
+      end
+
+      def h_dumpers
+        @h_dumpers ||= []
+      end
+
+      def h_dump(&dumper)
+        h_dumpers << dumper
+      end
+
+    end # class << self
+
+    # Factors an instance of `clazz`, which must be a Context-related
+    # abstraction (i.e. its constructor takes the context as last parameters).
+    #
+    # Factored abstractions are cached for a given context & arguments.
+    def factor(clazz, *args)
+      @factored ||= {}
+      key = args.empty? ? clazz : [clazz] + args
+      @factored[key] ||= clazz.new(*(args << self))
+    end
+
+    def to_h
+      self.class.h_dump!(self)
+    end
+
+    def to_json(*args, &bl)
+      to_h.to_json(*args, &bl)
+    end
+
   end # class Context
 end # module Startback
 require_relative 'context/middleware'