startback-jobs 0.13.0 → 0.14.0

data/lib/startback/event/bus.rb

@@ -0,0 +1,100 @@
+module Startback
+  class Event
+    #
+    # Sync and async bus abstraction allowing to register listeners and
+    # emitting events towards them.
+    #
+    # This bus actually decorates two busses, one in synchronous and the
+    # other one is asynchronous (optional).
+    #
+    # * A synchronous bus MUST call the listeners as part of emitting
+    #   process, and MUST re-raise any error occuring during that process.
+    #   See, e.g. Startback::Bus::Memory::Sync
+    #
+    # * An asynchronous bus MAY call the listeners later, but MUST hide
+    #   errors to the emitter.
+    #   See, e.g. Startback::Bus::Memory::Async
+    #
+    # This bus facade emits events to both sync and async busses (if any),
+    # and listen on the sync one by default.
+    #
+    # For emitters:
+    #
+    #     # This will synchronously call every listeners who `listen`
+    #     # on the synchronous bus (& reraise exceptions) then call
+    #     # (possibly later) all listeners who `listen` on the
+    #     # asynchronous bus if any (& hide exceptions).
+    #     bus.emit(event)
+    #
+    #     # This only reaches sync listeners
+    #     bus.sync.emit(event)
+    #
+    #     # This only reaches async listeners (an async bus must be set)
+    #     bus.async.emit(event)
+    #
+    # Please note that there is currently no way to reach sync listeners
+    # without having to implement error handling on the emitter side.
+    #
+    # For listeners:
+    #
+    #     # This will listen synchronously and make the emitter fail if
+    #     # anything goes wrong with the callback:
+    #     bus.listen(event_type) do |event|
+    #       ...
+    #     end
+    #
+    #     # It is a shortcut for:
+    #     bus.sync.listen(event_type) do |event| ... end
+    #
+    #     This will listen asynchronously and could not make the emitter
+    #     fail if something goes wrong with the callback.
+    #     bus.async.listen(event_type) do |event|
+    #       ...
+    #     end
+    #
+    # Feel free to access the sync and async busses directly for specific
+    # cases though.
+    #
+    class Bus
+      include Support::Robustness
+
+      def initialize(sync = Memory::Sync.new, async = nil)
+        @sync = sync
+        @async = async
+      end
+      attr_reader :sync, :async
+
+      def connect
+        sync.connect if sync
+        async.connect if async
+      end
+
+      # Emits a particular event to the listeners.
+      #
+      # @arg event an event, should be an Event instance (through duck
+      #      typing is allowed)
+      def emit(event)
+        monitor({
+          op: "Startback::Bus#emit",
+          op_data: {
+            event: { type: event.type }
+          }
+        }, event.context) do
+          sync.emit(event)
+          async.emit(event) if async
+        end
+      end
+
+      # Registers `listener` as being interested in receiving events of
+      # a specific type.
+      #
+      # @arg type: Symbol, the type of event the listener is interested in.
+      # @arg listener: Proc, the listener itself.
+      def listen(type, processor = nil, listener = nil, &bl)
+        sync.listen(type, processor, listener, &bl)
+      end
+
+    end # class Bus
+  end # class Event
+end # module Startback
+require_relative 'bus/memory'

data/lib/startback/event/engine.rb

@@ -0,0 +1,94 @@
+require 'rack'
+require 'startback'
+module Startback
+  class Event
+    #
+    # This class is the starting point of event handling in
+    # Startback. It holds a Bus instance to which emitters
+    # and listeners can connect.
+    #
+    # The Engine exposes a rack app (.rack_app) with a /healthcheck webservice.
+    # It is supposed to be mounted to a webserver such as puma.
+    #
+    # This class goes hand in hand with the `startback:engine`
+    # docker image. It can be extended by subclasses to override
+    # the following methods:
+    #
+    #   - bus to use something else than a simple memory bus
+    #   - on_health_check to check specific health conditions
+    #   - create_agents to instantiate all listening agents
+    #     (unless auto_create_agents is used)
+    #   - rack_app if you want to customize the API running
+    #
+    class Engine
+      include Support::Robustness
+
+      DEFAULT_OPTIONS = {
+
+      }
+
+      def initialize(options = {}, context = Context.new)
+        @options = DEFAULT_OPTIONS.merge(options)
+        @context = context
+        @context.engine = self
+      end
+      attr_reader :options, :context
+
+      class << self
+        def auto_create_agents?
+          !!@auto_create_agents
+        end
+
+        # Register a base class which will be used to discover
+        # the agents to start when the engine is ran.
+        def auto_create_agents(base_class = nil)
+          @auto_create_agents ||= base_class
+          @auto_create_agents
+        end
+      end
+
+      # This method is executed on health check and can be
+      # overriden by subclasses to perform specific checks.
+      def on_health_check
+        "Ok"
+      end
+
+      def bus
+        @bus ||= ::Startback::Event::Bus.new
+      end
+
+      def connect
+        log(:info, self, "Connecting to the bus now!")
+        bus.connect
+      end
+
+      def create_agents
+        return unless parent = self.class.auto_create_agents
+
+        ObjectSpace
+          .each_object(Class)
+          .select { |klass| klass <= parent }
+          .each { |klass| klass.new(self) }
+      end
+
+      def factor_event(event_data)
+        Event.json(event_data, context)
+      end
+
+      def rack_app
+        engine = self
+        Rack::Builder.new do
+          use Startback::Web::CatchAll
+
+          map '/health-check' do
+            health = Startback::Web::HealthCheck.new {
+              engine.on_health_check
+            }
+            run(health)
+          end
+        end
+      end
+
+    end # class Engine
+  end # class Event
+end # module Startback

data/lib/startback/event/ext/context.rb

@@ -0,0 +1,5 @@
+module Startback
+  class Context
+    attr_accessor :engine
+  end # class Context
+end # module Startback

data/lib/startback/event/ext/operation.rb

@@ -0,0 +1,13 @@
+module Startback
+  class Operation
+
+    def self.emits(type, &bl)
+      after_call do
+        event_data = instance_exec(&bl)
+        event = type.new(type.to_s, event_data, context)
+        context.engine.bus.emit(event)
+      end
+    end
+
+  end # class Operation
+end # module Startback

data/lib/startback/event.rb

@@ -0,0 +1,47 @@
+module Startback
+  #
+  # An Event occuring a given context and having a type and attached data.
+  #
+  # Event instances have String types that are by default unrelated to ruby
+  # classes. Also, this Event class has a `json` information contract that
+  # allows dumping & reloading them easily. A context or context_factory may
+  # be provided in dress world to reload the event context from data, but
+  # that logic is opaque to this class.
+  #
+  # This class is intended to be subclassed if a more specific event protocol
+  # is wanted.
+  #
+  class Event
+
+    def initialize(type, data, context = nil)
+      @type = type.to_s
+      @data = OpenStruct.new(data)
+      @context = context
+    end
+    attr_reader :context, :type, :data
+
+    def self.json(src, context)
+      return src if src.is_a?(Event)
+
+      parsed = JSON.parse(src)
+      klass = Kernel.const_get(parsed['type'])
+      context = context.fork(parsed['context']) if context
+      klass.new(parsed['type'], parsed['data'], context)
+    end
+
+    def to_json(*args, &bl)
+      h = {
+        type: self.type,
+        data: data.to_h
+      }
+      h[:context] = context if context
+      h.to_json(*args, &bl)
+    end
+
+  end # class Event
+end # module Startback
+require_relative 'event/ext/context'
+require_relative 'event/ext/operation'
+require_relative 'event/agent'
+require_relative 'event/bus'
+require_relative 'event/engine'

data/lib/startback/ext/date_time.rb

@@ -0,0 +1,9 @@
+class DateTime
+
+  # Makes sure that DateTime are exported with ISO8601
+  # conventions when using to_json and to_csv
+  def to_s(*args)
+    iso8601
+  end
+
+end

data/lib/startback/ext/time.rb

@@ -0,0 +1,9 @@
+class Time
+
+  # Makes sure that Time are exported with ISO8601
+  # conventions when using to_json and to_csv
+  def to_s(*args)
+    iso8601
+  end
+
+end

data/lib/startback/ext.rb

@@ -1,2 +1,2 @@
-require_relative 'ext/web/api'
-require_relative 'ext/support/operation_runner'
+require_relative 'ext/date_time'
+require_relative 'ext/time'

data/lib/startback/model.rb

@@ -0,0 +1,6 @@
+module Startback
+  class Model
+    include Support::DataObject
+
+  end # class Model
+end # module Startback

data/lib/startback/operation/error_operation.rb

@@ -0,0 +1,19 @@
+module Startback
+  class Operation
+    class ErrorOperation < Operation
+
+      def initialize(details)
+        @details = details
+      end
+      attr_reader :details
+
+      def call
+      end
+
+      def bind(world)
+        self
+      end
+
+    end # class ErrorOperation
+  end # class Operation
+end # module Startback

data/lib/startback/operation/multi_operation.rb

@@ -0,0 +1,28 @@
+module Startback
+  class Operation
+    class MultiOperation < Operation
+
+      def initialize(ops = [])
+        @ops = ops
+      end
+      attr_reader :ops
+
+      def size
+        ops.size
+      end
+
+      def +(other)
+        MultiOperation.new(@ops + Array(other))
+      end
+
+      def bind(world)
+        MultiOperation.new(ops.map{|op| op.bind(world) })
+      end
+
+      def call
+        ops.map{|op| op.call }
+      end
+
+    end # class MultiOperation
+  end # class Operation
+end # module Startback

data/lib/startback/operation.rb

@@ -0,0 +1,78 @@
+module Startback
+  #
+  # High-level Operation abstraction, that is a piece of code that executes
+  # on demand and (generally) changes the state of the software system.
+  #
+  # An operation is basically an object that respond to `call`, but that
+  # executes within a given world (see `bind`). It also has before and
+  # after hooks that allows specifying what needs to be done before invoking
+  # call and after having invoked it. All this protocol is actually under
+  # the responsibility of an `OperationRunner`. Operations should not be
+  # called manually by third-party code.
+  #
+  # Example:
+  #
+  #     class SayHello < Startback::Operation
+  #
+  #       before_call do
+  #         # e.g. check_some_permissions
+  #       end
+  #
+  #       def call
+  #         puts "Hello"
+  #       end
+  #
+  #       after_call do
+  #         # e.g. log and/or emit something on a bus
+  #       end
+  #
+  #     end
+  #
+  class Operation
+    extend Support::TransactionPolicy
+    include Errors
+    include Support::OperationRunner
+    include Support::Hooks.new(:call)
+
+    attr_accessor :world
+    protected :world=
+
+    def initialize(input = {})
+      @input = input
+    end
+    attr_reader :input
+
+    def bind(world)
+      return self unless world
+      self.world = world
+      self
+    end
+
+    def method_missing(name, *args, &bl)
+      return super unless args.empty? and bl.nil?
+      return super unless world
+      world.fetch(name){ super }
+    end
+
+    def respond_to?(name, *args)
+      super || (world && world.has_key?(name))
+    end
+
+    def with_context(ctx = nil)
+      old_world = self.world
+      self.world = self.world.merge(context: ctx || old_world.context.dup)
+      result = ctx ? yield : yield(self.world.context)
+      self.world = old_world
+      result
+    end
+
+  protected
+
+    def operation_world(op)
+      self.world
+    end
+
+  end # class Operation
+end # module Startback
+require_relative 'operation/error_operation'
+require_relative 'operation/multi_operation'

data/lib/startback/services.rb

@@ -0,0 +1,11 @@
+module Startback
+  class Services
+    include Startback::Errors
+
+    def initialize(context)
+      @context = context
+    end
+    attr_reader :context
+
+  end # class Services
+end # module Startback

data/lib/startback/support/data_object.rb

@@ -0,0 +1,71 @@
+module Startback
+  module Support
+    module DataObject
+
+      def initialize(data = {})
+        @_data = data.dup.freeze
+      end
+
+      attr_writer :_data
+      protected :_data=
+
+      def method_missing(name, *args, &bl)
+        return super unless args.empty? && bl.nil?
+        return super unless pair = _data_key_for(name)
+
+        pair.last ? !!@_data[pair.first] : @_data[pair.first]
+      end
+
+      def [](name)
+        return nil unless pair = _data_key_for(name, false, false)
+
+        @_data[pair.first]
+      end
+
+      def respond_to?(name)
+        super || !_data_key_for(name).nil?
+      end
+
+      def to_data
+        @_data
+      end
+      alias :to_h :to_data
+
+      def to_json(*args, &bl)
+        to_data.to_json(*args, &bl)
+      end
+
+    private
+
+      def _data_key_for(key, try_camelize = _data_allow_camelize, try_query = _data_allow_query)
+        if @_data.key?(key)
+          [key, false]
+        elsif @_data.key?(key.to_s)
+          [key.to_s, false]
+        elsif key.is_a?(String) && @_data.key?(key.to_sym)
+          [key.to_sym, false]
+        elsif try_camelize
+          cam = key.to_s.gsub(/_([a-z])/){ $1.upcase }.to_sym
+          _data_key_for(cam, false, true)
+        elsif try_query && key.to_s =~ /\?$/
+          got = _data_key_for(key[0...-1].to_sym, false, false)
+          got ? [got.first, true] : _data_key_not_found(key)
+        else
+          _data_key_not_found(key)
+        end
+      end
+
+      def _data_allow_camelize
+        true
+      end
+
+      def _data_allow_query
+        true
+      end
+
+      def _data_key_not_found(key)
+        nil
+      end
+    end # module DataObject
+  end # module Support
+end # module Startback

data/lib/startback/support/env.rb

@@ -0,0 +1,41 @@
+module Startback
+  module Support
+    # This method provides the `env` and `env!` methods that
+    # help querying environment variables easily.
+    module Env
+
+      # Returns an environment variable or raise an error if
+      # not set.
+      #
+      # The result is always a String with no leading/trailing
+      # spaces.
+      #
+      # If a block is given, the environment variable is yield
+      # and the result of the block returned.
+      def env!(key, default = nil, &bl)
+        v = ENV[key].to_s.strip
+        raise Startback::Error, "Missing ENV var `#{key}`" if v.empty?
+
+        env(key, default, &bl)
+      end
+      module_function :env!
+
+      # Returns an environment variable or the default value
+      # passed as second argument.
+      #
+      # The result is always a String with no leading/trailing
+      # spaces.
+      #
+      # If a block is given, the environment variable is yield
+      # and the result of the block returned.
+      def env(key, default = nil, &bl)
+        v = ENV[key].to_s.strip
+        v = v.empty? ? default : v
+        v = bl.call(v) if bl && v
+        v
+      end
+      module_function :env
+
+    end # module Env
+  end # module Support
+end # module Startback

data/lib/startback/support/fake_logger.rb

@@ -0,0 +1,18 @@
+module Startback
+  module Support
+    class FakeLogger < Logger
+
+      def initialize(*args)
+        @last_msg = nil
+      end
+      attr_reader :last_msg
+
+      [:debug, :info, :warn, :error, :fatal].each do |meth|
+        define_method(meth) do |msg|
+          @last_msg = msg
+        end        
+      end
+
+    end # class Logger
+  end # module Support
+end # module Startback

data/lib/startback/support/hooks.rb

@@ -0,0 +1,48 @@
+module Startback
+  module Support
+    class Hooks < Module
+
+      def initialize(suffix)
+        @suffix = suffix
+        define_method :"before_#{suffix}" do
+          self.class.__befores.each do |bl|
+            instance_exec(&bl)
+          end
+        end
+        define_method :"after_#{suffix}" do
+          self.class.__afters.each do |bl|
+            instance_exec(&bl)
+          end
+        end
+      end
+      attr_reader :suffix
+
+      def included(by)
+        by.instance_eval %Q{
+          def __befores(create = false)
+            if create
+              @__befores ||= (superclass.respond_to?(:__befores, false) ? superclass.__befores.dup : [])
+            end
+            @__befores || (superclass.respond_to?(:__befores, false) ? superclass.__befores : [])
+          end
+
+          def __afters(create = false)
+            if create
+              @__afters ||= (superclass.respond_to?(:__afters, false) ? superclass.__afters.dup : [])
+            end
+            @__afters || (superclass.respond_to?(:__afters, false) ? superclass.__afters : [])
+          end
+
+          def before_#{suffix}(&bl)
+            __befores(true) << bl
+          end
+
+          def after_#{suffix}(&bl)
+            __afters(true) << bl
+          end
+        }
+      end
+
+    end # class Hooks
+  end # module Support
+end # module Startback

data/lib/startback/support/log_formatter.rb

@@ -0,0 +1,34 @@
+module Startback
+  module Support
+    class LogFormatter
+
+      def call(severity, time, progname, msg)
+        msg = { message: msg } if msg.is_a?(String)
+        msg = { error: msg } if msg.is_a?(Exception)
+        {
+          severity: severity,
+          time: time
+        }.merge(msg)
+         .merge(error: error_to_json(msg[:error], severity))
+         .compact
+         .to_json << "\n"
+      end
+
+      def error_to_json(error, severity = nil)
+        return error if error.nil?
+        return error if error.is_a?(String)
+        return error.to_s unless error.is_a?(Exception)
+
+        backtrace = error.backtrace[0..25] if severity == "FATAL"
+        causes = error.causes.map{|c| error_to_json(c) } if error.respond_to?(:causes)
+        causes = nil if causes && causes.empty?
+        {
+          message: error.message,
+          backtrace: backtrace,
+          causes: causes
+        }.compact
+      end
+
+    end # class LogFormatter
+  end # module Support
+end # module Startback

data/lib/startback/support/logger.rb

@@ -0,0 +1,34 @@
+module Startback
+  module Support
+    #
+    # A Logger extension that sends info and debug messages to STDOUT
+    # and other messages to STDERR. This is not configurable.
+    #
+    class Logger < ::Logger
+
+      def initialize
+        super(STDOUT)
+        @err_logger = ::Logger.new(STDERR)
+      end
+
+      def self.level=(level)
+        super.tap{
+          @err_logger.level = level
+        }
+      end
+
+      def warn(*args, &bl)
+        @err_logger.warn(*args, &bl)
+      end
+
+      def error(*args, &bl)
+        @err_logger.error(*args, &bl)
+      end
+
+      def fatal(*args, &bl)
+        @err_logger.fatal(*args, &bl)
+      end
+
+    end # class Logger
+  end # module Support
+end # module Startback