startback-jobs 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 485b5ef6145820975b98f3f457c276c7b3fd824a0b63752e4688110c9935b642
-  data.tar.gz: 5cbadc773d07ea6108488e86a58d3cc9ce1e919fa23359ac87f340e94235610c
+  metadata.gz: b9ab7d38cf784d7fa959bf9ca1e9b4014034766165b3e54eb3ce4b03aadfecba
+  data.tar.gz: 8234e5f95a05e91f6698da787bc8479ce924a2bfc537866ea68d188f89df7b54
 SHA512:
-  metadata.gz: dddbd22a80647d96483a2e674a0e306e7414fb7cb445474571db569e8aff89e88178dbab5e5de5b29eaf4ce925f54b22df988d88d9e41c5e1fff96fe0a7cb796
-  data.tar.gz: 10de6fca4203d5e2ef706967545d3e24460c4593382a47b58e9fae2eeb240dce9ec19b213469e8a6ad05d87552e9aed5ee3314bcf6de4a453c330fd603f4742e
+  metadata.gz: 7dee93db8b64e8411898c26b174fe4f37ccdd5088db95076940cf8513c7af3f6061e882221de8cd82d90152323497a23ea18309428f2b7ff26b3a90ea3b69b46
+  data.tar.gz: e304a39fb8f4ce6a3c3df896c014f8b60eda3a58b18b4e9f52c7de1eeee67b3f7fd3b12e9652d7e50ccc7344377a9c289fd9c0aa14d4a6c13dd9ee75fcf39fab

data/Gemfile

@@ -1,4 +1,3 @@
 source "https://rubygems.org"
-gemspec
-
-gem 'startback', path: '../..'
+gem 'startback', path: "."
+gemspec :name => 'startback-web'

data/README.md

@@ -0,0 +1,13 @@
+# Startback - Got Your Ruby Back
+
+Yet another ruby framework, I'm afraid. Here, we srongly seperate between:
+
+1. the web layer, in charge of a quality HTTP handling
+2. the operations layer, in charge of the high-level software operations
+3. the database layer, abstracted using the Relations As First Class Citizen pattern
+
+Currently,
+
+1. is handled using extra support on top of Sinatra
+2. is handled using Startback specific classes
+3. is handled using Bmg

data/lib/startback/audit/prometheus.rb

@@ -0,0 +1,87 @@
+require_relative 'shared'
+require 'prometheus/client'
+
+module Startback
+  module Audit
+    #
+    # Prometheus exporter abstraction, that can be registered as an around
+    # hook on OperationRunner and as a prometheus client on Context instances.
+    #
+    # The exporter uses the ruby client for prometheus to expose metrics regarding Operation runs.
+    #
+    # The following metrics are exported:
+    #
+    # A counter 'operation_errors' (failed runs)
+    # A histogram 'operation_calls'
+    #
+    # All these metrics use the following labels
+    # - operation : class name of the operation executed
+    #
+    # Given that this Exporter is intended to be used as around hook on an
+    # `OperationRunner`, operations that fail at construction time will not be
+    # exported at all, since they can't be ran in the first place. This may lead
+    # to metrics not containing important errors cases if operations check their
+    # input at construction time.
+    #
+    class Prometheus
+      include Shared
+
+      def initialize(options = {})
+        @prefix = options[:prefix] || "startback"
+        @options = options
+        @registry = ::Prometheus::Client.registry
+        all_labels = [:operation, :startback_version] + option_labels.keys
+        @errors = @registry.counter(
+          :"#{prefix}_operation_errors",
+          docstring: 'A counter of operation errors',
+          labels: all_labels)
+        @calls = @registry.histogram(
+          :"#{prefix}_operation_calls",
+          docstring: 'A histogram of operation latency',
+          labels: all_labels)
+      end
+      attr_reader :registry, :calls, :errors, :options, :prefix
+
+      def call(runner, op)
+        name = op_name(op)
+        result = nil
+        time = Benchmark.realtime{
+          result = yield
+        }
+        ignore_safely {
+          @calls.observe(time, labels: get_labels(name))
+        }
+        result
+      rescue => ex
+        ignore_safely {
+          @errors.increment(labels: get_labels(name))
+        }
+        raise
+      end
+
+    protected
+
+      def ignore_safely
+        yield
+      rescue => ex
+        nil
+      end
+
+      def get_labels(op_name)
+        option_labels.merge({
+          operation: op_name,
+          startback_version: version
+        })
+      end
+
+      def option_labels
+        @options[:labels] || {}
+      end
+
+      def version
+        Startback::VERSION
+      end
+
+    end # class Prometheus
+  end # module Audit
+end # module Startback

data/lib/startback/audit/shared.rb

@@ -0,0 +1,17 @@
+module Startback
+  module Audit
+    module Shared
+
+      def op_name(op)
+        return op.op_name if op.respond_to?(:op_name)
+
+        case op
+        when String then op
+        when Class  then op.name
+        else op.class.name
+        end
+      end
+
+    end # module Shared
+  end # module Audit
+end # module Startback

data/lib/startback/audit/trailer.rb

@@ -0,0 +1,129 @@
+require_relative 'shared'
+require 'forwardable'
+module Startback
+  module Audit
+    #
+    # Log & Audit trail abstraction, that can be registered as an around
+    # hook on OperationRunner and as an actual logger on Context instances.
+    #
+    # The trail is outputted as JSON lines, using a Logger on the "device"
+    # passed at construction. The following JSON entries are dumped:
+    #
+    # - severity  : INFO or ERROR
+    # - time      : ISO8601 Datetime of operation execution
+    # - op        : class name of the operation executed
+    # - op_took   : Execution duration of the operation
+    # - op_data   : Dump of operation input data
+    # - context   : Execution context, through its `h` information contract (IC)
+    #
+    # Dumping of operation data follows the following duck typing conventions:
+    #
+    # - If the operation instance responds to `to_trail`, this data is taken
+    # - If the operation instance responds to `input`, this data is taken
+    # - If the operation instance responds to `request`, this data is taken
+    # - Otherwise op_data is a JSON null
+    #
+    # By contributing to the Context's `h` IC, users can easily dump information that
+    # makes sense (such as the operation execution requester).
+    #
+    # The class implements a sanitization process when dumping the context and
+    # operation data. Blacklisted words taken in construction options are used to
+    # prevent dumping hash keys that match them (insentively). Default stop words
+    # are equivalent to:
+    #
+    #     Trailer.new("/var/log/trail.log", {
+    #       blacklist: "token password secret credential"
+    #     })
+    #
+    # Please note that the sanitization process does not apply recursively if
+    # the operation data is hierarchic. It only applies to the top object of
+    # Hash and [Hash]. Use `Operation#to_trail` to fine-tune your audit trail.
+    #
+    # Given that this Trailer is intended to be used as around hook on an
+    # `OperationRunner`, operations that fail at construction time will not be
+    # trailed at all, since they can't be ran in the first place. This may lead
+    # to trails not containing important errors cases if operations check their
+    # input at construction time.
+    #
+    class Trailer
+      include Shared
+      extend Forwardable
+      def_delegators :@logger, :debug, :info, :warn, :error, :fatal
+
+      DEFAULT_OPTIONS = {
+
+        # Words used to stop dumping for, e.g., security reasons
+        blacklist: "token password secret credential"
+
+      }
+
+      def initialize(device, options = {})
+        @options = DEFAULT_OPTIONS.merge(options)
+        @logger = ::Logger.new(device, 'daily')
+        @logger.formatter = Support::LogFormatter.new
+      end
+      attr_reader :logger, :options
+
+      def call(runner, op)
+        result = nil
+        time = Benchmark.realtime{ result = yield }
+        logger.info(op_to_trail(op, time))
+        result
+      rescue => ex
+        logger.error(op_to_trail(op, time, ex))
+        raise
+      end
+
+    protected
+
+      def op_to_trail(op, time = nil, ex = nil)
+        log_msg = {
+          op_took: time ? time.round(8) : nil,
+          op: op_name(op),
+          context: op_context(op),
+          op_data: op_data(op)
+        }.compact
+        log_msg[:error] = ex if ex
+        log_msg
+      end
+
+      def op_context(op)
+        sanitize(op.respond_to?(:context, false) ? op.context.to_h : {})
+      end
+
+      def op_data(op)
+        data = if op.respond_to?(:op_data, false)
+          op.op_data
+        elsif op.respond_to?(:to_trail, false)
+          op.to_trail
+        elsif op.respond_to?(:input, false)
+          op.input
+        elsif op.respond_to?(:request, false)
+          op.request
+        elsif op.is_a?(Operation::MultiOperation)
+          op.ops.map{ |sub_op| op_to_trail(sub_op) }
+        end
+        sanitize(data)
+      end
+
+      def sanitize(data)
+        case data
+        when Hash, OpenStruct
+          data.dup.delete_if{|k| k.to_s =~ blacklist_rx }
+        when Enumerable
+          data.map{|elm| sanitize(elm) }.compact
+        else
+          data
+        end
+      end
+
+      def blacklist_rx
+        @blacklist_rx ||= Regexp.new(
+          options[:blacklist].split(/\s+/).join("|"),
+          Regexp::IGNORECASE
+        )
+      end
+
+    end # class Trailer
+  end # module Audit
+end # module Startback

data/lib/startback/audit.rb

@@ -0,0 +1,3 @@
+require_relative 'audit/shared'
+require_relative 'audit/trailer'
+require_relative 'audit/prometheus'

data/lib/startback/caching/entity_cache.rb

@@ -0,0 +1,157 @@
+module Startback
+  module Caching
+    #
+    # A overriable caching abstraction aiming at making Entity-based caching easy.
+    #
+    # This class MUST be overriden:
+    #
+    # * the `load_entity` protected method MUST be implemented to load data from
+    #   a primary & context unaware key.
+    #
+    # * the `primary_key` protected method MAY be implemented to convert candidate
+    #   keys (received from ultimate callers) to primary keys. The method is also
+    #   a good place to check and/or log the keys actually used by callers.
+    #
+    # * the `context_free_key` protected method MAY be overriden to provide
+    #   domain unrelated caching keys from primary keys, e.g. by encoding the
+    #   context into the caching key itself, if needed.
+    #
+    # * the `valid?` protected method MAY be overriden to check validity of data
+    #   extracted from the cache and force a refresh even if found.
+    #
+    # An EntityCache takes an actual store at construction. The object must meet the
+    # specification writtern in Store. The 'cache' ruby gem can be used in practice.
+    #
+    # Cache hits, outdated and miss are logged in debug, info, and info severity.
+    # The `cache_hit`, `cache_outdated`, `cache_miss` protected methods MAY be
+    # overriden to change that behavior.
+    #
+    class EntityCache
+      include Support::Robustness
+
+      class << self
+
+        # Default time to live, in seconds
+        attr_writer :default_ttl
+
+        def default_ttl
+          @default_ttl || (superclass.respond_to?(:default_ttl, true) && superclass.default_ttl) || 3600
+        end
+
+      end # class DSL
+
+      def initialize(store, context = nil)
+        @store = store
+        @context = context
+      end
+      attr_reader :store, :context
+
+      # Returns the entity corresponding to a given key.
+      #
+      # If the entity is not in cache, loads it and puts it in cache using
+      # the caching options passed as second parameter.
+      def get(candidate_key, caching_options = default_caching_options)
+        pkey = primary_key(candidate_key)
+        cache_key = encode_key(context_free_key(pkey))
+        if store.exist?(cache_key)
+          cached = store.get(cache_key)
+          if valid?(pkey, cached)
+            cache_hit(pkey, cached)
+            return cached
+          else
+            cache_outdated(pkey, cached)
+          end
+        end
+        cache_miss(pkey)
+        load_entity(pkey).tap{|to_cache|
+          store.set(cache_key, to_cache, caching_options)
+        }
+      end
+
+      # Invalidates the cache under a given key.
+      def invalidate(candidate_key)
+        pkey = primary_key(candidate_key)
+        cache_key = encode_key(context_free_key(pkey))
+        store.delete(cache_key)
+      end
+
+    protected
+
+      def cache_hit(pkey, cached)
+        log(:debug, self, "cache_hit", context, op_data: pkey)
+      end
+
+      def cache_outdated(pkey, cached)
+        log(:info, self, "cache_outdated", context, op_data: pkey)
+      end
+
+      def cache_miss(pkey)
+        log(:info, self, "cache_miss", context, op_data: pkey)
+      end
+
+      def default_caching_options
+        { ttl: self.class.default_ttl }
+      end
+
+      # Converts a candidate key to a primary key, so as to prevent
+      # cache duplicates if callers are allowed to request an entity
+      # through various keys.
+      #
+      # The default implementation returns the candidate key and MAY
+      # be overriden.
+      def primary_key(candidate_key)
+        candidate_key
+      end
+
+      # Encodes a context free key to an actual cache key.
+      #
+      # Default implementation uses JSON.fast_generate but MAY be
+      # overriden.
+      def encode_key(context_free_key)
+        JSON.fast_generate(context_free_key)
+      end
+
+      # Returns whether `cached` entity seems fresh enough to
+      # be returned as a cache hit.
+      #
+      # This method provides a way to check freshness using, e.g.
+      # `updated_at` or `etag` kind of entity fields. The default
+      # implementation returns true and MAY be overriden.
+      def valid?(primary_key, cached)
+        true
+      end
+
+      # Converts a primary_key to a context_free_key, using the
+      # context (instance variable) to encode the context itself
+      # into the actual cache key.
+      #
+      # The default implementation simply returns the primary key
+      # and MAY be overriden.
+      def context_free_key(primary_key)
+        full_key(primary_key)
+      end
+
+      # Deprecated, will be removed in 0.6.0. Use context_free_key
+      # instead.
+      def full_key(primary_key)
+        primary_key
+      end
+
+      # Actually loads the entity using the given primary key, and
+      # possibly the cache context.
+      #
+      # This method MUST be implemented and raises a NotImplementedError
+      # by default.
+      def load_entity(primary_key)
+        load_raw_data(primary_key)
+      end
+
+      # Deprecated, will be removed in 0.6.0. Use load_entity
+      # instead.
+      def load_raw_data(*args, &bl)
+        raise NotImplementedError, "#{self.class.name}#load_entity"
+      end
+
+    end # class EntityCache
+  end # module Caching
+end # module Startback

data/lib/startback/caching/no_store.rb

@@ -0,0 +1,28 @@
+module Startback
+  module Caching
+    #
+    # Caching store implementation that caches nothing at all.
+    #
+    class NoStore
+
+      def initialize
+      end
+
+      def exist?(key)
+        false
+      end
+
+      def get(key)
+        nil
+      end
+
+      def set(key, value, ttl)
+        value
+      end
+
+      def delete(key)
+      end
+
+    end # class NoStore
+  end # module Caching
+end # module Startback

data/lib/startback/caching/store.rb

@@ -0,0 +1,34 @@
+module Startback
+  module Caching
+    #
+    # Caching store specification & dummy implementation.
+    #
+    # This class should not be used in real project, as it implements
+    # See the 'cache' gem that provides conforming implementations.
+    #
+    class Store
+
+      def initialize
+        @saved = {}
+      end
+      attr_reader :saved
+
+      def exist?(key)
+        saved.has_key?(key)
+      end
+
+      def get(key)
+        saved[key]
+      end
+
+      def set(key, value, ttl)
+        saved[key] = value
+      end
+
+      def delete(key)
+        saved.delete(key)
+      end
+
+    end # class Store
+  end # module Caching
+end # module Startback

data/lib/startback/context/h_factory.rb

@@ -0,0 +1,43 @@
+module Startback
+  class Context
+    module HFactory
+
+      def h(hash)
+        h_factor!(self.new, hash)
+      end
+
+      def h_factor!(context, hash)
+        h_factories.each do |f|
+          f.call(context, hash)
+        end
+        context
+      end
+
+      def h_factories
+        @h_factories ||= []
+      end
+
+      def h_factory(&factory)
+        h_factories << factory
+      end
+
+      ###
+
+      def h_dump!(context, hash = {})
+        h_dumpers.each do |d|
+          context.instance_exec(hash, &d)
+        end
+        hash
+      end
+
+      def h_dumpers
+        @h_dumpers ||= []
+      end
+
+      def h_dump(&dumper)
+        h_dumpers << dumper
+      end
+
+    end # module HFactory
+  end # class Context
+end # module Startback

data/lib/startback/context/middleware.rb

@@ -0,0 +1,53 @@
+module Startback
+  class Context
+    #
+    # Rack middleware that installs a particular context instance
+    # on the Rack environment.
+    #
+    # Examples:
+    #
+    #     # Use the default context class
+    #     Rack::Builder.new do
+    #       use Startback::Context::Middleware
+    #
+    #       run ->(env){
+    #         ctx = env[Startback::Context::Middleware::RACK_ENV_KEY]
+    #         ctx.is_a?(Startback::Context) # => true
+    #       }
+    #     end
+    #
+    #     # Use a user defined context class
+    #     Rack::Builder.new do
+    #       use Startback::Context::Middleware, MyContextClass.new
+    #
+    #       run ->(env){
+    #         ctx = env[Startback::Context::Middleware::RACK_ENV_KEY]
+    #         ctx.is_a?(MyContextClass)     # => true (your subclass)
+    #         ctx.is_a?(Startback::Context) # => true (required!)
+    #       }
+    #     end
+    #
+    class Middleware
+
+      RACK_ENV_KEY = 'SAMBACK_CONTEXT'
+
+      def initialize(app, context = Context.new)
+        @app = app
+        @context = context
+      end
+      attr_reader :context
+
+      def call(env)
+        env[RACK_ENV_KEY] ||= context.dup.tap{|c|
+          c.original_rack_env = env.dup
+        }
+        @app.call(env)
+      end
+
+      def self.context(env)
+        env[RACK_ENV_KEY]
+      end
+
+    end # class Middleware
+  end # class Context
+end # module Startback