svcbase 0.1.16

data/lib/svcbase/exceptions.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+module Core
+  module Exceptions
+    # errors
+    class Error < RuntimeError
+      # This sets the locale file scope. It may be redefined by subclasses to change
+      # that scope.
+      MESSAGE_SCOPE = 'app.errors'
+
+      attr_reader :details, :headers, :loglevel, :msgcode, :msgobj
+
+      def self.classname
+        name.demodulize
+      end
+
+      # STATUS should be redefined by subclasses. It cannot be overridden by ctor so it is fixed for a
+      # given subclass.
+      STATUS = 500
+      def status
+        self.class::STATUS
+      end
+
+      # Each Error class's default msgcode will be based on the class's name unless
+      # DEFAULT_MSGCODE is set for it or a superclass.
+      # Supplying a msgcode on the constructor will override the default.
+      DEFAULT_MSGCODE = nil
+      def self.default_msgcode
+        self::DEFAULT_MSGCODE || classname.underscore.to_sym # Exceptions::AuthError -> :auth_error
+      end
+
+      # Each Error class's default loglevel will be based on the class's STATUS unless
+      # DEFAULT_LOGLEVEL is set for it or a superclass.
+      # Supplying a loglevel on the constructor will override the default.
+      DEFAULT_LOGLEVEL = nil
+      def self.default_loglevel
+        self::DEFAULT_LOGLEVEL || ((400..499).cover?(self::STATUS) ? :error : :fatal)
+      end
+
+      # rubocop:disable AbcSize
+      def initialize(msg = nil, loglevel: nil, logmsg: nil, headers: nil, **details)
+        raise "new not allowed for #{self.class}" if self.class == Exceptions::Error
+
+        @msgcode = msg || details.delete(:msg) || self.class.default_msgcode
+        raise 'msg must be a symbol' unless @msgcode.is_a? Symbol
+
+        message = I18n.t!(@msgcode, scope: self.class::MESSAGE_SCOPE,
+                                    classname: self.class.name,
+                                    **details)
+        @msgobj = details.delete :msgobj
+        @details = details
+        @loglevel = loglevel || self.class.default_loglevel
+        @logmsg = logmsg
+        @headers = headers || {}
+        unless details.empty? || details[:message]
+          details[:message] ||= message
+          details[:code] ||= msgcode
+          details[:obj] ||= msgobj if msgobj
+        end
+        super message
+      rescue I18n::ArgumentError => e
+        # make sure we see these if they happen!
+        log.fatal e
+        raise
+      end
+      # rubocop:enable AbcSize
+
+      # This is called by sa_logger.
+      def logmsg
+        msg = @logmsg || message
+        return msg unless msg.is_a? String # this lets us log objects or exceptions too
+        "#{self.class.classname} - #{msg}"
+      end
+
+      # The response object that is returned to the caller.
+      def response
+        response = {
+          status: :error,
+          error: {
+            code: msgcode,
+            message: message
+          }
+        }
+        response[:error][:obj] = msgobj if msgobj
+        response[:error][:details] = details unless details.empty?
+        response
+      end
+    end
+
+    class NotModified < Exceptions::Error
+      STATUS = 304
+      DEFAULT_LOGLEVEL = :none
+    end
+
+    class BadRequest < Exceptions::Error
+      STATUS = 400
+    end
+
+    class ValidationFailed < BadRequest
+      DEFAULT_MSGCODE = :invalid
+    end
+
+    class Forbidden < Exceptions::Error
+      STATUS = 403
+    end
+
+    class NotFound < Exceptions::Error
+      STATUS = 404
+    end
+
+    class Conflict < Exceptions::Error
+      STATUS = 409
+    end
+
+    class Gone < Exceptions::Error
+      STATUS = 410
+    end
+
+    class UnsupportedMediaType < Exceptions::Error
+      STATUS = 415
+    end
+
+    class Unprocessable < Exceptions::Error
+      STATUS = 422
+    end
+
+    class TooManyRequests < Exceptions::Error
+      STATUS = 429
+    end
+
+    # Opaque errors should write their msg to the log, but should not
+    # return any useful data to the caller.
+    #
+    # Note that the raise syntax is different for these messages, since they
+    # only take a single logmsg and an optional loglevel.
+    #
+    # Examples:
+    #   raise Exceptions::Fatal
+    #   raise Exceptions::Fatal, 'Something is amiss'
+    #   raise Exceptions::Fatal, 'Something is awry', loglevel: warn
+    #     # still returns a 500 internal server error but logs at warn instead of fatal
+    class OpaqueError < Exceptions::Error
+      def initialize(logmsg = 'UNKNOWN', loglevel: nil)
+        raise "new not allowed for #{self.class}" if self.class == Exceptions::OpaqueError
+        super(self.class.default_msgcode, logmsg: logmsg, loglevel: loglevel)
+      end
+
+      def response
+        { status: :error, error: { code: msgcode } }
+      end
+    end
+
+    # specialized class for 401
+    class Unauthorized < Exceptions::OpaqueError
+      STATUS = 401
+      DEFAULT_MSGCODE = :unauthorized # force all subclasses to return this msgcode as well
+      # ensure we have a WWW-Authenticate header as required by RFC
+      def headers
+        @headers.merge('WWW-Authenticate' => 'Bearer')
+      end
+    end
+
+    class Fatal < Exceptions::OpaqueError
+      STATUS = 500
+      DEFAULT_MSGCODE = :internal_server_error
+    end
+  end
+end

data/lib/svcbase/formatter.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'request_store'
+require 'time'
+
+# add a helper method to Exception to provide a relative backtrace
+class Exception
+  def relative_backtrace
+    return [] unless backtrace&.is_a?(Array)
+    trace = backtrace
+    stack = caller
+    while trace.last && stack.last && trace.last == stack.last
+      trace.pop
+      stack.pop
+    end
+    trace
+  end
+end
+
+# Log formatter
+module Core
+  module Formatters
+    # Request logger
+    class Log
+      def call(severity, datetime, prog, data)
+        base = base_format(severity, datetime, prog)
+
+        arr = Array(data) unless data.is_a? Hash
+        arr = [data] if data.is_a? Hash
+
+        # need the last "\n" because .join won't do it
+        arr.each do |datum|
+          if datum.is_a? Hash
+            base.merge!(datum)
+          else
+            base["message"] = format(datum)
+          end
+        end
+
+        base.to_json << "\n"
+      end
+
+      private def store_value(field)
+        f = RequestStore.store[field]
+        return nil if f.nil?
+        return nil if f.respond_to?(:empty?) && f.empty?
+        f
+      end
+
+      private def base_format(severity, datetime, prog)
+        req = store_value(:http_request_id)
+        mfa = store_value(:mfa_request_id)
+        uid = store_value(:uid)
+        sid = store_value(:session_id)
+
+        json_properties = {}
+        json_properties['ts'] = datetime.utc.iso8601(3)
+        json_properties['tid'] = Thread.current.object_id.to_s(32)
+        json_properties['sev'] = severity
+        json_properties['sid'] = sid unless sid.nil?
+        json_properties['req'] = req unless req.nil?
+        json_properties['mfa'] = mfa unless mfa.nil?
+        json_properties['uid'] = uid unless uid.nil?
+        json_properties['src'] = prog unless prog.nil?
+
+        json_properties
+      end
+
+      private def format(data)
+        if data.is_a?(String)
+          data
+        elsif data.is_a?(Exception)
+          format_exception(data)
+        else
+          data.inspect
+        end
+      end
+
+      private def format_exception(exc)
+        text = ["#{exc.class} - #{exc.message}", *exc.relative_backtrace]
+        if exc.cause
+          text << '--- Caused by: ---'
+          text << "#{exc.cause.class} - #{exc.cause.message}"
+          text << exc.cause.relative_backtrace
+        end
+        text.join("\n\t")
+      end
+    end
+  end
+end

data/lib/svcbase/ipaddr_helper.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+
+# monkey-patch IPAddr class to provide some cidr-related accessors
+class IPAddr
+  def cidr?
+    full_mask = case @family
+                when Socket::AF_INET then IN4MASK
+                when Socket::AF_INET6 then IN6MASK
+                end
+    raise AddressFamilyError, 'unsupported address family' unless full_mask
+    @mask_addr != full_mask
+  end
+
+  def mask_bits
+    @mask_addr.to_s(2).sub(/0+$/, '').length
+  end
+
+  def canonical
+    to_s + (cidr? ? "/#{mask_bits}" : '')
+  end
+
+  PRIVATE_IP_RANGES = [
+    IPAddr.new('10.0.0.0/8'),
+    IPAddr.new('172.16.0.0/12'),
+    IPAddr.new('192.168.0.0/16'),
+    IPAddr.new('fc00::/7')
+  ].freeze
+
+  def private?
+    @private ||= PRIVATE_IP_RANGES.any? { |rng| rng.include? self }
+  end
+end

data/lib/svcbase/middleware/apilogger.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/try'
+require 'svcbase/exceptions'
+require 'grape'
+require 'svcbase/corelogger'
+require 'svcbase/stats'
+require 'svcbase/thresholder'
+
+module Core
+  # cribbed heavily from grape-middleware-logger
+  class ApiLogger < Grape::Middleware::Globals
+    BACKSLASH = '/'
+
+    attr_reader :logger
+
+    class << self
+      attr_accessor :logger, :filter
+    end
+
+    def initialize(_unused, options = {})
+      super
+      @options[:filter] ||= self.class.filter
+      @logger = options[:logger] || self.class.logger || self.class.default_logger
+    end
+
+    # @note Error and exception handling are required for the +after+ hooks
+    #   Exceptions are logged as a 500 status and re-raised
+    #   Other "errors" are caught, logged and re-thrown
+    def call!(env)
+      @env = env
+      before
+      error = catch(:error) do
+        app_response = @app.call(@env)
+        @status, = *app_response
+        return app_response # NB: this exits the entire function, not just this block
+      end
+      # this is reached only if we caught an error! throw
+      @status = handle_throw(error)
+      # convert grape 401/500 to App 401/500 but skip logging (uncommon)
+      raise Core::Exceptions::Unauthorized, loglevel: :none if @status == 401
+      raise Core::Exceptions::Fatal, loglevel: :none if @status == 500
+      throw(:error, error)
+    rescue StandardError => e
+      @status = handle_exception(e)
+      raise
+    ensure
+      # this will run regardless of whether we exit via raise, throw, or return
+      after(@status)
+    end
+
+    private def before
+      @start_time = Time.now.utc
+      super
+    end
+
+    # rubocop:disable AbcSize
+    private def after(status)
+      @end_time = Time.now.utc
+      request_took_too_long = elapsed_time > Config.get_f!(:REQ_LOG_WARN_MILLIS, 1000)
+      auth_id = RequestStore.store[:auth_id]
+
+      if get_route_option(:stat_log)
+        identifiers = [RequestStore.store[:oid], auth_id].compact
+        Core::Stats.collect get_route_option(:stat_log), *identifiers, status, elapsed_time
+      end
+
+      # only suppress if successful and we didn't take too long
+      return if (status == 304 || (200..299).cover?(status)) && !request_took_too_long && suppress?(status)
+
+      r = env[Grape::Env::GRAPE_REQUEST]
+
+      info = { _status: status, _verb: r.request_method,
+               path: r.path, route: processed_by, api_id: auth_id,
+               params: filtered_parameters,
+               elapsed: elapsed_time,
+               ip: request.env['HTTP_X_FORWARDED_FOR'] || request.env['REMOTE_ADDR'],
+               ua: request.env['HTTP_USER_AGENT'] }
+
+      log.warn "#{processed_by} request took #{elapsed_time} msec" if request_took_too_long
+      log.info info
+    end
+    # rubocop:enable AbcSize
+
+    #
+    # Helpers
+    #
+
+    private def suppress?(status)
+      return true if get_route_option(:suppress_log)
+      # create either an array with the oid, or an empty array
+      # this will be splatted into the threshold calls below to qualify the namespace
+      oid = [RequestStore.store[:oid]].compact
+
+      if status == 304
+        threshold_304_log = get_route_option(:threshold_304_log)
+        Core::Thresholder.instance(threshold_304_log, *oid).limit { return true } if threshold_304_log
+      end
+
+      threshold_log = get_route_option(:threshold_log)
+      Core::Thresholder.instance(threshold_log, *oid).limit { return true } if threshold_log
+
+      false
+    end
+
+    private def handle_throw(exc)
+      # exc _should_ be a hash thrown by grape, but check just in case!
+      if exc.respond_to?(:[])
+        log.error "Error!: #{exc[:message] || 'UNKNOWN'}"
+        return exc[:status]
+      end
+      log.fatal "Unexpected error!: #{exc || 'UNKNOWN'}"
+      500
+    end
+
+    private def handle_exception(exc)
+      case exc
+      when Grape::Exceptions::ValidationErrors
+        loglevel = exc.all? { |_attr, err| err.try(:benign) } ? :debug : :error
+        logmsg = "Error: #{exc.message}"
+      when Grape::Exceptions::Base
+        loglevel = :error
+        logmsg = "Error: #{exc.message}"
+      when Core::Exceptions::Error
+        loglevel = exc.is_a?(Core::Exceptions::NotFound) && parameters['log404'] == 'false' ? :none : exc.loglevel
+        logmsg = exc.logmsg
+      else
+        loglevel = :none # this will get logged by the rescue_from at the top level
+      end
+
+      # don't log the exception here
+      log.send loglevel, logmsg
+
+      exc.respond_to?(:status) ? exc.status : 500
+    end
+
+    private def parameters
+      # we ignore these, these are params from the route
+      # request_params = env[Grape::Env::GRAPE_REQUEST_PARAMS].to_hash
+      @parameters ||= request
+                      .params
+                      .dup
+                      .merge!(env['action_dispatch.request.request_parameters'] || {})
+    end
+
+    private def filtered_parameters
+      @options[:filter]&.filter(parameters, sensitive_params) || parameters
+    end
+
+    private def elapsed_time
+      @elapsed_time ||= ((@end_time - @start_time) * 1000).round(2)
+    end
+
+    private def endpoint_object
+      @endpoint_object ||= env[Grape::Env::API_ENDPOINT]
+    end
+
+    private def endpoint_options
+      @endpoint_options ||= (endpoint_object&.options) || {}
+    end
+
+    private def route_options
+      @route_options ||= endpoint_options[:route_options] || {}
+    end
+
+    private def get_route_option(key)
+      route_options[key] || RequestStore.store[key]
+    end
+
+    private def sensitive_params
+      @sensitive_params ||= route_options[:mask_log]
+    end
+
+    private def processed_by
+      endpoint_options[:for].to_s << '#' << endpoint_options[:path].map do |path|
+        path.to_s.sub(BACKSLASH, '')
+      end.join(BACKSLASH)
+    end
+
+    private def request
+      @request ||= ::Rack::Request.new(env)
+    end
+  end
+end

data/lib/svcbase/middleware/dateheader.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Core
+  # Add Date header to output
+  class AddDateHeader
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      status, headers, body = @app.call(env)
+      headers['Date'] ||= Time.now.utc.httpdate
+      [status, headers, body]
+    end
+  end
+end

data/lib/svcbase/middleware/requestid.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'request_store'
+require 'svcbase/random'
+
+module Core
+  # Add per-request tracking
+  class RequestId
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      # RequestStore gem allows us to not muck with thread local storage directly
+      RequestStore.store[:http_request_id] = env['HTTP_X_REQUEST_ID'] || Core::Random.short_id
+      status, headers, body = @app.call(env)
+      headers['X-Request-Id'] ||= RequestStore.store[:http_request_id]
+      [status, headers, body]
+    end
+  end
+end

data/lib/svcbase/random.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'shortuuid'
+require 'base64'
+
+module Core
+  # random id helpers
+  module Random
+    def self.short_id
+      ShortUUID.shorten(SecureRandom.uuid)
+    end
+
+    def self.uuid
+      SecureRandom.uuid
+    end
+
+    def self.base64_id(bytes)
+      Base64.strict_encode64(SecureRandom.random_bytes(bytes))
+    end
+
+    def self.urlsafe_base64_id(bytes)
+      Base64.urlsafe_encode64(SecureRandom.random_bytes(bytes), padding: false)
+    end
+
+    # generate an integer i such that 0 <= i < max_plus_one
+    def self.number(max_plus_one)
+      SecureRandom.random_number(max_plus_one)
+    end
+  end
+end

data/lib/svcbase/server.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'svcbase/appversion'
+require 'svcbase/dumpstats'
+require 'svcbase/random'
+require 'svcbase/worker'
+
+module Core
+  # actual server manager
+  class Server
+    include Singleton
+
+    ID = Core::Random.short_id.freeze
+
+    private def initialize
+      @workers = []
+
+      register_worker(Core::Workers::DumpStats.instance)
+    end
+
+    def register_worker(klass)
+      log.info('server') { "Registered server worker #{klass}" }
+      @workers << klass
+    end
+
+    def start_workers
+      @workers.each do |inst|
+        inst.worker_start
+      rescue StandardError => e
+        log.fatal('server') { ["ERROR in worker #{inst}", e] }
+      end
+    end
+
+    def stop_workers
+      @workers.each do |inst|
+        inst.worker_stop
+      rescue StandardError => e
+        log.fatal('server') { ["ERROR in worker #{inst}", e] }
+      end
+    end
+
+    def start
+      log.info('server') { "Server #{ID} starting" }
+      log.info('server') { "Commit: #{Core::Version::ID}" }
+      log.info('server') { "Tag: #{Core::Version::TAG}" }
+
+      start_workers
+    end
+
+    def stop
+      log.info('server') { "Server #{ID} stopping" }
+
+      stop_workers
+
+      log.info('server') { "Server #{ID} stopped" }
+    end
+  end
+end

data/lib/svcbase/stats.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Core
+  # Utility module to collect statistics
+  module Stats
+    @mutex = Mutex.new
+    @buffer = {}
+
+    # accumulate count, sum, and sum of squares to support calculation of mean and population standard deviation
+    # note: use the first value as a zero offset to reduce floating point precision errors
+    class Accumulator
+      attr_reader :count, :max, :min
+
+      def initialize
+        @count = 0
+        @sum = 0
+        @sumsqr = 0
+      end
+
+      def add_value(value)
+        @offset ||= value
+        @max = value if @max.nil? || value > @max
+        @min = value if @min.nil? || value < @max
+        offset_val = value - @offset
+        @sum += offset_val
+        @sumsqr += offset_val**2
+        @count += 1
+      end
+
+      def mean
+        @offset + @sum / @count
+      end
+
+      def stddev
+        Math.sqrt((@sumsqr - @sum**2 / @count) / @count)
+      end
+    end
+
+    # collect a numeric data point
+    # first n-1 args are used as hash keys, the last arg is the data value
+    def self.collect(*args)
+      raise ArgumentError unless args.length >= 2
+      value = args.pop
+      @mutex.synchronize do
+        # find/create the appropriate object within the buffer
+        obj = args.inject(@buffer) { |buf, key| buf[key.to_s] ||= {} }
+        # add the data value to the object's accumulator
+        (obj[:_data] ||= Accumulator.new).add_value(value)
+      end
+    end
+
+    # traverse the buffer and yield with each
+    private_class_method def self.traverse_data(hash)
+      stack = []
+      # add top level keys to the array in order
+      hash.keys.sort.map { |k| stack.push [[k], @buffer[k]] unless k == :_data }
+      until stack.empty?
+        keylist, obj = stack.shift
+        yield keylist, obj[:_data] if obj.key? :_data
+        # add next level of keys in reverse order to the front of the list
+        obj.keys.sort.reverse.map { |k| stack.unshift [keylist.dup << k, obj[k]] unless k == :_data }
+      end
+    end
+
+    def self.log_and_reset
+      msgs = ["Server: #{Server::ID}", "Commit: #{Core::Version::ID}", "Tag: #{Core::Version::TAG}"]
+
+      # app can define a #custom_messages method to customize reporting
+      msgs.concat(Array(custom_messages)) if respond_to? :custom_messages
+
+      @mutex.synchronize do
+        traverse_data(@buffer) do |keylist, data|
+          count  = data.count
+          min    = data.min
+          mean   = data.mean.round(2)
+          max    = data.max
+          stddev = data.stddev.round(2)
+          msgs << "#{keylist.join('|')}: count: #{count} min: #{min} mean: #{mean} max: #{max} stddev: #{stddev}"
+        end
+        @buffer = {}
+      end
+      msgs.each { |msg| log.info('stats') { msg } }
+    end
+  end
+end