service_skeleton 0.0.0.49.g47046b9

data/lib/service_skeleton.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "service_skeleton/config_class"
+require_relative "service_skeleton/config_variables"
+require_relative "service_skeleton/generator"
+require_relative "service_skeleton/logging_helpers"
+require_relative "service_skeleton/metrics_methods"
+require_relative "service_skeleton/service_name"
+require_relative "service_skeleton/signals_methods"
+require_relative "service_skeleton/ultravisor_children"
+
+require "frankenstein/ruby_gc_metrics"
+require "frankenstein/ruby_vm_metrics"
+require "frankenstein/process_metrics"
+require "frankenstein/server"
+require "prometheus/client/registry"
+require "sigdump"
+
+module ServiceSkeleton
+  include ServiceSkeleton::LoggingHelpers
+  extend ServiceSkeleton::Generator
+
+  def self.included(mod)
+    mod.extend ServiceSkeleton::ServiceName
+    mod.extend ServiceSkeleton::ConfigVariables
+    mod.extend ServiceSkeleton::ConfigClass
+    mod.extend ServiceSkeleton::MetricsMethods
+    mod.extend ServiceSkeleton::SignalsMethods
+    mod.extend ServiceSkeleton::UltravisorChildren
+  end
+
+  attr_reader :config, :metrics, :logger
+
+  def initialize(*_, metrics:, config:)
+    @metrics = metrics
+    @config  = config
+    @logger  = @config.logger
+  end
+end
+
+require_relative "service_skeleton/runner"

data/lib/service_skeleton/config.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "to_regexp"
+
+require_relative "./filtering_logger"
+
+require "loggerstash"
+
+module ServiceSkeleton
+  class Config
+    attr_reader :logger, :env, :service_name
+
+    def initialize(env, service_name, variables)
+      @service_name = service_name
+
+      # Parsing variables will redact the environment, so we want to take a
+      # private unredacted copy before that happens for #[] lookup in the
+      # future.
+      @env = env.to_hash.dup.freeze
+
+      parse_variables(internal_variables + variables, env)
+
+      # Sadly, we can't setup the logger until we know *how* to setup the
+      # logger, which requires parsing config variables
+      setup_logger
+    end
+
+    def [](k)
+      @env[k].dup
+    end
+
+    private
+
+    def parse_variables(variables, env)
+      variables.map do |var|
+        var[:class].new(var[:name], env, **var[:opts])
+      end.each do |var|
+        val = var.value
+        method_name = var.method_name(@service_name).to_sym
+
+        define_singleton_method(method_name) do
+          val
+        end
+
+        define_singleton_method(:"#{method_name}=") do |new_value|
+          val = new_value
+        end
+      end.each do |var|
+        if var.redact?(env)
+          if env.object_id != ENV.object_id
+            raise ServiceSkeleton::Error::CannotSanitizeEnvironmentError,
+                  "Attempted to sanitize sensitive variable #{var.name}, but we're not operating on the process' environment"
+          else
+            var.redact!(env)
+          end
+        end
+      end
+    end
+
+    def setup_logger
+      shift_age, shift_size = if log_max_file_size == 0
+        [0, 0]
+      else
+        [log_max_files, log_max_file_size]
+      end
+
+      @logger = Logger.new(log_file || $stderr, shift_age, shift_size)
+
+      if Thread.main
+        Thread.main[:thread_map_number] = 0
+      else
+        #:nocov:
+        Thread.current[:thread_map_number] = 0
+        #:nocov:
+      end
+
+      thread_map_mutex = Mutex.new
+
+      @logger.formatter = ->(s, t, p, m) do
+        th_n = if Thread.current.name
+          #:nocov:
+          Thread.current.name
+          #:nocov:
+        else
+          thread_map_mutex.synchronize do
+            Thread.current[:thread_map_number] ||= begin
+              Thread.list.select { |th| th[:thread_map_number] }.length
+            end
+          end
+        end
+
+        ts = log_enable_timestamps ? "#{t.utc.strftime("%FT%T.%NZ")} " : ""
+        "#{ts}#{$$}##{th_n} #{s[0]} [#{p}] #{m}\n"
+      end
+
+      @logger.filters = []
+      @env.fetch("#{@service_name.upcase}_LOG_LEVEL", "INFO").split(/\s*,\s*/).each do |spec|
+        if spec.index("=")
+          # "Your developers were so preoccupied with whether or not they
+          # could, they didn't stop to think if they should."
+          re, sev = spec.split(/\s*=\s*(?=[^=]*\z)/)
+          match = re.to_regexp || re
+          begin
+            sev = Logger.const_get(sev.upcase)
+          rescue NameError
+            raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                  "Unknown logger severity #{sev.inspect} specified in #{spec.inspect}"
+          end
+          @logger.filters << [match, sev]
+        else
+          begin
+            @logger.level = Logger.const_get(spec.upcase)
+          rescue NameError
+            raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                  "Unknown logger severity #{spec.inspect} specified"
+          end
+        end
+      end
+    end
+
+    def internal_variables
+      [
+        { name: "#{@service_name.upcase}_LOG_LEVEL",             class: ConfigVariable::String,  opts: { default: "INFO" } },
+        { name: "#{@service_name.upcase}_LOG_ENABLE_TIMESTAMPS", class: ConfigVariable::Boolean, opts: { default: false } },
+        { name: "#{@service_name.upcase}_LOG_FILE",              class: ConfigVariable::String,  opts: { default: nil } },
+        { name: "#{@service_name.upcase}_LOG_MAX_FILE_SIZE",     class: ConfigVariable::Integer, opts: { default: 1048576, range: 0..Float::INFINITY } },
+        { name: "#{@service_name.upcase}_LOG_MAX_FILES",         class: ConfigVariable::Integer, opts: { default: 3,       range: 1..Float::INFINITY } },
+        { name: "#{@service_name.upcase}_LOGSTASH_SERVER",       class: ConfigVariable::String,  opts: { default: "" } },
+        { name: "#{@service_name.upcase}_METRICS_PORT",          class: ConfigVariable::Integer, opts: { default: nil,     range: 1..65535 } },
+      ]
+    end
+  end
+end

data/lib/service_skeleton/config_class.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ServiceSkeleton
+  module ConfigClass
+    Undefined = Module.new
+    private_constant :Undefined
+
+    def config_class(klass = Undefined)
+      unless klass == Undefined
+        @config_class = klass
+      end
+
+      @config_class
+    end
+  end
+end

data/lib/service_skeleton/config_variable.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module ServiceSkeleton
+  class ConfigVariable
+    attr_reader :name, :value
+
+    def initialize(name, env, **opts, &blk)
+      @name = name
+      @opts = opts
+      @blk  = blk
+
+      @value = pluck_value(env)
+    end
+
+    def method_name(svc_name)
+      @name.to_s.gsub(/\A#{Regexp.quote(svc_name)}_/i, '').downcase
+    end
+
+    def redact?(env)
+      @opts[:sensitive]
+    end
+
+    def redact!(env)
+      if @opts[:sensitive]
+        env[@name.to_s] = "*SENSITIVE*" if env.has_key?(@name.to_s)
+      end
+    end
+
+    private
+
+    def maybe_default(env)
+      if env.has_key?(@name.to_s)
+        yield
+      else
+        if @opts.has_key?(:default)
+          @opts[:default]
+        else
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Value for required environment variable #{@name} not specified"
+        end
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/boolean.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Boolean < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      case env[@name.to_s]
+      when /\A(no|n|off|0|false)\z/i
+        false
+      when /\A(yes|y|on|1|true)\z/i
+        true
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value #{env[@name.to_s].inspect} for environment variable #{@name} is not a valid boolean value"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/enum.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Enum < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      v = env[@name.to_s]
+
+      if @opts[:values].is_a?(Array)
+        unless @opts[:values].include?(v)
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Invalid value for #{@name}; must be one of #{@opts[:values].join(", ")}"
+        end
+        v
+      elsif @opts[:values].is_a?(Hash)
+        unless @opts[:values].keys.include?(v)
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Invalid value for #{@name}; must be one of #{@opts[:values].keys.join(", ")}"
+        end
+        @opts[:values][v]
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/float.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Float < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      value = env[@name.to_s]
+
+      if value =~ /\A-?\d+.?\d*\z/
+        value.to_f.tap do |f|
+          unless @opts[:range].include?(f)
+            raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                  "Value #{f} for environment variable #{@name} is out of the valid range (must be between #{@opts[:range].first} and #{@opts[:range].last} inclusive)"
+          end
+        end
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value #{value.inspect} for environment variable #{@name} is not a valid numeric value"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/integer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Integer < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      value = env[@name.to_s]
+
+      if value =~ /\A-?\d+\z/
+        value.to_i.tap do |i|
+          unless @opts[:range].include?(i)
+            raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                  "Value #{i} for environment variable #{@name} is out of the valid range (must be between #{@opts[:range].first} and #{@opts[:range].last} inclusive)"
+          end
+        end
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value #{value.inspect} for environment variable #{@name} is not a valid integer value"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/kv_list.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::KVList < ServiceSkeleton::ConfigVariable
+  def redact!(env)
+    env.keys.each { |k| env[k] = "*SENSITIVE*" if k =~ @opts[:key_pattern] }
+  end
+
+  private
+
+  def pluck_value(env)
+    matches = env.select { |k, _| k.to_s =~ @opts[:key_pattern] }
+
+    if matches.empty?
+      if @opts.has_key?(:default)
+        @opts[:default]
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "no keys for key-value list #{@name} specified"
+      end
+    else
+      matches.transform_keys { |k| @opts[:key_pattern].match(k.to_s)[1].to_sym }
+    end
+  end
+end

data/lib/service_skeleton/config_variable/path_list.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::PathList < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      env[@name.to_s].split(":")
+    end
+  end
+end

data/lib/service_skeleton/config_variable/string.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::String < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      env[@name.to_s].tap do |s|
+        if @opts[:match] && s !~ @opts[:match]
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Value for #{@name} must match #{@opts[:match]}"
+        end
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/url.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::URL < ServiceSkeleton::ConfigVariable
+  def redact?(env)
+    !!(env.has_key?(@name.to_s) && (@opts[:sensitive] || URI(env[@name.to_s] || "").password))
+  end
+
+  def redact!(env)
+    if env.has_key?(@name.to_s)
+      super
+      uri = URI(env[@name.to_s])
+      if uri.password
+        uri.password = "*REDACTED*"
+        env[@name.to_s] = uri.to_s
+      end
+    end
+  end
+
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      begin
+        v = env[@name.to_s]
+        URI(v)
+      rescue URI::InvalidURIError
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value for #{@name} (#{v}) does not appear to be a valid URL"
+      end
+
+      v
+    end
+  end
+end

data/lib/service_skeleton/config_variable/yaml_file.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::YamlFile < ServiceSkeleton::ConfigVariable
+  def redact!(env)
+    if env.has_key?(@name.to_s)
+      if File.world_readable?(env[@name.to_s])
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Sensitive file #{env[@name.to_s]} is world-readable!"
+      end
+
+      super
+    end
+  end
+
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      begin
+        val = YAML.safe_load(File.read(env[@name.to_s]))
+        if @opts[:klass]
+          val = @opts[:klass].new(val)
+        end
+
+        val
+      rescue Errno::ENOENT
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "YAML file #{env[@name.to_s]} does not exist"
+      rescue Errno::EPERM
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Do not have permission to read YAML file #{env[@name.to_s]}"
+      rescue Psych::SyntaxError => ex
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Invalid YAML syntax: #{ex.message}"
+      end
+    end
+  end
+end