service_skeleton 0.0.0.44.g75d07d7 → 1.0.3

data/lib/service_skeleton.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
-require_relative "service_skeleton/config"
+require_relative "service_skeleton/config_class"
 require_relative "service_skeleton/config_variables"
+require_relative "service_skeleton/generator"
 require_relative "service_skeleton/logging_helpers"
 require_relative "service_skeleton/metrics_methods"
-require_relative "service_skeleton/signal_handler"
+require_relative "service_skeleton/service_name"
+require_relative "service_skeleton/signals_methods"
+require_relative "service_skeleton/ultravisor_children"
 
 require "frankenstein/ruby_gc_metrics"
 require "frankenstein/ruby_vm_metrics"
@@ -13,195 +16,26 @@ require "frankenstein/server"
 require "prometheus/client/registry"
 require "sigdump"
 
-class ServiceSkeleton
-  extend ServiceSkeleton::ConfigVariables
-
+module ServiceSkeleton
   include ServiceSkeleton::LoggingHelpers
+  extend ServiceSkeleton::Generator
 
-  class Terminate < Exception; end
-
-  def self.config_class(klass)
-    @config_class = klass
-  end
-
-  def self.service_name
-    service_name_from_class(self)
+  def self.included(mod)
+    mod.extend ServiceSkeleton::ServiceName
+    mod.extend ServiceSkeleton::ConfigVariables
+    mod.extend ServiceSkeleton::ConfigClass
+    mod.extend ServiceSkeleton::MetricsMethods
+    mod.extend ServiceSkeleton::SignalsMethods
+    mod.extend ServiceSkeleton::UltravisorChildren
   end
 
   attr_reader :config, :metrics, :logger
 
-  def initialize(env)
-    @env      = env
-    @config   = (self.class.instance_variable_get(:@config_class) || ServiceSkeleton::Config).new(env, self)
-    @logger   = @config.logger
-    @op_mutex = Mutex.new
-
-    initialize_metrics
-    initialize_signals
-  end
-
-  def start
-    @op_mutex.synchronize { @thread = Thread.current }
-
-    begin
-      logger.info(logloc) { "Starting service #{service_name}" }
-      logger.info(logloc) { (["Environment:"] + config.env.map { |k, v| "#{k}=#{v.inspect}" }).join("\n  ") }
-
-      start_metrics_server
-      start_signal_handler
-      run
-    rescue ServiceSkeleton::Terminate
-      # This one is OK
-    rescue ServiceSkeleton::Error::InheritanceContractError
-      # We want this one to be fatal
-      raise
-    rescue StandardError => ex
-      log_exception(ex)
-    end
-
-    @thread = nil
-  end
-
-  def stop(force = false)
-    if force
-      #:nocov:
-      @op_mutex.synchronize do
-        if @thread
-          @thread.raise(ServiceSkeleton::Terminate)
-        end
-      end
-      #:nocov:
-    else
-      shutdown
-    end
-
-    if @metrics_server
-      @metrics_server.shutdown
-      @metrics_server = nil
-    end
-
-    @signal_handler.stop!
-  end
-
-  def service_name
-    self.class.service_name
-  end
-
-  def registered_variables
-    self.class.registered_variables
-  end
-
-  def hook_signal(spec, &blk)
-    @signal_handler.hook_signal(spec, &blk)
-  end
-
-  private
-
-  def self.service_name_from_class(klass)
-    klass.to_s
-      .gsub("::", "_")
-      .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-      .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-      .downcase
-      .gsub(/[^a-zA-Z0-9_]/, "_")
-  end
-
-  def run
-    raise ServiceSkeleton::Error::InheritanceContractError, "ServiceSkeleton#run method not overridden"
-  end
-
-  def shutdown
-    #:nocov:
-    @op_mutex.synchronize do
-      if @thread
-        @thread.raise(ServiceSkeleton::Terminate)
-        @thread.join
-        @thread = nil
-      end
-    end
-    #:nocov:
-  end
-
-  def initialize_metrics
-    @metrics = Prometheus::Client::Registry.new
-
-    Frankenstein::RubyGCMetrics.register(@metrics)
-    Frankenstein::RubyVMMetrics.register(@metrics)
-    Frankenstein::ProcessMetrics.register(@metrics)
-
-    @metrics.singleton_class.prepend(ServiceSkeleton::MetricsMethods)
-    @metrics.service = self
-  end
-
-  def start_metrics_server
-    if config.metrics_port
-      logger.info(logloc) { "Starting metrics server on port #{config.metrics_port}" }
-
-      @metrics_server = Frankenstein::Server.new(
-        port: config.metrics_port,
-        logger: logger,
-        metrics_prefix: :metrics_server,
-        registry: @metrics,
-      )
-      @metrics_server.run
-    end
-  end
-
-  def initialize_signals
-    metrics.counter(:"#{self.service_name}_signals_handled_total", "How many of each type of signal have been handled")
-    @signal_handler = ServiceSkeleton::SignalHandler.new(logger: logger, service: self, signal_counter: metrics.signals_handled_total)
-
-    @signal_handler.hook_signal("USR1") do
-      logger.level -= 1 unless logger.level == Logger::DEBUG
-      logger.info($0) { "Received SIGUSR1; log level is now #{Logger::SEV_LABEL[logger.level]}." }
-    end
-
-    @signal_handler.hook_signal("USR2") do
-      logger.level += 1 unless logger.level == Logger::ERROR
-      logger.info($0) { "Received SIGUSR2; log level is now #{Logger::SEV_LABEL[logger.level]}." }
-    end
-
-    @signal_handler.hook_signal("HUP") do
-      logger.reopen
-      logger.info($0) { "Received SIGHUP; log file handle reopened" }
-    end
-
-    @signal_handler.hook_signal("QUIT") do
-      Sigdump.dump("+")
-    end
-
-    @signal_handler.hook_signal("INT") do
-      self.stop(!!@terminating)
-      @terminating = true
-    end
-
-    @signal_handler.hook_signal("TERM") do
-      self.stop(!!@terminating)
-      @terminating = true
-    end
-  end
-
-  def start_signal_handler
-    @signal_handler.start!
-  end
-
-  @registered_variables = [
-    { name: :SERVICE_SKELETON_LOG_LEVEL,             class: ConfigVariable::String,  opts: { default: "INFO" } },
-    { name: :SERVICE_SKELETON_LOG_ENABLE_TIMESTAMPS, class: ConfigVariable::Boolean, opts: { default: false } },
-    { name: :SERVICE_SKELETON_LOG_FILE,              class: ConfigVariable::String,  opts: { default: nil } },
-    { name: :SERVICE_SKELETON_LOG_MAX_FILE_SIZE,     class: ConfigVariable::Integer, opts: { default: 1048576, range: 0..Float::INFINITY } },
-    { name: :SERVICE_SKELETON_LOG_MAX_FILES,         class: ConfigVariable::Integer, opts: { default: 3, range: 0..Float::INFINITY } },
-    { name: :SERVICE_SKELETON_LOGSTASH_SERVER,       class: ConfigVariable::String,  opts: { default: nil } },
-    { name: :SERVICE_SKELETON_METRICS_PORT,          class: ConfigVariable::Integer, opts: { default: nil, range: 1..65535 } },
-  ]
-
-  def self.inherited(subclass)
-    subclass.string(:"#{subclass.service_name.upcase}_LOG_LEVEL", default: "INFO")
-    subclass.boolean(:"#{subclass.service_name.upcase}_LOG_ENABLE_TIMESTAMPS", default: false)
-    subclass.string(:"#{subclass.service_name.upcase}_LOG_FILE", default: nil)
-    subclass.integer(:"#{subclass.service_name.upcase}_LOG_MAX_FILE_SIZE", default: 1048576, range: 0..Float::INFINITY)
-    subclass.integer(:"#{subclass.service_name.upcase}_LOG_MAX_FILES", default: 3, range: 1..Float::INFINITY)
-    subclass.string(:"#{subclass.service_name.upcase}_LOGSTASH_SERVER", default: "")
-    subclass.integer(:"#{subclass.service_name.upcase}_METRICS_PORT", default: nil, range: 1..65535)
+  def initialize(*_, metrics:, config:)
+    @metrics = metrics
+    @config  = config
+    @logger  = @config.logger
   end
 end
+
+require_relative "service_skeleton/runner"

data/lib/service_skeleton/config.rb

@@ -6,18 +6,19 @@ require_relative "./filtering_logger"
 
 require "loggerstash"
 
-class ServiceSkeleton
+module ServiceSkeleton
   class Config
-    attr_reader :logger, :env
+    attr_reader :logger, :pre_run_logger, :env, :service_name
 
-    def initialize(env, svc)
-      @svc = svc
+    def initialize(env, service_name, variables)
+      @service_name = service_name
 
-      # Parsing registered variables will redact the environment, so we want
-      # to take a private unredacted copy before that happens
+      # Parsing variables will redact the environment, so we want to take a
+      # private unredacted copy before that happens for #[] lookup in the
+      # future.
       @env = env.to_hash.dup.freeze
 
-      parse_registered_variables(env)
+      parse_variables(internal_variables + variables, env)
 
       # Sadly, we can't setup the logger until we know *how* to setup the
       # logger, which requires parsing config variables
@@ -25,17 +26,17 @@ class ServiceSkeleton
     end
 
     def [](k)
-      @env[k]
+      @env[k].dup
     end
 
     private
 
-    def parse_registered_variables(env)
-      (@svc.registered_variables || []).map do |var|
+    def parse_variables(variables, env)
+      variables.map do |var|
         var[:class].new(var[:name], env, **var[:opts])
       end.each do |var|
         val = var.value
-        method_name = var.method_name(@svc.service_name).to_sym
+        method_name = var.method_name(@service_name).to_sym
 
         define_singleton_method(method_name) do
           val
@@ -65,30 +66,42 @@ class ServiceSkeleton
 
       @logger = Logger.new(log_file || $stderr, shift_age, shift_size)
 
-      if self.logstash_server && !self.logstash_server.empty?
-        loggerstash = Loggerstash.new(logstash_server: logstash_server, logger: @logger)
-        loggerstash.metrics_registry = @svc.metrics
-        loggerstash.attach(@logger)
-      end
+      # Can be used prior to a call to ultravisor#run. This prevents a race condition
+      # when a logstash server is configured but the logstash writer is not yet
+      # initialised. This should never be updated after it is configured.
+      @pre_run_logger = Logger.new(log_file || $stderr, shift_age, shift_size)
 
-      thread_id_map = {}
       if Thread.main
-        thread_id_map[Thread.main.object_id] = 0
+        Thread.main[:thread_map_number] = 0
       else
         #:nocov:
-        thread_id_map[Thread.current.object_id] = 0
+        Thread.current[:thread_map_number] = 0
         #:nocov:
       end
 
-      @logger.formatter = ->(s, t, p, m) do
-        th_n = thread_id_map[Thread.current.object_id] || (thread_id_map[Thread.current.object_id] = thread_id_map.length)
+      thread_map_mutex = Mutex.new
+
+      [@logger, @pre_run_logger].each do |logger|
+        logger.formatter = ->(s, t, p, m) do
+          th_n = if Thread.current.name
+            #:nocov:
+            Thread.current.name
+            #:nocov:
+          else
+            thread_map_mutex.synchronize do
+              Thread.current[:thread_map_number] ||= begin
+                Thread.list.select { |th| th[:thread_map_number] }.length
+              end
+            end
+          end
 
-        ts = log_enable_timestamps ? "#{t.utc.strftime("%FT%T.%NZ")} " : ""
-        "#{ts}#{$$}##{th_n} #{s[0]} [#{p}] #{m}\n"
+          ts = log_enable_timestamps ? "#{t.utc.strftime("%FT%T.%NZ")} " : ""
+          "#{ts}#{$$}##{th_n} #{s[0]} [#{p}] #{m}\n"
+        end
       end
 
       @logger.filters = []
-      @env.fetch("#{@svc.service_name.upcase}_LOG_LEVEL", "INFO").split(/\s*,\s*/).each do |spec|
+      @env.fetch("#{@service_name.upcase}_LOG_LEVEL", "INFO").split(/\s*,\s*/).each do |spec|
         if spec.index("=")
           # "Your developers were so preoccupied with whether or not they
           # could, they didn't stop to think if they should."
@@ -111,5 +124,17 @@ class ServiceSkeleton
         end
       end
     end
+
+    def internal_variables
+      [
+        { name: "#{@service_name.upcase}_LOG_LEVEL",             class: ConfigVariable::String,  opts: { default: "INFO" } },
+        { name: "#{@service_name.upcase}_LOG_ENABLE_TIMESTAMPS", class: ConfigVariable::Boolean, opts: { default: false } },
+        { name: "#{@service_name.upcase}_LOG_FILE",              class: ConfigVariable::String,  opts: { default: nil } },
+        { name: "#{@service_name.upcase}_LOG_MAX_FILE_SIZE",     class: ConfigVariable::Integer, opts: { default: 1048576, range: 0..Float::INFINITY } },
+        { name: "#{@service_name.upcase}_LOG_MAX_FILES",         class: ConfigVariable::Integer, opts: { default: 3,       range: 1..Float::INFINITY } },
+        { name: "#{@service_name.upcase}_LOGSTASH_SERVER",       class: ConfigVariable::String,  opts: { default: "" } },
+        { name: "#{@service_name.upcase}_METRICS_PORT",          class: ConfigVariable::Integer, opts: { default: nil,     range: 1..65535 } },
+      ]
+    end
   end
 end

data/lib/service_skeleton/config_class.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ServiceSkeleton
+  module ConfigClass
+    Undefined = Module.new
+    private_constant :Undefined
+
+    def config_class(klass = Undefined)
+      unless klass == Undefined
+        @config_class = klass
+      end
+
+      @config_class
+    end
+  end
+end

data/lib/service_skeleton/config_variable.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ServiceSkeleton
+module ServiceSkeleton
   class ConfigVariable
     attr_reader :name, :value
 

data/lib/service_skeleton/config_variable/yaml_file.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::YamlFile < ServiceSkeleton::ConfigVariable
+  def redact!(env)
+    if env.has_key?(@name.to_s)
+      if File.world_readable?(env[@name.to_s])
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Sensitive file #{env[@name.to_s]} is world-readable!"
+      end
+
+      super
+    end
+  end
+
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      begin
+        val = YAML.safe_load(File.read(env[@name.to_s]))
+        if @opts[:klass]
+          val = @opts[:klass].new(val)
+        end
+
+        val
+      rescue Errno::ENOENT
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "YAML file #{env[@name.to_s]} does not exist"
+      rescue Errno::EPERM
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Do not have permission to read YAML file #{env[@name.to_s]}"
+      rescue Psych::SyntaxError => ex
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Invalid YAML syntax: #{ex.message}"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variables.rb

@@ -4,71 +4,76 @@ require_relative "./error"
 
 require "service_skeleton/config_variable"
 
-class ServiceSkeleton
+module ServiceSkeleton
   module ConfigVariables
-    attr_reader :registered_variables
-
     UNDEFINED = Module.new
     private_constant :UNDEFINED
 
-    def register_variable(name, klass, **opts)
+    def registered_variables
       @registered_variables ||= []
+    end
 
+    def register_variable(name, klass, **opts)
       if opts[:default] == UNDEFINED
         opts.delete(:default)
       end
 
-      @registered_variables << {
+      registered_variables << {
         name: name,
         class: klass,
         opts: opts,
       }
     end
 
-    def string(var_name, default: UNDEFINED, sensitive: false, match: nil)
-      register_variable(var_name, ConfigVariable::String, default: default, sensitive: sensitive, match: match)
-    end
-
     def boolean(var_name, default: UNDEFINED, sensitive: false)
       register_variable(var_name, ConfigVariable::Boolean, default: default, sensitive: sensitive)
     end
 
-    def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
-      register_variable(var_name, ConfigVariable::Integer, default: default, sensitive: sensitive, range: range)
+    def enum(var_name, values:, default: UNDEFINED, sensitive: false)
+      unless values.is_a?(Hash) || values.is_a?(Array)
+        raise ArgumentError,
+              "values option to enum must be a hash or array"
+      end
+
+      register_variable(var_name, ConfigVariable::Enum, default: default, sensitive: sensitive, values: values)
     end
 
     def float(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
       register_variable(var_name, ConfigVariable::Float, default: default, sensitive: sensitive, range: range)
     end
 
-    def path_list(var_name, default: UNDEFINED, sensitive: false)
-      register_variable(var_name, ConfigVariable::PathList, default: default, sensitive: sensitive)
+    def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
+      register_variable(var_name, ConfigVariable::Integer, default: default, sensitive: sensitive, range: range)
     end
 
     def kv_list(var_name, default: UNDEFINED, sensitive: false, key_pattern: /\A#{var_name}_(.*)\z/)
       register_variable(var_name, ConfigVariable::KVList, default: default, sensitive: sensitive, key_pattern: key_pattern)
     end
 
-    def enum(var_name, values:, default: UNDEFINED, sensitive: false)
-      unless values.is_a?(Hash) || values.is_a?(Array)
-        raise ArgumentError,
-              "values option to enum must be a hash or array"
-      end
+    def path_list(var_name, default: UNDEFINED, sensitive: false)
+      register_variable(var_name, ConfigVariable::PathList, default: default, sensitive: sensitive)
+    end
 
-      register_variable(var_name, ConfigVariable::Enum, default: default, sensitive: sensitive, values: values)
+    def string(var_name, default: UNDEFINED, sensitive: false, match: nil)
+      register_variable(var_name, ConfigVariable::String, default: default, sensitive: sensitive, match: match)
     end
 
     def url(var_name, default: UNDEFINED, sensitive: false)
       register_variable(var_name, ConfigVariable::URL, default: default, sensitive: sensitive)
     end
+
+    def yaml_file(var_name, default: UNDEFINED, sensitive: false, klass: nil)
+      register_variable(var_name, ConfigVariable::YamlFile, default: default, sensitive: sensitive, klass: klass)
+    end
   end
 end
 
-require_relative "config_variable/string"
 require_relative "config_variable/boolean"
-require_relative "config_variable/integer"
+require_relative "config_variable/enum"
 require_relative "config_variable/float"
-require_relative "config_variable/path_list"
+require_relative "config_variable/integer"
 require_relative "config_variable/kv_list"
-require_relative "config_variable/enum"
+require_relative "config_variable/path_list"
+require_relative "config_variable/string"
 require_relative "config_variable/url"
+require_relative "config_variable/yaml_file"