service_skeleton 0.0.0.34.g4f6fdb0 → 0.0.0.49.g47046b9

data/lib/service_skeleton/config_variable/enum.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Enum < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      v = env[@name.to_s]
+
+      if @opts[:values].is_a?(Array)
+        unless @opts[:values].include?(v)
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Invalid value for #{@name}; must be one of #{@opts[:values].join(", ")}"
+        end
+        v
+      elsif @opts[:values].is_a?(Hash)
+        unless @opts[:values].keys.include?(v)
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Invalid value for #{@name}; must be one of #{@opts[:values].keys.join(", ")}"
+        end
+        @opts[:values][v]
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/float.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Float < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      value = env[@name.to_s]
+
+      if value =~ /\A-?\d+.?\d*\z/
+        value.to_f.tap do |f|
+          unless @opts[:range].include?(f)
+            raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                  "Value #{f} for environment variable #{@name} is out of the valid range (must be between #{@opts[:range].first} and #{@opts[:range].last} inclusive)"
+          end
+        end
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value #{value.inspect} for environment variable #{@name} is not a valid numeric value"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/integer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::Integer < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      value = env[@name.to_s]
+
+      if value =~ /\A-?\d+\z/
+        value.to_i.tap do |i|
+          unless @opts[:range].include?(i)
+            raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                  "Value #{i} for environment variable #{@name} is out of the valid range (must be between #{@opts[:range].first} and #{@opts[:range].last} inclusive)"
+          end
+        end
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value #{value.inspect} for environment variable #{@name} is not a valid integer value"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/kv_list.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::KVList < ServiceSkeleton::ConfigVariable
+  def redact!(env)
+    env.keys.each { |k| env[k] = "*SENSITIVE*" if k =~ @opts[:key_pattern] }
+  end
+
+  private
+
+  def pluck_value(env)
+    matches = env.select { |k, _| k.to_s =~ @opts[:key_pattern] }
+
+    if matches.empty?
+      if @opts.has_key?(:default)
+        @opts[:default]
+      else
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "no keys for key-value list #{@name} specified"
+      end
+    else
+      matches.transform_keys { |k| @opts[:key_pattern].match(k.to_s)[1].to_sym }
+    end
+  end
+end

data/lib/service_skeleton/config_variable/path_list.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::PathList < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      env[@name.to_s].split(":")
+    end
+  end
+end

data/lib/service_skeleton/config_variable/string.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::String < ServiceSkeleton::ConfigVariable
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      env[@name.to_s].tap do |s|
+        if @opts[:match] && s !~ @opts[:match]
+          raise ServiceSkeleton::Error::InvalidEnvironmentError,
+                "Value for #{@name} must match #{@opts[:match]}"
+        end
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variable/url.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::URL < ServiceSkeleton::ConfigVariable
+  def redact?(env)
+    !!(env.has_key?(@name.to_s) && (@opts[:sensitive] || URI(env[@name.to_s] || "").password))
+  end
+
+  def redact!(env)
+    if env.has_key?(@name.to_s)
+      super
+      uri = URI(env[@name.to_s])
+      if uri.password
+        uri.password = "*REDACTED*"
+        env[@name.to_s] = uri.to_s
+      end
+    end
+  end
+
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      begin
+        v = env[@name.to_s]
+        URI(v)
+      rescue URI::InvalidURIError
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Value for #{@name} (#{v}) does not appear to be a valid URL"
+      end
+
+      v
+    end
+  end
+end

data/lib/service_skeleton/config_variable/yaml_file.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+require "service_skeleton/config_variable"
+
+class ServiceSkeleton::ConfigVariable::YamlFile < ServiceSkeleton::ConfigVariable
+  def redact!(env)
+    if env.has_key?(@name.to_s)
+      if File.world_readable?(env[@name.to_s])
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Sensitive file #{env[@name.to_s]} is world-readable!"
+      end
+
+      super
+    end
+  end
+
+  private
+
+  def pluck_value(env)
+    maybe_default(env) do
+      begin
+        val = YAML.safe_load(File.read(env[@name.to_s]))
+        if @opts[:klass]
+          val = @opts[:klass].new(val)
+        end
+
+        val
+      rescue Errno::ENOENT
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "YAML file #{env[@name.to_s]} does not exist"
+      rescue Errno::EPERM
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Do not have permission to read YAML file #{env[@name.to_s]}"
+      rescue Psych::SyntaxError => ex
+        raise ServiceSkeleton::Error::InvalidEnvironmentError,
+              "Invalid YAML syntax: #{ex.message}"
+      end
+    end
+  end
+end

data/lib/service_skeleton/config_variables.rb

@@ -1,112 +1,79 @@
+# frozen_string_literal: true
+
 require_relative "./error"
 
 require "service_skeleton/config_variable"
 
-class ServiceSkeleton
+module ServiceSkeleton
   module ConfigVariables
-    attr_reader :registered_variables
-
-    UNDEFINED = Class.new
+    UNDEFINED = Module.new
     private_constant :UNDEFINED
 
-    def register_variable(name, **opts, &callback)
+    def registered_variables
       @registered_variables ||= []
-
-      @registered_variables << ServiceSkeleton::ConfigVariable.new(name, **opts, &callback)
     end
 
-    def string(var_name, default: UNDEFINED, sensitive: false)
-      register_variable(var_name, sensitive: sensitive) do |value|
-        maybe_default(value, default, var_name) do
-          value
-        end
+    def register_variable(name, klass, **opts)
+      if opts[:default] == UNDEFINED
+        opts.delete(:default)
       end
+
+      registered_variables << {
+        name: name,
+        class: klass,
+        opts: opts,
+      }
     end
 
     def boolean(var_name, default: UNDEFINED, sensitive: false)
-      register_variable(var_name, sensitive: sensitive) do |value|
-        maybe_default(value, default, var_name) do
-          case value
-          when /\A(no|n|off|0|false)\z/i
-            false
-          when /\A(yes|y|on|1|true)\z/i
-            true
-          else
-            raise ServiceSkeleton::Error::InvalidEnvironmentError,
-                  "Value #{value.inspect} for environment variable #{var_name} is not a valid boolean value"
-          end
-        end
-      end
+      register_variable(var_name, ConfigVariable::Boolean, default: default, sensitive: sensitive)
     end
 
-    def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
-      register_variable(var_name, sensitive: sensitive) do |value|
-        maybe_default(value, default, var_name) do
-          if value =~ /\A-?\d+\z/
-            value.to_i.tap do |i|
-              unless range.include?(i)
-                raise ServiceSkeleton::Error::InvalidEnvironmentError,
-                  "Value #{i} for environment variable #{var_name} is out of the valid range (must be between #{range.first} and #{range.last} inclusive)"
-              end
-            end
-          else
-            raise ServiceSkeleton::Error::InvalidEnvironmentError,
-                  "Value #{value.inspect} for environment variable #{var_name} is not a valid integer value"
-          end
-        end
+    def enum(var_name, values:, default: UNDEFINED, sensitive: false)
+      unless values.is_a?(Hash) || values.is_a?(Array)
+        raise ArgumentError,
+              "values option to enum must be a hash or array"
       end
+
+      register_variable(var_name, ConfigVariable::Enum, default: default, sensitive: sensitive, values: values)
     end
 
     def float(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
-      register_variable(var_name, sensitive: sensitive) do |value|
-        maybe_default(value, default, var_name) do
-          if value =~ /\A-?\d+.?\d*\z/
-            value.to_f.tap do |i|
-              unless range.include?(i)
-                raise ServiceSkeleton::Error::InvalidEnvironmentError,
-                  "Value #{i} for environment variable #{var_name} is out of the valid range (must be between #{range.first} and #{range.last} inclusive)"
-              end
-            end
-          else
-            raise ServiceSkeleton::Error::InvalidEnvironmentError,
-                  "Value #{value.inspect} for environment variable #{var_name} is not a valid numeric value"
-          end
-        end
-      end
+      register_variable(var_name, ConfigVariable::Float, default: default, sensitive: sensitive, range: range)
+    end
+
+    def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
+      register_variable(var_name, ConfigVariable::Integer, default: default, sensitive: sensitive, range: range)
+    end
+
+    def kv_list(var_name, default: UNDEFINED, sensitive: false, key_pattern: /\A#{var_name}_(.*)\z/)
+      register_variable(var_name, ConfigVariable::KVList, default: default, sensitive: sensitive, key_pattern: key_pattern)
     end
 
     def path_list(var_name, default: UNDEFINED, sensitive: false)
-      register_variable(var_name, sensitive: sensitive) do |value|
-        maybe_default(value, default, var_name) do
-          value.split(":")
-        end
-      end
+      register_variable(var_name, ConfigVariable::PathList, default: default, sensitive: sensitive)
     end
 
-    def kv_list(var_name, default: UNDEFINED, sensitive: false, key_pattern: nil)
-      key_pattern ||= /\A#{var_name}_(.*)\z/
-      register_variable(var_name, sensitive: sensitive, key_pattern: key_pattern) do |matches|
-        maybe_default(matches, default, var_name) do
-          matches.transform_keys do |k|
-            key_pattern.match(k)[1].to_sym
-          end
-        end
-      end
+    def string(var_name, default: UNDEFINED, sensitive: false, match: nil)
+      register_variable(var_name, ConfigVariable::String, default: default, sensitive: sensitive, match: match)
     end
 
-    private
-
-    def maybe_default(value, default, var_name)
-      if value.nil? || value == {}
-        if default == UNDEFINED
-          raise ServiceSkeleton::Error::InvalidEnvironmentError,
-                "Value for required environment variable #{var_name} not specified"
-        else
-          default
-        end
-      else
-        yield
-      end
+    def url(var_name, default: UNDEFINED, sensitive: false)
+      register_variable(var_name, ConfigVariable::URL, default: default, sensitive: sensitive)
+    end
+
+    def yaml_file(var_name, default: UNDEFINED, sensitive: false, klass: nil)
+      register_variable(var_name, ConfigVariable::YamlFile, default: default, sensitive: sensitive, klass: klass)
     end
   end
 end
+
+require_relative "config_variable/boolean"
+require_relative "config_variable/enum"
+require_relative "config_variable/float"
+require_relative "config_variable/integer"
+require_relative "config_variable/kv_list"
+require_relative "config_variable/path_list"
+require_relative "config_variable/string"
+require_relative "config_variable/url"
+require_relative "config_variable/yaml_file"

data/lib/service_skeleton/error.rb

@@ -1,8 +1,10 @@
-class ServiceSkeleton
+# frozen_string_literal: true
+
+module ServiceSkeleton
   class Error < StandardError
-    class InvalidEnvironmentError < Error; end
     class CannotSanitizeEnvironmentError < Error; end
-    class InheritanceContractError < Error; end
+    class InvalidEnvironmentError < Error; end
     class InvalidMetricNameError < Error; end
+    class InvalidServiceClassError < Error; end
   end
 end

data/lib/service_skeleton/filtering_logger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'logger'
 
 module FilteringLogger

data/lib/service_skeleton/generator.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require_relative "config"
+require_relative "signal_manager"
+require_relative "ultravisor_loggerstash"
+
+require "frankenstein/ruby_gc_metrics"
+require "frankenstein/ruby_vm_metrics"
+require "frankenstein/process_metrics"
+require "frankenstein/server"
+require "prometheus/client/registry"
+require "sigdump"
+require "ultravisor"
+
+module ServiceSkeleton
+  module Generator
+    def generate(config:, metrics_registry:, service_metrics:, service_signal_handlers:)
+      Ultravisor.new(logger: config.logger).tap do |ultravisor|
+        initialize_metrics(ultravisor, config, metrics_registry, service_metrics)
+        initialize_loggerstash(ultravisor, config, metrics_registry)
+        initialize_signals(ultravisor, config, service_signal_handlers, metrics_registry)
+      end
+    end
+
+    private
+
+    def initialize_metrics(ultravisor, config, registry, metrics)
+      Frankenstein::RubyGCMetrics.register(registry)
+      Frankenstein::RubyVMMetrics.register(registry)
+      Frankenstein::ProcessMetrics.register(registry)
+
+      metrics.each do |m|
+        registry.register(m)
+
+        method_name = m.method_name(config.service_name)
+
+        if registry.singleton_class.method_defined?(method_name)
+          raise ServiceSkeleton::Error::InvalidMetricNameError,
+                "Metric method #{method_name} is already defined"
+        end
+
+        registry.define_singleton_method(method_name) do
+          m
+        end
+      end
+
+      if config.metrics_port
+        config.logger.info(config.service_name) { "Starting metrics server on port #{config.metrics_port}" }
+        ultravisor.add_child(
+          id: :metrics_server,
+          klass: Frankenstein::Server,
+          method: :run,
+          args: [
+            port: config.metrics_port,
+            logger: config.logger,
+            metrics_prefix: :"#{config.service_name}_metrics_server",
+            registry: registry,
+          ]
+        )
+      end
+    end
+
+    def initialize_loggerstash(ultravisor, config, registry)
+      if config.logstash_server && !config.logstash_server.empty?
+        config.logger.info(config.service_name) { "Configuring loggerstash to send to #{config.logstash_server}" }
+
+        ultravisor.add_child(
+          id: :logstash_writer,
+          klass: LogstashWriter,
+          method: :run,
+          args: [
+            server_name: config.logstash_server,
+            metrics_registry: registry,
+            logger: config.logger,
+          ],
+          access: :unsafe
+        )
+
+        config.logger.singleton_class.prepend(Loggerstash::Mixin)
+
+        config.logger.instance_variable_set(:@ultravisor, ultravisor)
+        config.logger.singleton_class.prepend(ServiceSkeleton::UltravisorLoggerstash)
+      end
+    end
+
+    def initialize_signals(ultravisor, config, service_signals, metrics_registry)
+      counter = metrics_registry.counter(:"#{config.service_name}_signals_handled_total", docstring: "How many of each signal have been handled", labels: %i{signal})
+
+      ultravisor.add_child(
+        id: :signal_manager,
+        klass: ServiceSkeleton::SignalManager,
+        method: :run,
+        args: [
+          logger: config.logger,
+          counter: counter,
+          signals: global_signals(ultravisor, config.logger) + wrap_service_signals(service_signals, ultravisor),
+        ],
+        shutdown: {
+          method: :shutdown,
+          timeout: 1,
+        }
+      )
+    end
+
+    def global_signals(ultravisor, logger)
+      # For mysterious reasons of mystery, simplecov doesn't recognise these
+      # procs as being called, even though there are definitely tests for
+      # them.  So...
+      #:nocov:
+      [
+        [
+          "USR1",
+          ->() {
+            logger.level -= 1 unless logger.level == Logger::DEBUG
+            logger.info($0) { "Received SIGUSR1; log level is now #{Logger::SEV_LABEL[logger.level]}." }
+          }
+        ],
+        [
+          "USR2",
+          ->() {
+            logger.level += 1 unless logger.level == Logger::ERROR
+            logger.info($0) { "Received SIGUSR2; log level is now #{Logger::SEV_LABEL[logger.level]}." }
+          }
+        ],
+        [
+          "HUP",
+          ->() {
+            logger.reopen
+            logger.info($0) { "Received SIGHUP; log file handle reopened" }
+          }
+        ],
+        [
+          "QUIT",
+          ->() { Sigdump.dump("+") }
+        ],
+        [
+          "INT",
+          ->() {
+            ultravisor.shutdown(wait: false, force: !!@shutting_down)
+            @shutting_down = true
+          }
+        ],
+        [
+          "TERM",
+          ->() {
+            ultravisor.shutdown(wait: false, force: !!@shutting_down)
+            @shutting_down = true
+          }
+        ]
+      ]
+      #:nocov:
+    end
+
+    def wrap_service_signals(signals, ultravisor)
+      [].tap do |signal_list|
+        signals.each do |service_name, sigs|
+          sigs.each do |sig, proc|
+            wrapped_proc = ->() { ultravisor[service_name.to_sym].unsafe_instance.instance_eval(&proc) }
+            signal_list << [sig, wrapped_proc]
+          end
+        end
+      end
+    end
+  end
+end