service_skeleton 0.0.0.1.ENOTAG

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +7 -0
  2. data/.editorconfig +7 -0
  3. data/.git-blame-ignore-revs +2 -0
  4. data/.github/workflows/ci.yml +50 -0
  5. data/.gitignore +9 -0
  6. data/.rubocop.yml +11 -0
  7. data/.yardopts +1 -0
  8. data/CODE_OF_CONDUCT.md +49 -0
  9. data/CONTRIBUTING.md +13 -0
  10. data/LICENCE +674 -0
  11. data/README.md +767 -0
  12. data/lib/service_skeleton.rb +41 -0
  13. data/lib/service_skeleton/config.rb +140 -0
  14. data/lib/service_skeleton/config_class.rb +16 -0
  15. data/lib/service_skeleton/config_variable.rb +44 -0
  16. data/lib/service_skeleton/config_variable/boolean.rb +21 -0
  17. data/lib/service_skeleton/config_variable/enum.rb +27 -0
  18. data/lib/service_skeleton/config_variable/float.rb +25 -0
  19. data/lib/service_skeleton/config_variable/integer.rb +25 -0
  20. data/lib/service_skeleton/config_variable/kv_list.rb +26 -0
  21. data/lib/service_skeleton/config_variable/path_list.rb +13 -0
  22. data/lib/service_skeleton/config_variable/string.rb +18 -0
  23. data/lib/service_skeleton/config_variable/url.rb +36 -0
  24. data/lib/service_skeleton/config_variable/yaml_file.rb +42 -0
  25. data/lib/service_skeleton/config_variables.rb +79 -0
  26. data/lib/service_skeleton/error.rb +10 -0
  27. data/lib/service_skeleton/filtering_logger.rb +38 -0
  28. data/lib/service_skeleton/generator.rb +165 -0
  29. data/lib/service_skeleton/logging_helpers.rb +28 -0
  30. data/lib/service_skeleton/metric_method_name.rb +9 -0
  31. data/lib/service_skeleton/metrics_methods.rb +37 -0
  32. data/lib/service_skeleton/runner.rb +46 -0
  33. data/lib/service_skeleton/service_name.rb +20 -0
  34. data/lib/service_skeleton/signal_manager.rb +202 -0
  35. data/lib/service_skeleton/signals_methods.rb +15 -0
  36. data/lib/service_skeleton/ultravisor_children.rb +20 -0
  37. data/lib/service_skeleton/ultravisor_loggerstash.rb +11 -0
  38. data/service_skeleton.gemspec +54 -0
  39. data/ultravisor/.yardopts +1 -0
  40. data/ultravisor/Guardfile +9 -0
  41. data/ultravisor/README.md +404 -0
  42. data/ultravisor/lib/ultravisor.rb +216 -0
  43. data/ultravisor/lib/ultravisor/child.rb +481 -0
  44. data/ultravisor/lib/ultravisor/child/call.rb +21 -0
  45. data/ultravisor/lib/ultravisor/child/call_receiver.rb +14 -0
  46. data/ultravisor/lib/ultravisor/child/cast.rb +16 -0
  47. data/ultravisor/lib/ultravisor/child/cast_receiver.rb +11 -0
  48. data/ultravisor/lib/ultravisor/child/process_cast_call.rb +39 -0
  49. data/ultravisor/lib/ultravisor/error.rb +25 -0
  50. data/ultravisor/lib/ultravisor/logging_helpers.rb +32 -0
  51. data/ultravisor/spec/example_group_methods.rb +19 -0
  52. data/ultravisor/spec/example_methods.rb +8 -0
  53. data/ultravisor/spec/spec_helper.rb +52 -0
  54. data/ultravisor/spec/ultravisor/add_child_spec.rb +79 -0
  55. data/ultravisor/spec/ultravisor/child/call_spec.rb +121 -0
  56. data/ultravisor/spec/ultravisor/child/cast_spec.rb +111 -0
  57. data/ultravisor/spec/ultravisor/child/id_spec.rb +21 -0
  58. data/ultravisor/spec/ultravisor/child/new_spec.rb +152 -0
  59. data/ultravisor/spec/ultravisor/child/restart_delay_spec.rb +40 -0
  60. data/ultravisor/spec/ultravisor/child/restart_spec.rb +70 -0
  61. data/ultravisor/spec/ultravisor/child/run_spec.rb +95 -0
  62. data/ultravisor/spec/ultravisor/child/shutdown_spec.rb +124 -0
  63. data/ultravisor/spec/ultravisor/child/spawn_spec.rb +107 -0
  64. data/ultravisor/spec/ultravisor/child/unsafe_instance_spec.rb +55 -0
  65. data/ultravisor/spec/ultravisor/child/wait_spec.rb +32 -0
  66. data/ultravisor/spec/ultravisor/new_spec.rb +71 -0
  67. data/ultravisor/spec/ultravisor/remove_child_spec.rb +49 -0
  68. data/ultravisor/spec/ultravisor/run_spec.rb +334 -0
  69. data/ultravisor/spec/ultravisor/shutdown_spec.rb +106 -0
  70. metadata +375 -0
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "yaml"
4
+
5
+ require "service_skeleton/config_variable"
6
+
7
+ class ServiceSkeleton::ConfigVariable::YamlFile < ServiceSkeleton::ConfigVariable
8
+ def redact!(env)
9
+ if env.has_key?(@name.to_s)
10
+ if File.world_readable?(env[@name.to_s])
11
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
12
+ "Sensitive file #{env[@name.to_s]} is world-readable!"
13
+ end
14
+
15
+ super
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ def pluck_value(env)
22
+ maybe_default(env) do
23
+ begin
24
+ val = YAML.safe_load(File.read(env[@name.to_s]))
25
+ if @opts[:klass]
26
+ val = @opts[:klass].new(val)
27
+ end
28
+
29
+ val
30
+ rescue Errno::ENOENT
31
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
32
+ "YAML file #{env[@name.to_s]} does not exist"
33
+ rescue Errno::EPERM
34
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
35
+ "Do not have permission to read YAML file #{env[@name.to_s]}"
36
+ rescue Psych::SyntaxError => ex
37
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
38
+ "Invalid YAML syntax: #{ex.message}"
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "./error"
4
+
5
+ require "service_skeleton/config_variable"
6
+
7
+ module ServiceSkeleton
8
+ module ConfigVariables
9
+ UNDEFINED = Module.new
10
+ private_constant :UNDEFINED
11
+
12
+ def registered_variables
13
+ @registered_variables ||= []
14
+ end
15
+
16
+ def register_variable(name, klass, **opts)
17
+ if opts[:default] == UNDEFINED
18
+ opts.delete(:default)
19
+ end
20
+
21
+ registered_variables << {
22
+ name: name,
23
+ class: klass,
24
+ opts: opts,
25
+ }
26
+ end
27
+
28
+ def boolean(var_name, default: UNDEFINED, sensitive: false)
29
+ register_variable(var_name, ConfigVariable::Boolean, default: default, sensitive: sensitive)
30
+ end
31
+
32
+ def enum(var_name, values:, default: UNDEFINED, sensitive: false)
33
+ unless values.is_a?(Hash) || values.is_a?(Array)
34
+ raise ArgumentError,
35
+ "values option to enum must be a hash or array"
36
+ end
37
+
38
+ register_variable(var_name, ConfigVariable::Enum, default: default, sensitive: sensitive, values: values)
39
+ end
40
+
41
+ def float(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
42
+ register_variable(var_name, ConfigVariable::Float, default: default, sensitive: sensitive, range: range)
43
+ end
44
+
45
+ def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
46
+ register_variable(var_name, ConfigVariable::Integer, default: default, sensitive: sensitive, range: range)
47
+ end
48
+
49
+ def kv_list(var_name, default: UNDEFINED, sensitive: false, key_pattern: /\A#{var_name}_(.*)\z/)
50
+ register_variable(var_name, ConfigVariable::KVList, default: default, sensitive: sensitive, key_pattern: key_pattern)
51
+ end
52
+
53
+ def path_list(var_name, default: UNDEFINED, sensitive: false)
54
+ register_variable(var_name, ConfigVariable::PathList, default: default, sensitive: sensitive)
55
+ end
56
+
57
+ def string(var_name, default: UNDEFINED, sensitive: false, match: nil)
58
+ register_variable(var_name, ConfigVariable::String, default: default, sensitive: sensitive, match: match)
59
+ end
60
+
61
+ def url(var_name, default: UNDEFINED, sensitive: false)
62
+ register_variable(var_name, ConfigVariable::URL, default: default, sensitive: sensitive)
63
+ end
64
+
65
+ def yaml_file(var_name, default: UNDEFINED, sensitive: false, klass: nil)
66
+ register_variable(var_name, ConfigVariable::YamlFile, default: default, sensitive: sensitive, klass: klass)
67
+ end
68
+ end
69
+ end
70
+
71
+ require_relative "config_variable/boolean"
72
+ require_relative "config_variable/enum"
73
+ require_relative "config_variable/float"
74
+ require_relative "config_variable/integer"
75
+ require_relative "config_variable/kv_list"
76
+ require_relative "config_variable/path_list"
77
+ require_relative "config_variable/string"
78
+ require_relative "config_variable/url"
79
+ require_relative "config_variable/yaml_file"
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ class Error < StandardError
5
+ class CannotSanitizeEnvironmentError < Error; end
6
+ class InvalidEnvironmentError < Error; end
7
+ class InvalidMetricNameError < Error; end
8
+ class InvalidServiceClassError < Error; end
9
+ end
10
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+
5
+ module FilteringLogger
6
+ attr_reader :filters
7
+
8
+ def filters=(f)
9
+ raise ArgumentError, "Must provide an array" unless f.is_a?(Array)
10
+
11
+ @filters = f
12
+ end
13
+
14
+ def add(s, m = nil, p = nil, &blk)
15
+ p ||= @progname
16
+
17
+ if @filters && p
18
+ @filters.each do |re, sev|
19
+ if re === p
20
+ if s < sev
21
+ return true
22
+ else
23
+ # We force the severity to nil for this call to override
24
+ # the logger's default severity filtering logic, because
25
+ # messages without a severity are always logged
26
+ return super(nil, m, p, &blk)
27
+ end
28
+ end
29
+ end
30
+ end
31
+
32
+ super
33
+ end
34
+
35
+ alias log add
36
+ end
37
+
38
+ Logger.prepend(FilteringLogger)
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "config"
4
+ require_relative "signal_manager"
5
+ require_relative "ultravisor_loggerstash"
6
+
7
+ require "frankenstein/ruby_gc_metrics"
8
+ require "frankenstein/ruby_vm_metrics"
9
+ require "frankenstein/process_metrics"
10
+ require "frankenstein/server"
11
+ require "prometheus/client/registry"
12
+ require "sigdump"
13
+ require_relative "../../ultravisor/lib/ultravisor"
14
+
15
+ module ServiceSkeleton
16
+ module Generator
17
+ def generate(config:, metrics_registry:, service_metrics:, service_signal_handlers:)
18
+ Ultravisor.new(logger: config.logger).tap do |ultravisor|
19
+ initialize_metrics(ultravisor, config, metrics_registry, service_metrics)
20
+ initialize_loggerstash(ultravisor, config, metrics_registry)
21
+ initialize_signals(ultravisor, config, service_signal_handlers, metrics_registry)
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def initialize_metrics(ultravisor, config, registry, metrics)
28
+ Frankenstein::RubyGCMetrics.register(registry)
29
+ Frankenstein::RubyVMMetrics.register(registry)
30
+ Frankenstein::ProcessMetrics.register(registry)
31
+
32
+ metrics.each do |m|
33
+ registry.register(m)
34
+
35
+ method_name = m.method_name(config.service_name)
36
+
37
+ if registry.singleton_class.method_defined?(method_name)
38
+ raise ServiceSkeleton::Error::InvalidMetricNameError,
39
+ "Metric method #{method_name} is already defined"
40
+ end
41
+
42
+ registry.define_singleton_method(method_name) do
43
+ m
44
+ end
45
+ end
46
+
47
+ if config.metrics_port
48
+ config.pre_run_logger.info(config.service_name) { "Starting metrics server on port #{config.metrics_port}" }
49
+ ultravisor.add_child(
50
+ id: :metrics_server,
51
+ klass: Frankenstein::Server,
52
+ method: :run,
53
+ args: [
54
+ port: config.metrics_port,
55
+ logger: config.logger,
56
+ metrics_prefix: :"#{config.service_name}_metrics_server",
57
+ registry: registry,
58
+ ]
59
+ )
60
+ end
61
+ end
62
+
63
+ def initialize_loggerstash(ultravisor, config, registry)
64
+ if config.logstash_server && !config.logstash_server.empty?
65
+ config.pre_run_logger.info(config.service_name) { "Configuring loggerstash to send to #{config.logstash_server}" }
66
+
67
+ ultravisor.add_child(
68
+ id: :logstash_writer,
69
+ klass: LogstashWriter,
70
+ method: :run,
71
+ args: [
72
+ server_name: config.logstash_server,
73
+ metrics_registry: registry,
74
+ logger: config.logger,
75
+ ],
76
+ access: :unsafe
77
+ )
78
+
79
+ config.logger.singleton_class.prepend(Loggerstash::Mixin)
80
+
81
+ config.logger.instance_variable_set(:@ultravisor, ultravisor)
82
+ config.logger.singleton_class.prepend(ServiceSkeleton::UltravisorLoggerstash)
83
+ end
84
+ end
85
+
86
+ def initialize_signals(ultravisor, config, service_signals, metrics_registry)
87
+ counter = metrics_registry.counter(:"#{config.service_name}_signals_handled_total", docstring: "How many of each signal have been handled", labels: %i{signal})
88
+
89
+ ultravisor.add_child(
90
+ id: :signal_manager,
91
+ klass: ServiceSkeleton::SignalManager,
92
+ method: :run,
93
+ args: [
94
+ logger: config.logger,
95
+ counter: counter,
96
+ signals: global_signals(ultravisor, config.logger) + wrap_service_signals(service_signals, ultravisor),
97
+ ],
98
+ shutdown: {
99
+ method: :shutdown,
100
+ timeout: 1,
101
+ }
102
+ )
103
+ end
104
+
105
+ def global_signals(ultravisor, logger)
106
+ # For mysterious reasons of mystery, simplecov doesn't recognise these
107
+ # procs as being called, even though there are definitely tests for
108
+ # them. So...
109
+ #:nocov:
110
+ [
111
+ [
112
+ "USR1",
113
+ ->() {
114
+ logger.level -= 1 unless logger.level == Logger::DEBUG
115
+ logger.info($0) { "Received SIGUSR1; log level is now #{Logger::SEV_LABEL[logger.level]}." }
116
+ }
117
+ ],
118
+ [
119
+ "USR2",
120
+ ->() {
121
+ logger.level += 1 unless logger.level == Logger::ERROR
122
+ logger.info($0) { "Received SIGUSR2; log level is now #{Logger::SEV_LABEL[logger.level]}." }
123
+ }
124
+ ],
125
+ [
126
+ "HUP",
127
+ ->() {
128
+ logger.reopen
129
+ logger.info($0) { "Received SIGHUP; log file handle reopened" }
130
+ }
131
+ ],
132
+ [
133
+ "QUIT",
134
+ ->() { Sigdump.dump("+") }
135
+ ],
136
+ [
137
+ "INT",
138
+ ->() {
139
+ ultravisor.shutdown(wait: false, force: !!@shutting_down)
140
+ @shutting_down = true
141
+ }
142
+ ],
143
+ [
144
+ "TERM",
145
+ ->() {
146
+ ultravisor.shutdown(wait: false, force: !!@shutting_down)
147
+ @shutting_down = true
148
+ }
149
+ ]
150
+ ]
151
+ #:nocov:
152
+ end
153
+
154
+ def wrap_service_signals(signals, ultravisor)
155
+ [].tap do |signal_list|
156
+ signals.each do |service_name, sigs|
157
+ sigs.each do |sig, proc|
158
+ wrapped_proc = ->() { ultravisor[service_name.to_sym].unsafe_instance.instance_eval(&proc) }
159
+ signal_list << [sig, wrapped_proc]
160
+ end
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module LoggingHelpers
5
+ private
6
+
7
+ def log_exception(ex, progname = nil)
8
+ #:nocov:
9
+ progname ||= "#{self.class.to_s}##{caller_locations(2, 1).first.label}"
10
+
11
+ logger.error(progname) do
12
+ explanation = if block_given?
13
+ yield
14
+ else
15
+ nil
16
+ end
17
+
18
+ (["#{explanation}#{explanation ? ": " : ""}#{ex.message} (#{ex.class})"] + ex.backtrace).join("\n ")
19
+ end
20
+ #:nocov:
21
+ end
22
+
23
+ def logloc
24
+ loc = caller_locations.first
25
+ "#{self.class}##{loc.label}"
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module MetricMethodName
5
+ def method_name(svc_name)
6
+ @name.to_s.gsub(/\A#{Regexp.quote(svc_name)}_/i, '').downcase
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "prometheus/client"
4
+
5
+ require_relative "metric_method_name"
6
+
7
+ Prometheus::Client::Metric.include(ServiceSkeleton::MetricMethodName)
8
+
9
+ module ServiceSkeleton
10
+ module MetricsMethods
11
+ def registered_metrics
12
+ @registered_metrics || []
13
+ end
14
+
15
+ def metric(metric)
16
+ @registered_metrics ||= []
17
+
18
+ @registered_metrics << metric
19
+ end
20
+
21
+ def counter(name, docstring:, labels: [], preset_labels: {})
22
+ metric(Prometheus::Client::Counter.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels))
23
+ end
24
+
25
+ def gauge(name, docstring:, labels: [], preset_labels: {})
26
+ metric(Prometheus::Client::Gauge.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels))
27
+ end
28
+
29
+ def summary(name, docstring:, labels: [], preset_labels: {})
30
+ metric(Prometheus::Client::Summary.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels))
31
+ end
32
+
33
+ def histogram(name, docstring:, labels: [], preset_labels: {}, buckets: Prometheus::Client::Histogram::DEFAULT_BUCKETS)
34
+ metric(Prometheus::Client::Histogram.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels, buckets: buckets))
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "config"
4
+ require_relative "logging_helpers"
5
+ require_relative "signal_manager"
6
+
7
+ require "frankenstein/ruby_gc_metrics"
8
+ require "frankenstein/ruby_vm_metrics"
9
+ require "frankenstein/process_metrics"
10
+ require "frankenstein/server"
11
+ require "prometheus/client/registry"
12
+ require "sigdump"
13
+ require_relative "../../ultravisor/lib/ultravisor"
14
+
15
+ module ServiceSkeleton
16
+ class Runner
17
+ include ServiceSkeleton::LoggingHelpers
18
+
19
+ def initialize(klass, env)
20
+ @config = (klass.config_class || ServiceSkeleton::Config).new(env, klass.service_name, klass.registered_variables)
21
+ @logger = @config.logger
22
+
23
+ @metrics_registry = Prometheus::Client::Registry.new
24
+
25
+ @ultravisor = ServiceSkeleton.generate(
26
+ config: @config,
27
+ metrics_registry: @metrics_registry,
28
+ service_metrics: klass.registered_metrics,
29
+ service_signal_handlers: { klass.service_name.to_sym => klass.registered_signal_handlers }
30
+ )
31
+
32
+ klass.register_ultravisor_children(@ultravisor, config: @config, metrics_registry: @metrics_registry)
33
+ end
34
+
35
+ def run
36
+ @config.pre_run_logger.info(logloc) { "Starting service #{@config.service_name}" }
37
+ @config.pre_run_logger.info(logloc) { (["Environment:"] + @config.env.map { |k, v| "#{k}=#{v.inspect}" }).join("\n ") }
38
+
39
+ @ultravisor.run
40
+ end
41
+
42
+ private
43
+
44
+ attr_reader :logger
45
+ end
46
+ end