service_skeleton 0.0.0.34.g4f6fdb0 → 0.0.0.49.g47046b9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +260 -145
  3. data/lib/service_skeleton.rb +22 -186
  4. data/lib/service_skeleton/config.rb +57 -30
  5. data/lib/service_skeleton/config_class.rb +16 -0
  6. data/lib/service_skeleton/config_variable.rb +24 -16
  7. data/lib/service_skeleton/config_variable/boolean.rb +21 -0
  8. data/lib/service_skeleton/config_variable/enum.rb +27 -0
  9. data/lib/service_skeleton/config_variable/float.rb +25 -0
  10. data/lib/service_skeleton/config_variable/integer.rb +25 -0
  11. data/lib/service_skeleton/config_variable/kv_list.rb +26 -0
  12. data/lib/service_skeleton/config_variable/path_list.rb +13 -0
  13. data/lib/service_skeleton/config_variable/string.rb +18 -0
  14. data/lib/service_skeleton/config_variable/url.rb +36 -0
  15. data/lib/service_skeleton/config_variable/yaml_file.rb +42 -0
  16. data/lib/service_skeleton/config_variables.rb +49 -82
  17. data/lib/service_skeleton/error.rb +5 -3
  18. data/lib/service_skeleton/filtering_logger.rb +2 -0
  19. data/lib/service_skeleton/generator.rb +165 -0
  20. data/lib/service_skeleton/logging_helpers.rb +5 -3
  21. data/lib/service_skeleton/metric_method_name.rb +9 -0
  22. data/lib/service_skeleton/metrics_methods.rb +28 -13
  23. data/lib/service_skeleton/runner.rb +46 -0
  24. data/lib/service_skeleton/service_name.rb +20 -0
  25. data/lib/service_skeleton/signal_manager.rb +202 -0
  26. data/lib/service_skeleton/signals_methods.rb +15 -0
  27. data/lib/service_skeleton/ultravisor_children.rb +17 -0
  28. data/lib/service_skeleton/ultravisor_loggerstash.rb +11 -0
  29. data/service_skeleton.gemspec +8 -7
  30. metadata +65 -15
  31. data/lib/service_skeleton/background_worker.rb +0 -94
  32. data/lib/service_skeleton/signal_handler.rb +0 -195
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::Enum < ServiceSkeleton::ConfigVariable
6
+ private
7
+
8
+ def pluck_value(env)
9
+ maybe_default(env) do
10
+ v = env[@name.to_s]
11
+
12
+ if @opts[:values].is_a?(Array)
13
+ unless @opts[:values].include?(v)
14
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
15
+ "Invalid value for #{@name}; must be one of #{@opts[:values].join(", ")}"
16
+ end
17
+ v
18
+ elsif @opts[:values].is_a?(Hash)
19
+ unless @opts[:values].keys.include?(v)
20
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
21
+ "Invalid value for #{@name}; must be one of #{@opts[:values].keys.join(", ")}"
22
+ end
23
+ @opts[:values][v]
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::Float < ServiceSkeleton::ConfigVariable
6
+ private
7
+
8
+ def pluck_value(env)
9
+ maybe_default(env) do
10
+ value = env[@name.to_s]
11
+
12
+ if value =~ /\A-?\d+.?\d*\z/
13
+ value.to_f.tap do |f|
14
+ unless @opts[:range].include?(f)
15
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
16
+ "Value #{f} for environment variable #{@name} is out of the valid range (must be between #{@opts[:range].first} and #{@opts[:range].last} inclusive)"
17
+ end
18
+ end
19
+ else
20
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
21
+ "Value #{value.inspect} for environment variable #{@name} is not a valid numeric value"
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::Integer < ServiceSkeleton::ConfigVariable
6
+ private
7
+
8
+ def pluck_value(env)
9
+ maybe_default(env) do
10
+ value = env[@name.to_s]
11
+
12
+ if value =~ /\A-?\d+\z/
13
+ value.to_i.tap do |i|
14
+ unless @opts[:range].include?(i)
15
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
16
+ "Value #{i} for environment variable #{@name} is out of the valid range (must be between #{@opts[:range].first} and #{@opts[:range].last} inclusive)"
17
+ end
18
+ end
19
+ else
20
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
21
+ "Value #{value.inspect} for environment variable #{@name} is not a valid integer value"
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::KVList < ServiceSkeleton::ConfigVariable
6
+ def redact!(env)
7
+ env.keys.each { |k| env[k] = "*SENSITIVE*" if k =~ @opts[:key_pattern] }
8
+ end
9
+
10
+ private
11
+
12
+ def pluck_value(env)
13
+ matches = env.select { |k, _| k.to_s =~ @opts[:key_pattern] }
14
+
15
+ if matches.empty?
16
+ if @opts.has_key?(:default)
17
+ @opts[:default]
18
+ else
19
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
20
+ "no keys for key-value list #{@name} specified"
21
+ end
22
+ else
23
+ matches.transform_keys { |k| @opts[:key_pattern].match(k.to_s)[1].to_sym }
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::PathList < ServiceSkeleton::ConfigVariable
6
+ private
7
+
8
+ def pluck_value(env)
9
+ maybe_default(env) do
10
+ env[@name.to_s].split(":")
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::String < ServiceSkeleton::ConfigVariable
6
+ private
7
+
8
+ def pluck_value(env)
9
+ maybe_default(env) do
10
+ env[@name.to_s].tap do |s|
11
+ if @opts[:match] && s !~ @opts[:match]
12
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
13
+ "Value for #{@name} must match #{@opts[:match]}"
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::URL < ServiceSkeleton::ConfigVariable
6
+ def redact?(env)
7
+ !!(env.has_key?(@name.to_s) && (@opts[:sensitive] || URI(env[@name.to_s] || "").password))
8
+ end
9
+
10
+ def redact!(env)
11
+ if env.has_key?(@name.to_s)
12
+ super
13
+ uri = URI(env[@name.to_s])
14
+ if uri.password
15
+ uri.password = "*REDACTED*"
16
+ env[@name.to_s] = uri.to_s
17
+ end
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ def pluck_value(env)
24
+ maybe_default(env) do
25
+ begin
26
+ v = env[@name.to_s]
27
+ URI(v)
28
+ rescue URI::InvalidURIError
29
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
30
+ "Value for #{@name} (#{v}) does not appear to be a valid URL"
31
+ end
32
+
33
+ v
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "yaml"
4
+
5
+ require "service_skeleton/config_variable"
6
+
7
+ class ServiceSkeleton::ConfigVariable::YamlFile < ServiceSkeleton::ConfigVariable
8
+ def redact!(env)
9
+ if env.has_key?(@name.to_s)
10
+ if File.world_readable?(env[@name.to_s])
11
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
12
+ "Sensitive file #{env[@name.to_s]} is world-readable!"
13
+ end
14
+
15
+ super
16
+ end
17
+ end
18
+
19
+ private
20
+
21
+ def pluck_value(env)
22
+ maybe_default(env) do
23
+ begin
24
+ val = YAML.safe_load(File.read(env[@name.to_s]))
25
+ if @opts[:klass]
26
+ val = @opts[:klass].new(val)
27
+ end
28
+
29
+ val
30
+ rescue Errno::ENOENT
31
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
32
+ "YAML file #{env[@name.to_s]} does not exist"
33
+ rescue Errno::EPERM
34
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
35
+ "Do not have permission to read YAML file #{env[@name.to_s]}"
36
+ rescue Psych::SyntaxError => ex
37
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
38
+ "Invalid YAML syntax: #{ex.message}"
39
+ end
40
+ end
41
+ end
42
+ end
@@ -1,112 +1,79 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative "./error"
2
4
 
3
5
  require "service_skeleton/config_variable"
4
6
 
5
- class ServiceSkeleton
7
+ module ServiceSkeleton
6
8
  module ConfigVariables
7
- attr_reader :registered_variables
8
-
9
- UNDEFINED = Class.new
9
+ UNDEFINED = Module.new
10
10
  private_constant :UNDEFINED
11
11
 
12
- def register_variable(name, **opts, &callback)
12
+ def registered_variables
13
13
  @registered_variables ||= []
14
-
15
- @registered_variables << ServiceSkeleton::ConfigVariable.new(name, **opts, &callback)
16
14
  end
17
15
 
18
- def string(var_name, default: UNDEFINED, sensitive: false)
19
- register_variable(var_name, sensitive: sensitive) do |value|
20
- maybe_default(value, default, var_name) do
21
- value
22
- end
16
+ def register_variable(name, klass, **opts)
17
+ if opts[:default] == UNDEFINED
18
+ opts.delete(:default)
23
19
  end
20
+
21
+ registered_variables << {
22
+ name: name,
23
+ class: klass,
24
+ opts: opts,
25
+ }
24
26
  end
25
27
 
26
28
  def boolean(var_name, default: UNDEFINED, sensitive: false)
27
- register_variable(var_name, sensitive: sensitive) do |value|
28
- maybe_default(value, default, var_name) do
29
- case value
30
- when /\A(no|n|off|0|false)\z/i
31
- false
32
- when /\A(yes|y|on|1|true)\z/i
33
- true
34
- else
35
- raise ServiceSkeleton::Error::InvalidEnvironmentError,
36
- "Value #{value.inspect} for environment variable #{var_name} is not a valid boolean value"
37
- end
38
- end
39
- end
29
+ register_variable(var_name, ConfigVariable::Boolean, default: default, sensitive: sensitive)
40
30
  end
41
31
 
42
- def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
43
- register_variable(var_name, sensitive: sensitive) do |value|
44
- maybe_default(value, default, var_name) do
45
- if value =~ /\A-?\d+\z/
46
- value.to_i.tap do |i|
47
- unless range.include?(i)
48
- raise ServiceSkeleton::Error::InvalidEnvironmentError,
49
- "Value #{i} for environment variable #{var_name} is out of the valid range (must be between #{range.first} and #{range.last} inclusive)"
50
- end
51
- end
52
- else
53
- raise ServiceSkeleton::Error::InvalidEnvironmentError,
54
- "Value #{value.inspect} for environment variable #{var_name} is not a valid integer value"
55
- end
56
- end
32
+ def enum(var_name, values:, default: UNDEFINED, sensitive: false)
33
+ unless values.is_a?(Hash) || values.is_a?(Array)
34
+ raise ArgumentError,
35
+ "values option to enum must be a hash or array"
57
36
  end
37
+
38
+ register_variable(var_name, ConfigVariable::Enum, default: default, sensitive: sensitive, values: values)
58
39
  end
59
40
 
60
41
  def float(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
61
- register_variable(var_name, sensitive: sensitive) do |value|
62
- maybe_default(value, default, var_name) do
63
- if value =~ /\A-?\d+.?\d*\z/
64
- value.to_f.tap do |i|
65
- unless range.include?(i)
66
- raise ServiceSkeleton::Error::InvalidEnvironmentError,
67
- "Value #{i} for environment variable #{var_name} is out of the valid range (must be between #{range.first} and #{range.last} inclusive)"
68
- end
69
- end
70
- else
71
- raise ServiceSkeleton::Error::InvalidEnvironmentError,
72
- "Value #{value.inspect} for environment variable #{var_name} is not a valid numeric value"
73
- end
74
- end
75
- end
42
+ register_variable(var_name, ConfigVariable::Float, default: default, sensitive: sensitive, range: range)
43
+ end
44
+
45
+ def integer(var_name, default: UNDEFINED, sensitive: false, range: -Float::INFINITY..Float::INFINITY)
46
+ register_variable(var_name, ConfigVariable::Integer, default: default, sensitive: sensitive, range: range)
47
+ end
48
+
49
+ def kv_list(var_name, default: UNDEFINED, sensitive: false, key_pattern: /\A#{var_name}_(.*)\z/)
50
+ register_variable(var_name, ConfigVariable::KVList, default: default, sensitive: sensitive, key_pattern: key_pattern)
76
51
  end
77
52
 
78
53
  def path_list(var_name, default: UNDEFINED, sensitive: false)
79
- register_variable(var_name, sensitive: sensitive) do |value|
80
- maybe_default(value, default, var_name) do
81
- value.split(":")
82
- end
83
- end
54
+ register_variable(var_name, ConfigVariable::PathList, default: default, sensitive: sensitive)
84
55
  end
85
56
 
86
- def kv_list(var_name, default: UNDEFINED, sensitive: false, key_pattern: nil)
87
- key_pattern ||= /\A#{var_name}_(.*)\z/
88
- register_variable(var_name, sensitive: sensitive, key_pattern: key_pattern) do |matches|
89
- maybe_default(matches, default, var_name) do
90
- matches.transform_keys do |k|
91
- key_pattern.match(k)[1].to_sym
92
- end
93
- end
94
- end
57
+ def string(var_name, default: UNDEFINED, sensitive: false, match: nil)
58
+ register_variable(var_name, ConfigVariable::String, default: default, sensitive: sensitive, match: match)
95
59
  end
96
60
 
97
- private
98
-
99
- def maybe_default(value, default, var_name)
100
- if value.nil? || value == {}
101
- if default == UNDEFINED
102
- raise ServiceSkeleton::Error::InvalidEnvironmentError,
103
- "Value for required environment variable #{var_name} not specified"
104
- else
105
- default
106
- end
107
- else
108
- yield
109
- end
61
+ def url(var_name, default: UNDEFINED, sensitive: false)
62
+ register_variable(var_name, ConfigVariable::URL, default: default, sensitive: sensitive)
63
+ end
64
+
65
+ def yaml_file(var_name, default: UNDEFINED, sensitive: false, klass: nil)
66
+ register_variable(var_name, ConfigVariable::YamlFile, default: default, sensitive: sensitive, klass: klass)
110
67
  end
111
68
  end
112
69
  end
70
+
71
+ require_relative "config_variable/boolean"
72
+ require_relative "config_variable/enum"
73
+ require_relative "config_variable/float"
74
+ require_relative "config_variable/integer"
75
+ require_relative "config_variable/kv_list"
76
+ require_relative "config_variable/path_list"
77
+ require_relative "config_variable/string"
78
+ require_relative "config_variable/url"
79
+ require_relative "config_variable/yaml_file"
@@ -1,8 +1,10 @@
1
- class ServiceSkeleton
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
2
4
  class Error < StandardError
3
- class InvalidEnvironmentError < Error; end
4
5
  class CannotSanitizeEnvironmentError < Error; end
5
- class InheritanceContractError < Error; end
6
+ class InvalidEnvironmentError < Error; end
6
7
  class InvalidMetricNameError < Error; end
8
+ class InvalidServiceClassError < Error; end
7
9
  end
8
10
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'logger'
2
4
 
3
5
  module FilteringLogger
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "config"
4
+ require_relative "signal_manager"
5
+ require_relative "ultravisor_loggerstash"
6
+
7
+ require "frankenstein/ruby_gc_metrics"
8
+ require "frankenstein/ruby_vm_metrics"
9
+ require "frankenstein/process_metrics"
10
+ require "frankenstein/server"
11
+ require "prometheus/client/registry"
12
+ require "sigdump"
13
+ require "ultravisor"
14
+
15
+ module ServiceSkeleton
16
+ module Generator
17
+ def generate(config:, metrics_registry:, service_metrics:, service_signal_handlers:)
18
+ Ultravisor.new(logger: config.logger).tap do |ultravisor|
19
+ initialize_metrics(ultravisor, config, metrics_registry, service_metrics)
20
+ initialize_loggerstash(ultravisor, config, metrics_registry)
21
+ initialize_signals(ultravisor, config, service_signal_handlers, metrics_registry)
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def initialize_metrics(ultravisor, config, registry, metrics)
28
+ Frankenstein::RubyGCMetrics.register(registry)
29
+ Frankenstein::RubyVMMetrics.register(registry)
30
+ Frankenstein::ProcessMetrics.register(registry)
31
+
32
+ metrics.each do |m|
33
+ registry.register(m)
34
+
35
+ method_name = m.method_name(config.service_name)
36
+
37
+ if registry.singleton_class.method_defined?(method_name)
38
+ raise ServiceSkeleton::Error::InvalidMetricNameError,
39
+ "Metric method #{method_name} is already defined"
40
+ end
41
+
42
+ registry.define_singleton_method(method_name) do
43
+ m
44
+ end
45
+ end
46
+
47
+ if config.metrics_port
48
+ config.logger.info(config.service_name) { "Starting metrics server on port #{config.metrics_port}" }
49
+ ultravisor.add_child(
50
+ id: :metrics_server,
51
+ klass: Frankenstein::Server,
52
+ method: :run,
53
+ args: [
54
+ port: config.metrics_port,
55
+ logger: config.logger,
56
+ metrics_prefix: :"#{config.service_name}_metrics_server",
57
+ registry: registry,
58
+ ]
59
+ )
60
+ end
61
+ end
62
+
63
+ def initialize_loggerstash(ultravisor, config, registry)
64
+ if config.logstash_server && !config.logstash_server.empty?
65
+ config.logger.info(config.service_name) { "Configuring loggerstash to send to #{config.logstash_server}" }
66
+
67
+ ultravisor.add_child(
68
+ id: :logstash_writer,
69
+ klass: LogstashWriter,
70
+ method: :run,
71
+ args: [
72
+ server_name: config.logstash_server,
73
+ metrics_registry: registry,
74
+ logger: config.logger,
75
+ ],
76
+ access: :unsafe
77
+ )
78
+
79
+ config.logger.singleton_class.prepend(Loggerstash::Mixin)
80
+
81
+ config.logger.instance_variable_set(:@ultravisor, ultravisor)
82
+ config.logger.singleton_class.prepend(ServiceSkeleton::UltravisorLoggerstash)
83
+ end
84
+ end
85
+
86
+ def initialize_signals(ultravisor, config, service_signals, metrics_registry)
87
+ counter = metrics_registry.counter(:"#{config.service_name}_signals_handled_total", docstring: "How many of each signal have been handled", labels: %i{signal})
88
+
89
+ ultravisor.add_child(
90
+ id: :signal_manager,
91
+ klass: ServiceSkeleton::SignalManager,
92
+ method: :run,
93
+ args: [
94
+ logger: config.logger,
95
+ counter: counter,
96
+ signals: global_signals(ultravisor, config.logger) + wrap_service_signals(service_signals, ultravisor),
97
+ ],
98
+ shutdown: {
99
+ method: :shutdown,
100
+ timeout: 1,
101
+ }
102
+ )
103
+ end
104
+
105
+ def global_signals(ultravisor, logger)
106
+ # For mysterious reasons of mystery, simplecov doesn't recognise these
107
+ # procs as being called, even though there are definitely tests for
108
+ # them. So...
109
+ #:nocov:
110
+ [
111
+ [
112
+ "USR1",
113
+ ->() {
114
+ logger.level -= 1 unless logger.level == Logger::DEBUG
115
+ logger.info($0) { "Received SIGUSR1; log level is now #{Logger::SEV_LABEL[logger.level]}." }
116
+ }
117
+ ],
118
+ [
119
+ "USR2",
120
+ ->() {
121
+ logger.level += 1 unless logger.level == Logger::ERROR
122
+ logger.info($0) { "Received SIGUSR2; log level is now #{Logger::SEV_LABEL[logger.level]}." }
123
+ }
124
+ ],
125
+ [
126
+ "HUP",
127
+ ->() {
128
+ logger.reopen
129
+ logger.info($0) { "Received SIGHUP; log file handle reopened" }
130
+ }
131
+ ],
132
+ [
133
+ "QUIT",
134
+ ->() { Sigdump.dump("+") }
135
+ ],
136
+ [
137
+ "INT",
138
+ ->() {
139
+ ultravisor.shutdown(wait: false, force: !!@shutting_down)
140
+ @shutting_down = true
141
+ }
142
+ ],
143
+ [
144
+ "TERM",
145
+ ->() {
146
+ ultravisor.shutdown(wait: false, force: !!@shutting_down)
147
+ @shutting_down = true
148
+ }
149
+ ]
150
+ ]
151
+ #:nocov:
152
+ end
153
+
154
+ def wrap_service_signals(signals, ultravisor)
155
+ [].tap do |signal_list|
156
+ signals.each do |service_name, sigs|
157
+ sigs.each do |sig, proc|
158
+ wrapped_proc = ->() { ultravisor[service_name.to_sym].unsafe_instance.instance_eval(&proc) }
159
+ signal_list << [sig, wrapped_proc]
160
+ end
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end