service_skeleton 0.0.0.30.g32b8169 → 0.0.0.48.g4a40599

Sign up to get free protection for your applications and to get access to all the features.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +260 -143
  3. data/lib/service_skeleton.rb +22 -186
  4. data/lib/service_skeleton/config.rb +58 -31
  5. data/lib/service_skeleton/config_class.rb +16 -0
  6. data/lib/service_skeleton/config_variable.rb +24 -16
  7. data/lib/service_skeleton/config_variable/boolean.rb +21 -0
  8. data/lib/service_skeleton/config_variable/enum.rb +27 -0
  9. data/lib/service_skeleton/config_variable/float.rb +25 -0
  10. data/lib/service_skeleton/config_variable/integer.rb +25 -0
  11. data/lib/service_skeleton/config_variable/kv_list.rb +26 -0
  12. data/lib/service_skeleton/config_variable/path_list.rb +13 -0
  13. data/lib/service_skeleton/config_variable/string.rb +18 -0
  14. data/lib/service_skeleton/config_variable/url.rb +36 -0
  15. data/lib/service_skeleton/config_variable/yaml_file.rb +42 -0
  16. data/lib/service_skeleton/config_variables.rb +49 -82
  17. data/lib/service_skeleton/error.rb +5 -3
  18. data/lib/service_skeleton/filtering_logger.rb +2 -0
  19. data/lib/service_skeleton/generator.rb +165 -0
  20. data/lib/service_skeleton/logging_helpers.rb +5 -3
  21. data/lib/service_skeleton/metric_method_name.rb +9 -0
  22. data/lib/service_skeleton/metrics_methods.rb +28 -13
  23. data/lib/service_skeleton/runner.rb +46 -0
  24. data/lib/service_skeleton/service_name.rb +20 -0
  25. data/lib/service_skeleton/signal_manager.rb +202 -0
  26. data/lib/service_skeleton/signals_methods.rb +15 -0
  27. data/lib/service_skeleton/ultravisor_children.rb +17 -0
  28. data/lib/service_skeleton/ultravisor_loggerstash.rb +11 -0
  29. data/service_skeleton.gemspec +8 -7
  30. metadata +65 -15
  31. data/lib/service_skeleton/background_worker.rb +0 -89
  32. data/lib/service_skeleton/signal_handler.rb +0 -195
@@ -1,8 +1,13 @@
1
- require_relative "service_skeleton/config"
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "service_skeleton/config_class"
2
4
  require_relative "service_skeleton/config_variables"
5
+ require_relative "service_skeleton/generator"
3
6
  require_relative "service_skeleton/logging_helpers"
4
7
  require_relative "service_skeleton/metrics_methods"
5
- require_relative "service_skeleton/signal_handler"
8
+ require_relative "service_skeleton/service_name"
9
+ require_relative "service_skeleton/signals_methods"
10
+ require_relative "service_skeleton/ultravisor_children"
6
11
 
7
12
  require "frankenstein/ruby_gc_metrics"
8
13
  require "frankenstein/ruby_vm_metrics"
@@ -11,195 +16,26 @@ require "frankenstein/server"
11
16
  require "prometheus/client/registry"
12
17
  require "sigdump"
13
18
 
14
- class ServiceSkeleton
15
- extend ServiceSkeleton::ConfigVariables
16
-
19
+ module ServiceSkeleton
17
20
  include ServiceSkeleton::LoggingHelpers
21
+ extend ServiceSkeleton::Generator
18
22
 
19
- class Terminate < Exception; end
20
-
21
- def self.config_class(klass)
22
- @config_class = klass
23
- end
24
-
25
- def self.service_name
26
- service_name_from_class(self)
23
+ def self.included(mod)
24
+ mod.extend ServiceSkeleton::ServiceName
25
+ mod.extend ServiceSkeleton::ConfigVariables
26
+ mod.extend ServiceSkeleton::ConfigClass
27
+ mod.extend ServiceSkeleton::MetricsMethods
28
+ mod.extend ServiceSkeleton::SignalsMethods
29
+ mod.extend ServiceSkeleton::UltravisorChildren
27
30
  end
28
31
 
29
32
  attr_reader :config, :metrics, :logger
30
33
 
31
- def initialize(env)
32
- @env = env
33
- @config = (self.class.instance_variable_get(:@config_class) || ServiceSkeleton::Config).new(env, self)
34
- @logger = @config.logger
35
- @op_mutex = Mutex.new
36
-
37
- initialize_metrics
38
- initialize_signals
39
- end
40
-
41
- def start
42
- @op_mutex.synchronize { @thread = Thread.current }
43
-
44
- begin
45
- logger.info(logloc) { "Starting service #{service_name}" }
46
- logger.info(logloc) { (["Environment:"] + config.env.map { |k, v| "#{k}=#{v.inspect}" }).join("\n ") }
47
-
48
- start_metrics_server
49
- start_signal_handler
50
- run
51
- rescue ServiceSkeleton::Terminate
52
- # This one is OK
53
- rescue ServiceSkeleton::Error::InheritanceContractError
54
- # We want this one to be fatal
55
- raise
56
- rescue StandardError => ex
57
- log_exception(ex)
58
- end
59
-
60
- @thread = nil
61
- end
62
-
63
- def stop(force = false)
64
- if force
65
- #:nocov:
66
- @op_mutex.synchronize do
67
- if @thread
68
- @thread.raise(ServiceSkeleton::Terminate)
69
- end
70
- end
71
- #:nocov:
72
- else
73
- shutdown
74
- end
75
-
76
- if @metrics_server
77
- @metrics_server.shutdown
78
- @metrics_server = nil
79
- end
80
-
81
- @signal_handler.stop!
82
- end
83
-
84
- def service_name
85
- self.class.service_name
86
- end
87
-
88
- def registered_variables
89
- self.class.registered_variables
90
- end
91
-
92
- def hook_signal(spec, &blk)
93
- @signal_handler.hook_signal(spec, &blk)
94
- end
95
-
96
- private
97
-
98
- def self.service_name_from_class(klass)
99
- klass.to_s
100
- .gsub("::", "_")
101
- .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
102
- .gsub(/([a-z\d])([A-Z])/, '\1_\2')
103
- .downcase
104
- .gsub(/[^a-zA-Z0-9_]/, "_")
105
- end
106
-
107
- def run
108
- raise ServiceSkeleton::Error::InheritanceContractError, "ServiceSkeleton#run method not overridden"
109
- end
110
-
111
- def shutdown
112
- #:nocov:
113
- @op_mutex.synchronize do
114
- if @thread
115
- @thread.raise(ServiceSkeleton::Terminate)
116
- @thread.join
117
- @thread = nil
118
- end
119
- end
120
- #:nocov:
121
- end
122
-
123
- def initialize_metrics
124
- @metrics = Prometheus::Client::Registry.new
125
-
126
- Frankenstein::RubyGCMetrics.register(@metrics)
127
- Frankenstein::RubyVMMetrics.register(@metrics)
128
- Frankenstein::ProcessMetrics.register(@metrics)
129
-
130
- @metrics.singleton_class.prepend(ServiceSkeleton::MetricsMethods)
131
- @metrics.service = self
132
- end
133
-
134
- def start_metrics_server
135
- if config.metrics_port
136
- logger.info(logloc) { "Starting metrics server on port #{config.metrics_port}" }
137
-
138
- @metrics_server = Frankenstein::Server.new(
139
- port: config.metrics_port,
140
- logger: logger,
141
- metrics_prefix: :metrics_server,
142
- registry: @metrics,
143
- )
144
- @metrics_server.run
145
- end
146
- end
147
-
148
- def initialize_signals
149
- metrics.counter(:"#{self.service_name}_signals_handled_total", "How many of each type of signal have been handled")
150
- @signal_handler = ServiceSkeleton::SignalHandler.new(logger: logger, service: self, signal_counter: metrics.signals_handled_total)
151
-
152
- @signal_handler.hook_signal("USR1") do
153
- logger.level -= 1 unless logger.level == Logger::DEBUG
154
- logger.info($0) { "Received SIGUSR1; log level is now #{Logger::SEV_LABEL[logger.level]}." }
155
- end
156
-
157
- @signal_handler.hook_signal("USR2") do
158
- logger.level += 1 unless logger.level == Logger::ERROR
159
- logger.info($0) { "Received SIGUSR2; log level is now #{Logger::SEV_LABEL[logger.level]}." }
160
- end
161
-
162
- @signal_handler.hook_signal("HUP") do
163
- logger.reopen
164
- logger.info($0) { "Received SIGHUP; log file handle reopened" }
165
- end
166
-
167
- @signal_handler.hook_signal("QUIT") do
168
- Sigdump.dump("+")
169
- end
170
-
171
- @signal_handler.hook_signal("INT") do
172
- self.stop(!!@terminating)
173
- @terminating = true
174
- end
175
-
176
- @signal_handler.hook_signal("TERM") do
177
- self.stop(!!@terminating)
178
- @terminating = true
179
- end
180
- end
181
-
182
- def start_signal_handler
183
- @signal_handler.start!
184
- end
185
-
186
- @registered_variables = [
187
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_LOG_LEVEL) { "INFO" },
188
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_LOGSTASH_SERVER) { "" },
189
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_LOG_ENABLE_TIMESTAMPS) { false },
190
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_LOG_FILE) { nil },
191
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_LOG_MAX_FILE_SIZE) { 1048576 },
192
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_LOG_MAX_FILES) { 3 },
193
- ServiceSkeleton::ConfigVariable.new(:SERVICE_SKELETON_METRICS_PORT) { nil },
194
- ]
195
-
196
- def self.inherited(subclass)
197
- subclass.string(:"#{subclass.service_name.upcase}_LOG_LEVEL", default: "INFO")
198
- subclass.string(:"#{subclass.service_name.upcase}_LOGSTASH_SERVER", default: "")
199
- subclass.boolean(:"#{subclass.service_name.upcase}_LOG_ENABLE_TIMESTAMPS", default: false)
200
- subclass.string(:"#{subclass.service_name.upcase}_LOG_FILE", default: nil)
201
- subclass.integer(:"#{subclass.service_name.upcase}_LOG_MAX_FILE_SIZE", default: 1048576, range: 0..Float::INFINITY)
202
- subclass.integer(:"#{subclass.service_name.upcase}_LOG_MAX_FILES", default: 3, range: 1..Float::INFINITY)
203
- subclass.integer(:"#{subclass.service_name.upcase}_METRICS_PORT", default: nil, range: 1..65535)
34
+ def initialize(*_, metrics:, config:)
35
+ @metrics = metrics
36
+ @config = config
37
+ @logger = @config.logger
204
38
  end
205
39
  end
40
+
41
+ require_relative "service_skeleton/runner"
@@ -1,47 +1,57 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "to_regexp"
2
4
 
3
5
  require_relative "./filtering_logger"
4
6
 
5
7
  require "loggerstash"
6
8
 
7
- class ServiceSkeleton
9
+ module ServiceSkeleton
8
10
  class Config
9
- attr_reader :logger, :env
11
+ attr_reader :logger, :env, :service_name
10
12
 
11
- def initialize(env, svc)
12
- @svc = svc
13
+ def initialize(env, service_name, variables)
14
+ @service_name = service_name
13
15
 
14
- parse_registered_variables(env)
16
+ # Parsing variables will redact the environment, so we want to take a
17
+ # private unredacted copy before that happens for #[] lookup in the
18
+ # future.
15
19
  @env = env.to_hash.dup.freeze
20
+
21
+ parse_variables(internal_variables + variables, env)
22
+
23
+ # Sadly, we can't setup the logger until we know *how* to setup the
24
+ # logger, which requires parsing config variables
16
25
  setup_logger
17
26
  end
18
27
 
19
28
  def [](k)
20
- @env[k]
29
+ @env[k].dup
21
30
  end
22
31
 
23
32
  private
24
33
 
25
- def parse_registered_variables(env)
26
- @svc.registered_variables.each do |var|
27
- val = var.value(env)
34
+ def parse_variables(variables, env)
35
+ variables.map do |var|
36
+ var[:class].new(var[:name], env, **var[:opts])
37
+ end.each do |var|
38
+ val = var.value
39
+ method_name = var.method_name(@service_name).to_sym
28
40
 
29
- define_singleton_method(var.method_name(@svc.service_name)) do
41
+ define_singleton_method(method_name) do
30
42
  val
31
43
  end
32
44
 
33
- define_singleton_method(var.method_name(@svc.service_name) + "=") do |v|
34
- val = v
45
+ define_singleton_method(:"#{method_name}=") do |new_value|
46
+ val = new_value
35
47
  end
36
-
37
- if var.sensitive?
48
+ end.each do |var|
49
+ if var.redact?(env)
38
50
  if env.object_id != ENV.object_id
39
51
  raise ServiceSkeleton::Error::CannotSanitizeEnvironmentError,
40
- "Attempted to sanitize sensitive variable #{var.name}, but was not passed the ENV object"
41
- end
42
-
43
- var.env_keys(env).each do |k|
44
- env[k] = "*SENSITIVE*"
52
+ "Attempted to sanitize sensitive variable #{var.name}, but we're not operating on the process' environment"
53
+ else
54
+ var.redact!(env)
45
55
  end
46
56
  end
47
57
  end
@@ -56,30 +66,35 @@ class ServiceSkeleton
56
66
 
57
67
  @logger = Logger.new(log_file || $stderr, shift_age, shift_size)
58
68
 
59
- if self.logstash_server && !self.logstash_server.empty?
60
- loggerstash = Loggerstash.new(logstash_server: logstash_server, logger: @logger)
61
- loggerstash.metrics_registry = @svc.metrics
62
- loggerstash.attach(@logger)
63
- end
64
-
65
- thread_id_map = {}
66
69
  if Thread.main
67
- thread_id_map[Thread.main.object_id] = 0
70
+ Thread.main[:thread_map_number] = 0
68
71
  else
69
72
  #:nocov:
70
- thread_id_map[Thread.current.object_id] = 0
73
+ Thread.current[:thread_map_number] = 0
71
74
  #:nocov:
72
75
  end
73
76
 
77
+ thread_map_mutex = Mutex.new
78
+
74
79
  @logger.formatter = ->(s, t, p, m) do
75
- th_n = thread_id_map[Thread.current.object_id] || (thread_id_map[Thread.current.object_id] = thread_id_map.length)
80
+ th_n = if Thread.current.name
81
+ #:nocov:
82
+ Thread.current.name
83
+ #:nocov:
84
+ else
85
+ thread_map_mutex.synchronize do
86
+ Thread.current[:thread_map_number] ||= begin
87
+ Thread.list.select { |th| th[:thread_map_number] }.length
88
+ end
89
+ end
90
+ end
76
91
 
77
92
  ts = log_enable_timestamps ? "#{t.utc.strftime("%FT%T.%NZ")} " : ""
78
- "#{ts}##{th_n} #{s[0]} [#{p}] #{m}\n"
93
+ "#{ts}#{$$}##{th_n} #{s[0]} [#{p}] #{m}\n"
79
94
  end
80
95
 
81
96
  @logger.filters = []
82
- @env.fetch("#{@svc.service_name.upcase}_LOG_LEVEL", "INFO").split(/\s*,\s*/).each do |spec|
97
+ @env.fetch("#{@service_name.upcase}_LOG_LEVEL", "INFO").split(/\s*,\s*/).each do |spec|
83
98
  if spec.index("=")
84
99
  # "Your developers were so preoccupied with whether or not they
85
100
  # could, they didn't stop to think if they should."
@@ -102,5 +117,17 @@ class ServiceSkeleton
102
117
  end
103
118
  end
104
119
  end
120
+
121
+ def internal_variables
122
+ [
123
+ { name: "#{@service_name.upcase}_LOG_LEVEL", class: ConfigVariable::String, opts: { default: "INFO" } },
124
+ { name: "#{@service_name.upcase}_LOG_ENABLE_TIMESTAMPS", class: ConfigVariable::Boolean, opts: { default: false } },
125
+ { name: "#{@service_name.upcase}_LOG_FILE", class: ConfigVariable::String, opts: { default: nil } },
126
+ { name: "#{@service_name.upcase}_LOG_MAX_FILE_SIZE", class: ConfigVariable::Integer, opts: { default: 1048576, range: 0..Float::INFINITY } },
127
+ { name: "#{@service_name.upcase}_LOG_MAX_FILES", class: ConfigVariable::Integer, opts: { default: 3, range: 1..Float::INFINITY } },
128
+ { name: "#{@service_name.upcase}_LOGSTASH_SERVER", class: ConfigVariable::String, opts: { default: "" } },
129
+ { name: "#{@service_name.upcase}_METRICS_PORT", class: ConfigVariable::Integer, opts: { default: nil, range: 1..65535 } },
130
+ ]
131
+ end
105
132
  end
106
133
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module ConfigClass
5
+ Undefined = Module.new
6
+ private_constant :Undefined
7
+
8
+ def config_class(klass = Undefined)
9
+ unless klass == Undefined
10
+ @config_class = klass
11
+ end
12
+
13
+ @config_class
14
+ end
15
+ end
16
+ end
@@ -1,35 +1,43 @@
1
- class ServiceSkeleton
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
2
4
  class ConfigVariable
3
- attr_reader :name, :key_pattern
5
+ attr_reader :name, :value
4
6
 
5
- def initialize(name, **opts, &blk)
7
+ def initialize(name, env, **opts, &blk)
6
8
  @name = name
7
9
  @opts = opts
8
10
  @blk = blk
11
+
12
+ @value = pluck_value(env)
9
13
  end
10
14
 
11
15
  def method_name(svc_name)
12
- name.to_s.gsub(/\A#{Regexp.quote(svc_name)}_/i, '').downcase
16
+ @name.to_s.gsub(/\A#{Regexp.quote(svc_name)}_/i, '').downcase
13
17
  end
14
18
 
15
- def sensitive?
16
- !!@opts[:sensitive]
19
+ def redact?(env)
20
+ @opts[:sensitive]
17
21
  end
18
22
 
19
- def value(env)
20
- if @opts[:key_pattern]
21
- matches = env.select { |k, _| @opts[:key_pattern] === k.to_s }
22
- @blk.call(matches)
23
- else
24
- @blk.call(env[@name.to_s])
23
+ def redact!(env)
24
+ if @opts[:sensitive]
25
+ env[@name.to_s] = "*SENSITIVE*" if env.has_key?(@name.to_s)
25
26
  end
26
27
  end
27
28
 
28
- def env_keys(env)
29
- if @opts[:key_pattern]
30
- env.keys.select { |k| @opts[:key_pattern] === k.to_s }
29
+ private
30
+
31
+ def maybe_default(env)
32
+ if env.has_key?(@name.to_s)
33
+ yield
31
34
  else
32
- env.keys.include?(@name.to_s) ? [@name.to_s] : []
35
+ if @opts.has_key?(:default)
36
+ @opts[:default]
37
+ else
38
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
39
+ "Value for required environment variable #{@name} not specified"
40
+ end
33
41
  end
34
42
  end
35
43
  end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "service_skeleton/config_variable"
4
+
5
+ class ServiceSkeleton::ConfigVariable::Boolean < ServiceSkeleton::ConfigVariable
6
+ private
7
+
8
+ def pluck_value(env)
9
+ maybe_default(env) do
10
+ case env[@name.to_s]
11
+ when /\A(no|n|off|0|false)\z/i
12
+ false
13
+ when /\A(yes|y|on|1|true)\z/i
14
+ true
15
+ else
16
+ raise ServiceSkeleton::Error::InvalidEnvironmentError,
17
+ "Value #{env[@name.to_s].inspect} for environment variable #{@name} is not a valid boolean value"
18
+ end
19
+ end
20
+ end
21
+ end