service_skeleton 0.0.0.30.g32b8169 → 0.0.0.48.g4a40599

Sign up to get free protection for your applications and to get access to all the features.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +260 -143
  3. data/lib/service_skeleton.rb +22 -186
  4. data/lib/service_skeleton/config.rb +58 -31
  5. data/lib/service_skeleton/config_class.rb +16 -0
  6. data/lib/service_skeleton/config_variable.rb +24 -16
  7. data/lib/service_skeleton/config_variable/boolean.rb +21 -0
  8. data/lib/service_skeleton/config_variable/enum.rb +27 -0
  9. data/lib/service_skeleton/config_variable/float.rb +25 -0
  10. data/lib/service_skeleton/config_variable/integer.rb +25 -0
  11. data/lib/service_skeleton/config_variable/kv_list.rb +26 -0
  12. data/lib/service_skeleton/config_variable/path_list.rb +13 -0
  13. data/lib/service_skeleton/config_variable/string.rb +18 -0
  14. data/lib/service_skeleton/config_variable/url.rb +36 -0
  15. data/lib/service_skeleton/config_variable/yaml_file.rb +42 -0
  16. data/lib/service_skeleton/config_variables.rb +49 -82
  17. data/lib/service_skeleton/error.rb +5 -3
  18. data/lib/service_skeleton/filtering_logger.rb +2 -0
  19. data/lib/service_skeleton/generator.rb +165 -0
  20. data/lib/service_skeleton/logging_helpers.rb +5 -3
  21. data/lib/service_skeleton/metric_method_name.rb +9 -0
  22. data/lib/service_skeleton/metrics_methods.rb +28 -13
  23. data/lib/service_skeleton/runner.rb +46 -0
  24. data/lib/service_skeleton/service_name.rb +20 -0
  25. data/lib/service_skeleton/signal_manager.rb +202 -0
  26. data/lib/service_skeleton/signals_methods.rb +15 -0
  27. data/lib/service_skeleton/ultravisor_children.rb +17 -0
  28. data/lib/service_skeleton/ultravisor_loggerstash.rb +11 -0
  29. data/service_skeleton.gemspec +8 -7
  30. metadata +65 -15
  31. data/lib/service_skeleton/background_worker.rb +0 -89
  32. data/lib/service_skeleton/signal_handler.rb +0 -195
@@ -1,21 +1,23 @@
1
- class ServiceSkeleton
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
2
4
  module LoggingHelpers
3
5
  private
4
6
 
5
7
  def log_exception(ex, progname = nil)
8
+ #:nocov:
6
9
  progname ||= "#{self.class.to_s}##{caller_locations(2, 1).first.label}"
7
10
 
8
11
  logger.error(progname) do
9
- #:nocov:
10
12
  explanation = if block_given?
11
13
  yield
12
14
  else
13
15
  nil
14
16
  end
15
- #:nocov:
16
17
 
17
18
  (["#{explanation}#{explanation ? ": " : ""}#{ex.message} (#{ex.class})"] + ex.backtrace).join("\n ")
18
19
  end
20
+ #:nocov:
19
21
  end
20
22
 
21
23
  def logloc
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module MetricMethodName
5
+ def method_name(svc_name)
6
+ @name.to_s.gsub(/\A#{Regexp.quote(svc_name)}_/i, '').downcase
7
+ end
8
+ end
9
+ end
@@ -1,22 +1,37 @@
1
- class ServiceSkeleton
1
+ # frozen_string_literal: true
2
+
3
+ require "prometheus/client"
4
+
5
+ require_relative "metric_method_name"
6
+
7
+ Prometheus::Client::Metric.include(ServiceSkeleton::MetricMethodName)
8
+
9
+ module ServiceSkeleton
2
10
  module MetricsMethods
3
- def service=(svc)
4
- @service = svc
11
+ def registered_metrics
12
+ @registered_metrics || []
13
+ end
14
+
15
+ def metric(metric)
16
+ @registered_metrics ||= []
17
+
18
+ @registered_metrics << metric
5
19
  end
6
20
 
7
- def register(metric)
8
- method_name = metric.name.to_s.gsub(/\A#{Regexp.quote(@service.service_name)}_/, '').to_sym
21
+ def counter(name, docstring:, labels: [], preset_labels: {})
22
+ metric(Prometheus::Client::Counter.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels))
23
+ end
9
24
 
10
- if self.class.method_defined?(method_name)
11
- raise ServiceSkeleton::Error::InvalidMetricNameError,
12
- "There is already a method named #{method_name} on ##metrics, so you can't have a metric named #{metric.name}"
13
- end
25
+ def gauge(name, docstring:, labels: [], preset_labels: {})
26
+ metric(Prometheus::Client::Gauge.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels))
27
+ end
14
28
 
15
- define_singleton_method(method_name) do
16
- metric
17
- end
29
+ def summary(name, docstring:, labels: [], preset_labels: {})
30
+ metric(Prometheus::Client::Summary.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels))
31
+ end
18
32
 
19
- super
33
+ def histogram(name, docstring:, labels: [], preset_labels: {}, buckets: Prometheus::Client::Histogram::DEFAULT_BUCKETS)
34
+ metric(Prometheus::Client::Histogram.new(name, docstring: docstring, labels: labels, preset_labels: preset_labels, buckets: buckets))
20
35
  end
21
36
  end
22
37
  end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "config"
4
+ require_relative "logging_helpers"
5
+ require_relative "signal_manager"
6
+
7
+ require "frankenstein/ruby_gc_metrics"
8
+ require "frankenstein/ruby_vm_metrics"
9
+ require "frankenstein/process_metrics"
10
+ require "frankenstein/server"
11
+ require "prometheus/client/registry"
12
+ require "sigdump"
13
+ require "ultravisor"
14
+
15
+ module ServiceSkeleton
16
+ class Runner
17
+ include ServiceSkeleton::LoggingHelpers
18
+
19
+ def initialize(klass, env)
20
+ @config = (klass.config_class || ServiceSkeleton::Config).new(env, klass.service_name, klass.registered_variables)
21
+ @logger = @config.logger
22
+
23
+ @metrics_registry = Prometheus::Client::Registry.new
24
+
25
+ @ultravisor = ServiceSkeleton.generate(
26
+ config: @config,
27
+ metrics_registry: @metrics_registry,
28
+ service_metrics: klass.registered_metrics,
29
+ service_signal_handlers: { klass.service_name.to_sym => klass.registered_signal_handlers }
30
+ )
31
+
32
+ klass.register_ultravisor_children(@ultravisor, config: @config, metrics_registry: @metrics_registry)
33
+ end
34
+
35
+ def run
36
+ logger.info(logloc) { "Starting service #{@config.service_name}" }
37
+ logger.info(logloc) { (["Environment:"] + @config.env.map { |k, v| "#{k}=#{v.inspect}" }).join("\n ") }
38
+
39
+ @ultravisor.run
40
+ end
41
+
42
+ private
43
+
44
+ attr_reader :logger
45
+ end
46
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module ServiceName
5
+ def service_name
6
+ service_name_from_class(self)
7
+ end
8
+
9
+ private
10
+
11
+ def service_name_from_class(klass)
12
+ klass.to_s
13
+ .gsub("::", "_")
14
+ .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
15
+ .gsub(/([a-z\d])([A-Z])/, '\1_\2')
16
+ .downcase
17
+ .gsub(/[^a-zA-Z0-9_]/, "_")
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,202 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "./logging_helpers"
4
+
5
+ module ServiceSkeleton
6
+ # Manage signals in a sane and safe manner.
7
+ #
8
+ # Signal handling is a shit of a thing. The code that runs when a signal is
9
+ # triggered can't use mutexes (which are used in all sorts of places you
10
+ # might not expect, like Logger!) or anything else that might block. This
11
+ # greatly constrains what you can do inside a signal handler, so the standard
12
+ # approach is to stuff a character down a pipe, and then have the *real*
13
+ # signal handling run later.
14
+ #
15
+ # Also, there's always the (slim) possibility that something else might have
16
+ # hooked into a signal we want to receive. Because only a single signal
17
+ # handler can be active for a given signal at a time, we need to "chain" the
18
+ # existing handler, by calling the previous signal handler from our signal
19
+ # handler after we've done what we need to do. This class takes care of
20
+ # that, too, because it's a legend.
21
+ #
22
+ # So that's what this class does: it allows you to specify signals and
23
+ # associated blocks of code to run, it sets up signal handlers which send
24
+ # notifications to a background thread and chain correctly, and it manages
25
+ # the background thread to receive the notifications and execute the
26
+ # associated blocks of code outside of the context of the signal handler.
27
+ #
28
+ class SignalManager
29
+ include ServiceSkeleton::LoggingHelpers
30
+
31
+ # Setup a signal handler instance.
32
+ #
33
+ # @param logger [Logger] the logger to use for all the interesting information
34
+ # about what we're up to.
35
+ #
36
+ def initialize(logger:, counter:, signals:)
37
+ @logger, @signal_counter, @signal_list = logger, counter, signals
38
+
39
+ @registry = Hash.new { |h, k| h[k] = SignalHandler.new(k) }
40
+
41
+ @signal_list.each do |sig, proc|
42
+ @registry[signum(sig)] << proc
43
+ end
44
+ end
45
+
46
+ def run
47
+ logger.info(logloc) { "Starting signal manager for #{@signal_list.length} signals" }
48
+
49
+ @r, @w = IO.pipe
50
+
51
+ install_signal_handlers
52
+
53
+ signals_loop
54
+ ensure
55
+ remove_signal_handlers
56
+ end
57
+
58
+ def shutdown
59
+ @r.close
60
+ end
61
+
62
+ private
63
+
64
+ attr_reader :logger
65
+
66
+ def signals_loop
67
+ #:nocov:
68
+ loop do
69
+ begin
70
+ if ios = IO.select([@r])
71
+ if ios.first.include?(@r)
72
+ if ios.first.first.eof?
73
+ logger.info(logloc) { "Signal pipe closed; shutting down" }
74
+ break
75
+ else
76
+ c = ios.first.first.read_nonblock(1)
77
+ logger.debug(logloc) { "Received character #{c.inspect} from signal pipe" }
78
+ handle_signal(c)
79
+ end
80
+ else
81
+ logger.error(logloc) { "Mysterious return from select: #{ios.inspect}" }
82
+ end
83
+ end
84
+ rescue IOError
85
+ # Something has gone terribly wrong here... bail
86
+ break
87
+ rescue StandardError => ex
88
+ log_exception(ex) { "Exception in select loop" }
89
+ end
90
+ end
91
+ #:nocov:
92
+ end
93
+
94
+ # Given a character (presumably) received via the signal pipe, execute the
95
+ # associated handler.
96
+ #
97
+ # @param char [String] a single character, corresponding to an entry in the
98
+ # signal registry.
99
+ #
100
+ # @return [void]
101
+ #
102
+ def handle_signal(char)
103
+ if @registry.has_key?(char.ord)
104
+ handler = @registry[char.ord]
105
+ logger.debug(logloc) { "#{handler.signame} received" }
106
+ @signal_counter.increment(labels: { signal: handler.signame.to_s })
107
+
108
+ begin
109
+ handler.call
110
+ rescue StandardError => ex
111
+ log_exception(ex) { "Exception while calling signal handler" }
112
+ end
113
+ else
114
+ logger.error(logloc) { "Unrecognised signal character: #{char.inspect}" }
115
+ end
116
+ end
117
+
118
+ def install_signal_handlers
119
+ @registry.values.each do |h|
120
+ h.write_pipe = @w
121
+ h.hook
122
+ end
123
+ end
124
+
125
+ def signum(spec)
126
+ if spec.is_a?(Integer)
127
+ return spec
128
+ end
129
+
130
+ if spec.is_a?(Symbol)
131
+ str = spec.to_s
132
+ elsif spec.is_a?(String)
133
+ str = spec.dup
134
+ else
135
+ raise ArgumentError,
136
+ "Unsupported class (#{spec.class}) of signal specifier #{spec.inspect}"
137
+ end
138
+
139
+ str.sub!(/\ASIG/i, '')
140
+
141
+ if Signal.list[str.upcase]
142
+ Signal.list[str.upcase]
143
+ else
144
+ raise ArgumentError,
145
+ "Unrecognised signal specifier #{spec.inspect}"
146
+ end
147
+ end
148
+
149
+ def remove_signal_handlers
150
+ @registry.values.each { |h| h.unhook }
151
+ end
152
+
153
+ class SignalHandler
154
+ attr_reader :signame
155
+ attr_writer :write_pipe
156
+
157
+ def initialize(signum)
158
+ @signum = signum
159
+ @callbacks = []
160
+
161
+ @signame = Signal.list.invert[@signum]
162
+ end
163
+
164
+ def <<(proc)
165
+ @callbacks << proc
166
+ end
167
+
168
+ def call
169
+ @callbacks.each { |cb| cb.call }
170
+ end
171
+
172
+ def hook
173
+ @handler = ->(_) do
174
+ #:nocov:
175
+ @write_pipe.write_nonblock(@signum.chr) rescue nil
176
+ @chain.call if @chain.respond_to?(:call)
177
+ #:nocov:
178
+ end
179
+
180
+ @chain = Signal.trap(@signum, &@handler)
181
+ end
182
+
183
+ def unhook
184
+ #:nocov:
185
+ tmp_handler = Signal.trap(@signum, "IGNORE")
186
+ if tmp_handler == @handler
187
+ # The current handler is ours, so we can replace it
188
+ # with the chained handler
189
+ Signal.trap(@signum, @chain)
190
+ else
191
+ # The current handler *isn't* ours, so we better
192
+ # put it back, because whoever owns it might get
193
+ # angry.
194
+ Signal.trap(@signum, tmp_handler)
195
+ end
196
+ #:nocov:
197
+ end
198
+ end
199
+
200
+ private_constant :SignalHandler
201
+ end
202
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module SignalsMethods
5
+ def registered_signal_handlers
6
+ @registered_signal_handlers || []
7
+ end
8
+
9
+ def hook_signal(sigspec, &blk)
10
+ @registered_signal_handlers ||= []
11
+
12
+ @registered_signal_handlers << [sigspec, blk]
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,17 @@
1
+ module ServiceSkeleton
2
+ module UltravisorChildren
3
+ def register_ultravisor_children(ultravisor, config:, metrics_registry:)
4
+ begin
5
+ ultravisor.add_child(
6
+ id: self.service_name.to_sym,
7
+ klass: self,
8
+ method: :run,
9
+ args: [config: config, metrics: metrics_registry]
10
+ )
11
+ rescue Ultravisor::InvalidKAMError
12
+ raise ServiceSkeleton::Error::InvalidServiceClassError,
13
+ "Class #{self.to_s} does not implement the `run' instance method"
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServiceSkeleton
4
+ module UltravisorLoggerstash
5
+ def logstash_writer
6
+ #:nocov:
7
+ @ultravisor[:logstash_writer].unsafe_instance
8
+ #:nocov:
9
+ end
10
+ end
11
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  begin
2
4
  require 'git-version-bump'
3
5
  rescue LoadError
@@ -30,14 +32,12 @@ Gem::Specification.new do |s|
30
32
 
31
33
  s.required_ruby_version = ">= 2.5.0"
32
34
 
33
- s.add_runtime_dependency "frankenstein", "~> 1.2"
34
- s.add_runtime_dependency "loggerstash", "~> 0.0"
35
- # prometheus-client provides no guaranteed backwards compatibility,
36
- # and in fact happily breaks things with no notice, so we're stuck
37
- # with hard-coding a specific version to avoid unexpected disaster.
38
- s.add_runtime_dependency "prometheus-client", "0.8.0"
35
+ s.add_runtime_dependency "frankenstein", "~> 2.0"
36
+ s.add_runtime_dependency "loggerstash", ">= 0.0.9", "< 1"
37
+ s.add_runtime_dependency "prometheus-client", "~> 2.0"
39
38
  s.add_runtime_dependency "sigdump", "~> 0.2"
40
39
  s.add_runtime_dependency "to_regexp", "~> 0.2"
40
+ s.add_runtime_dependency "ultravisor", "~> 0.a"
41
41
 
42
42
  s.add_development_dependency 'bundler'
43
43
  s.add_development_dependency 'github-release'
@@ -48,7 +48,8 @@ Gem::Specification.new do |s|
48
48
  s.add_development_dependency 'rake', "~> 12.0"
49
49
  s.add_development_dependency 'redcarpet'
50
50
  s.add_development_dependency 'rspec'
51
- s.add_development_dependency 'rubocop'
51
+ s.add_development_dependency 'rubocop', "~> 0.79"
52
+ s.add_development_dependency 'rubocop-discourse'
52
53
  s.add_development_dependency 'simplecov'
53
54
  s.add_development_dependency 'yard'
54
55
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: service_skeleton
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0.30.g32b8169
4
+ version: 0.0.0.48.g4a40599
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matt Palmer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-05-04 00:00:00.000000000 Z
11
+ date: 2020-06-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: frankenstein
@@ -16,42 +16,48 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.2'
19
+ version: '2.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.2'
26
+ version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: loggerstash
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - "~>"
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '0.0'
33
+ version: 0.0.9
34
+ - - "<"
35
+ - !ruby/object:Gem::Version
36
+ version: '1'
34
37
  type: :runtime
35
38
  prerelease: false
36
39
  version_requirements: !ruby/object:Gem::Requirement
37
40
  requirements:
38
- - - "~>"
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: 0.0.9
44
+ - - "<"
39
45
  - !ruby/object:Gem::Version
40
- version: '0.0'
46
+ version: '1'
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: prometheus-client
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
- - - '='
51
+ - - "~>"
46
52
  - !ruby/object:Gem::Version
47
- version: 0.8.0
53
+ version: '2.0'
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
- - - '='
58
+ - - "~>"
53
59
  - !ruby/object:Gem::Version
54
- version: 0.8.0
60
+ version: '2.0'
55
61
  - !ruby/object:Gem::Dependency
56
62
  name: sigdump
57
63
  requirement: !ruby/object:Gem::Requirement
@@ -80,6 +86,20 @@ dependencies:
80
86
  - - "~>"
81
87
  - !ruby/object:Gem::Version
82
88
  version: '0.2'
89
+ - !ruby/object:Gem::Dependency
90
+ name: ultravisor
91
+ requirement: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - "~>"
94
+ - !ruby/object:Gem::Version
95
+ version: 0.a
96
+ type: :runtime
97
+ prerelease: false
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: 0.a
83
103
  - !ruby/object:Gem::Dependency
84
104
  name: bundler
85
105
  requirement: !ruby/object:Gem::Requirement
@@ -208,6 +228,20 @@ dependencies:
208
228
  version: '0'
209
229
  - !ruby/object:Gem::Dependency
210
230
  name: rubocop
231
+ requirement: !ruby/object:Gem::Requirement
232
+ requirements:
233
+ - - "~>"
234
+ - !ruby/object:Gem::Version
235
+ version: '0.79'
236
+ type: :development
237
+ prerelease: false
238
+ version_requirements: !ruby/object:Gem::Requirement
239
+ requirements:
240
+ - - "~>"
241
+ - !ruby/object:Gem::Version
242
+ version: '0.79'
243
+ - !ruby/object:Gem::Dependency
244
+ name: rubocop-discourse
211
245
  requirement: !ruby/object:Gem::Requirement
212
246
  requirements:
213
247
  - - ">="
@@ -271,15 +305,31 @@ files:
271
305
  - LICENCE
272
306
  - README.md
273
307
  - lib/service_skeleton.rb
274
- - lib/service_skeleton/background_worker.rb
275
308
  - lib/service_skeleton/config.rb
309
+ - lib/service_skeleton/config_class.rb
276
310
  - lib/service_skeleton/config_variable.rb
311
+ - lib/service_skeleton/config_variable/boolean.rb
312
+ - lib/service_skeleton/config_variable/enum.rb
313
+ - lib/service_skeleton/config_variable/float.rb
314
+ - lib/service_skeleton/config_variable/integer.rb
315
+ - lib/service_skeleton/config_variable/kv_list.rb
316
+ - lib/service_skeleton/config_variable/path_list.rb
317
+ - lib/service_skeleton/config_variable/string.rb
318
+ - lib/service_skeleton/config_variable/url.rb
319
+ - lib/service_skeleton/config_variable/yaml_file.rb
277
320
  - lib/service_skeleton/config_variables.rb
278
321
  - lib/service_skeleton/error.rb
279
322
  - lib/service_skeleton/filtering_logger.rb
323
+ - lib/service_skeleton/generator.rb
280
324
  - lib/service_skeleton/logging_helpers.rb
325
+ - lib/service_skeleton/metric_method_name.rb
281
326
  - lib/service_skeleton/metrics_methods.rb
282
- - lib/service_skeleton/signal_handler.rb
327
+ - lib/service_skeleton/runner.rb
328
+ - lib/service_skeleton/service_name.rb
329
+ - lib/service_skeleton/signal_manager.rb
330
+ - lib/service_skeleton/signals_methods.rb
331
+ - lib/service_skeleton/ultravisor_children.rb
332
+ - lib/service_skeleton/ultravisor_loggerstash.rb
283
333
  - service_skeleton.gemspec
284
334
  homepage: https://github.com/discourse/service_skeleton
285
335
  licenses: []
@@ -299,7 +349,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
299
349
  - !ruby/object:Gem::Version
300
350
  version: 1.3.1
301
351
  requirements: []
302
- rubygems_version: 3.0.1
352
+ rubygems_version: 3.0.3
303
353
  signing_key:
304
354
  specification_version: 4
305
355
  summary: The bare bones of a service