consul-templaterb 1.23.0 → 1.24.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +20 -6
- data/.travis.yml +1 -1
- data/CHANGELOG.md +6 -0
- data/bin/consul-templaterb +25 -20
- data/consul-templaterb.gemspec +4 -7
- data/lib/consul/async/consul_endpoint.rb +16 -6
- data/lib/consul/async/consul_template.rb +53 -6
- data/lib/consul/async/consul_template_engine.rb +9 -7
- data/lib/consul/async/consul_template_render.rb +13 -7
- data/lib/consul/async/debug.rb +5 -3
- data/lib/consul/async/json_endpoint.rb +4 -0
- data/lib/consul/async/process_handler.rb +16 -6
- data/lib/consul/async/stats.rb +2 -0
- data/lib/consul/async/utilities.rb +3 -1
- data/lib/consul/async/vault_endpoint.rb +10 -6
- data/lib/consul/async/version.rb +1 -1
- data/samples/consul-ui/css/style.css +4 -0
- data/samples/consul-ui/decorators.js.erb +58 -1
- data/samples/consul-ui/js/utils.js +0 -16
- data/samples/consul-ui/ringbuffer.rb +12 -3
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f4f6bc7af6e5c8ccdb5417157b443de3c43ae33ffbcd90953b9960033d1cf960
|
4
|
+
data.tar.gz: f9635b5936d0f11cbfa32b7088faba4d90e1b43ed4ac7c4f4127f6d450981435
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 68aae80142edd53811fe41b427573f6dd06e5dfdd19bba59607050927a7f17aa4ff1cb70f697e51213105fd97771fccafef833592d3d2a58f6857a5d8f982231
|
7
|
+
data.tar.gz: 3a74bf0df8f28e3578615db1e2155772668528814c14ae6010849cc7cbdccc2c662485f9fead52b5849993407cf708d97902c9cf4ae2ff1ce1345bff3f769281
|
data/.rubocop.yml
CHANGED
@@ -3,6 +3,9 @@ AllCops:
|
|
3
3
|
- bundle/**/*
|
4
4
|
TargetRubyVersion: 2.4
|
5
5
|
|
6
|
+
Layout/LineLength:
|
7
|
+
Max: 175
|
8
|
+
|
6
9
|
Metrics/AbcSize:
|
7
10
|
Max: 82
|
8
11
|
|
@@ -18,9 +21,6 @@ Metrics/ClassLength:
|
|
18
21
|
Metrics/CyclomaticComplexity:
|
19
22
|
Max: 20
|
20
23
|
|
21
|
-
Metrics/LineLength:
|
22
|
-
Max: 175
|
23
|
-
|
24
24
|
Metrics/MethodLength:
|
25
25
|
Max: 65
|
26
26
|
|
@@ -30,14 +30,28 @@ Metrics/ParameterLists:
|
|
30
30
|
Metrics/PerceivedComplexity:
|
31
31
|
Max: 23
|
32
32
|
|
33
|
-
|
33
|
+
# We use `dc` as a parameter in many methods
|
34
|
+
Naming/MethodParameterName:
|
35
|
+
Enabled: false
|
36
|
+
|
37
|
+
Naming/VariableNumber:
|
34
38
|
Enabled: false
|
35
39
|
|
40
|
+
Style/Documentation:
|
41
|
+
Enabled: true
|
42
|
+
|
36
43
|
Style/FrozenStringLiteralComment:
|
37
44
|
Enabled: false
|
38
45
|
|
46
|
+
Style/HashEachMethods:
|
47
|
+
Enabled: true
|
48
|
+
|
49
|
+
Style/HashTransformKeys:
|
50
|
+
Enabled: true
|
51
|
+
|
52
|
+
Style/HashTransformValues:
|
53
|
+
Enabled: true
|
54
|
+
|
39
55
|
Style/MultilineBlockChain:
|
40
56
|
Enabled: false
|
41
57
|
|
42
|
-
Style/VariableNumber:
|
43
|
-
Enabled: false
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/bin/consul-templaterb
CHANGED
@@ -14,12 +14,13 @@ end
|
|
14
14
|
def compute_default_output(source)
|
15
15
|
dest = source.gsub(/\.erb$/, '')
|
16
16
|
raise "Source and destination cannot be the same in #{source}" if source == dest || dest.empty?
|
17
|
+
|
17
18
|
dest
|
18
19
|
end
|
19
20
|
|
20
21
|
options = {
|
21
22
|
erb: {
|
22
|
-
trim_mode: '-'
|
23
|
+
trim_mode: '-' # trim_mode for ERB
|
23
24
|
},
|
24
25
|
vault: {
|
25
26
|
debug: {
|
@@ -59,22 +60,22 @@ options = {
|
|
59
60
|
enable_gzip_compression: true,
|
60
61
|
paths: {
|
61
62
|
'/v1/catalog/services': {
|
62
|
-
min_duration: 15
|
63
|
+
min_duration: 15 # Since services change a lot, refresh services every 15 seconds
|
63
64
|
},
|
64
65
|
'/v1/catalog/nodes': {
|
65
|
-
min_duration: 15
|
66
|
+
min_duration: 15 # Do not wake up before 15 seconds when node appear/disappear
|
66
67
|
},
|
67
68
|
'/v1/coordinate/nodes': {
|
68
|
-
min_duration: 60
|
69
|
+
min_duration: 60 # Since coordinates change a lot, refresh coordinates every 30 seconds
|
69
70
|
},
|
70
71
|
'/v1/catalog/datacenters': {
|
71
|
-
min_duration: 60
|
72
|
+
min_duration: 60 # Datacenters are not added every minute, right?
|
72
73
|
},
|
73
74
|
'/v1/agent/metrics': {
|
74
|
-
min_duration: 60
|
75
|
+
min_duration: 60 # Refresh metrics only minute max
|
75
76
|
},
|
76
77
|
'/v1/agent/self': {
|
77
|
-
min_duration: 60
|
78
|
+
min_duration: 60 # Refresh self info every minute max
|
78
79
|
}
|
79
80
|
}
|
80
81
|
}
|
@@ -89,18 +90,19 @@ optparse = OptionParser.new do |opts|
|
|
89
90
|
opts.banner = usage_text
|
90
91
|
|
91
92
|
opts.on('-h', '--help', 'Show help') do
|
92
|
-
|
93
|
+
warn opts
|
93
94
|
exit 0
|
94
95
|
end
|
95
96
|
|
96
97
|
opts.on('-v', '--version', 'Show Version') do
|
97
|
-
|
98
|
+
warn Consul::Async::VERSION
|
98
99
|
exit 0
|
99
100
|
end
|
100
101
|
|
101
102
|
opts.on('--retry [RETRIES]', '--consul-retry-attempts [RETRIES]', Integer,
|
102
103
|
"If consul fails after n retries, stop the program, default=#{options[:consul][:max_consecutive_errors_on_endpoint]}") do |value|
|
103
104
|
raise "#{value} Must be greater or equal to 0" unless value >= 0
|
105
|
+
|
104
106
|
options[:consul][:max_consecutive_errors_on_endpoint] = value
|
105
107
|
end
|
106
108
|
|
@@ -141,6 +143,7 @@ optparse = OptionParser.new do |opts|
|
|
141
143
|
opts.on('--vault-retry [RETRIES]', '--vault-retry-attempts [RETRIES]', Integer,
|
142
144
|
"If vault fails after n retries, stop the program, default=#{options[:vault][:max_consecutive_errors_on_endpoint]}") do |value|
|
143
145
|
raise "#{value} Must be greater or equal to 0" unless value >= 0
|
146
|
+
|
144
147
|
options[:vault][:max_consecutive_errors_on_endpoint] = value
|
145
148
|
end
|
146
149
|
|
@@ -150,6 +153,7 @@ optparse = OptionParser.new do |opts|
|
|
150
153
|
|
151
154
|
opts.on('-w', '--wait=<min_duration>', Integer, 'Wait at least n seconds before each template generation') do |min_duration|
|
152
155
|
raise "--wait=#{min_duration} must be greater than 0" unless min_duration.positive?
|
156
|
+
|
153
157
|
consul_engine.template_frequency = min_duration
|
154
158
|
end
|
155
159
|
|
@@ -169,6 +173,7 @@ optparse = OptionParser.new do |opts|
|
|
169
173
|
raise "Please specify a signal, use any of: #{valid_signals.inspect}" unless val
|
170
174
|
return nil if val == none_value
|
171
175
|
raise "Invalid signal #{val}, valid signals: #{valid_signals.inspect}" unless valid_signals.include? val
|
176
|
+
|
172
177
|
val
|
173
178
|
end
|
174
179
|
|
@@ -198,7 +203,7 @@ optparse = OptionParser.new do |opts|
|
|
198
203
|
if all_ready
|
199
204
|
modified = results.any?(&:modified)
|
200
205
|
if @programs[cmd].nil?
|
201
|
-
|
206
|
+
warn "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload || 'NONE'} on_term=#{sig_term}"
|
202
207
|
@programs[cmd] = Consul::Async::ProcessHandler.new(cmd, sig_reload: sig_reload, sig_term: sig_term)
|
203
208
|
@programs[cmd].start
|
204
209
|
else
|
@@ -207,7 +212,7 @@ optparse = OptionParser.new do |opts|
|
|
207
212
|
begin
|
208
213
|
@programs[cmd].process_status
|
209
214
|
rescue Consul::Async::ProcessDoesNotExist => e
|
210
|
-
|
215
|
+
warn "[FATAL] The process is dead, aborting run: #{e.inspect}"
|
211
216
|
template_manager.terminate
|
212
217
|
EventMachine.stop
|
213
218
|
end
|
@@ -243,9 +248,10 @@ optparse = OptionParser.new do |opts|
|
|
243
248
|
dest = splitted[1]
|
244
249
|
unless dest
|
245
250
|
dest = compute_default_output(source)
|
246
|
-
|
251
|
+
warn "-t --template #{tpl} : Since output has not been set, using #{dest}"
|
247
252
|
end
|
248
253
|
raise "Source and destination cannot be the same in #{tpl}" if source == dest || dest.empty?
|
254
|
+
|
249
255
|
command = splitted[2]
|
250
256
|
parameter_file = splitted[3]
|
251
257
|
params = {}
|
@@ -255,6 +261,7 @@ optparse = OptionParser.new do |opts|
|
|
255
261
|
consul_engine.add_template_callback do |_all_ready, _template_manager, results|
|
256
262
|
results.each do |res|
|
257
263
|
next unless res.ready? && res.modified && res.output_file == dest && res.template_file == source
|
264
|
+
|
258
265
|
# Our template has been fully rendered
|
259
266
|
system(command)
|
260
267
|
end
|
@@ -265,7 +272,7 @@ optparse = OptionParser.new do |opts|
|
|
265
272
|
opts.on('-o', '--once', 'Do not run the process as a daemon') do
|
266
273
|
consul_engine.add_template_callback do |all_ready, template_manager, _|
|
267
274
|
if all_ready
|
268
|
-
|
275
|
+
warn '[INFO] Program ends since daemon mode has been disabled, file(s) has been written'
|
269
276
|
template_manager.terminate
|
270
277
|
EventMachine.stop
|
271
278
|
end
|
@@ -275,7 +282,7 @@ end
|
|
275
282
|
|
276
283
|
def kill_program
|
277
284
|
@programs.each do |k, v|
|
278
|
-
|
285
|
+
warn "Killing process #{k}..."
|
279
286
|
v.kill
|
280
287
|
end
|
281
288
|
@programs = {}
|
@@ -295,14 +302,12 @@ def find_max_descriptors(max_descripors)
|
|
295
302
|
end
|
296
303
|
|
297
304
|
%i[consul vault].each do |type|
|
298
|
-
unless options[type][:base_url].start_with? 'http', 'https'
|
299
|
-
options[type][:base_url] = "http://#{options[type][:base_url]}"
|
300
|
-
end
|
305
|
+
options[type][:base_url] = "http://#{options[type][:base_url]}" unless options[type][:base_url].start_with? 'http', 'https'
|
301
306
|
end
|
302
307
|
|
303
308
|
# Since we might be using a lots of descriptors, document this
|
304
309
|
new_size = find_max_descriptors(65_536)
|
305
|
-
|
310
|
+
warn "Max number of descriptors set to #{new_size}" if options[:consul][:debug][:network]
|
306
311
|
|
307
312
|
# This is needed to avoid EM not to crash on some Linux Hosts
|
308
313
|
# When using a very large number of Consul Endpoints
|
@@ -329,7 +334,7 @@ Signal.trap('USR1', 'IGNORE') unless Gem.win_platform?
|
|
329
334
|
Signal.trap(sig) do |signo|
|
330
335
|
# handle nicely forks
|
331
336
|
m_p = my_pid == Process.pid
|
332
|
-
|
337
|
+
warn "[KILL] received #{Signal.signame(signo)}, stopping myself" if m_p
|
333
338
|
template_manager.terminate
|
334
339
|
kill_program if m_p
|
335
340
|
end
|
@@ -338,7 +343,7 @@ end
|
|
338
343
|
begin
|
339
344
|
consul_engine.run(template_manager)
|
340
345
|
rescue Interrupt => e
|
341
|
-
|
346
|
+
warn "[WARN] consul-templaterb has been interrupted #{e}"
|
342
347
|
end
|
343
348
|
|
344
349
|
exit consul_engine.result
|
data/consul-templaterb.gemspec
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
lib = File.expand_path('../lib', __FILE__)
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
4
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
3
|
require 'consul/async/version'
|
6
4
|
|
@@ -16,8 +14,8 @@ Gem::Specification.new do |spec|
|
|
16
14
|
spec.description = 'A ruby implementation of Consul Template with support of erb templating ' \
|
17
15
|
'with hi-performance on large clusters and advanced process management features.'
|
18
16
|
spec.metadata = { 'bug_tracker_uri' => 'https://github.com/criteo/consul-templaterb/issues',
|
19
|
-
'changelog_uri'
|
20
|
-
'homepage_uri'
|
17
|
+
'changelog_uri' => 'https://github.com/criteo/consul-templaterb/blob/master/CHANGELOG.md',
|
18
|
+
'homepage_uri' => 'https://github.com/criteo/consul-templaterb',
|
21
19
|
'source_code_uri' => 'https://github.com/criteo/consul-templaterb' }
|
22
20
|
spec.license = 'Apache v2'
|
23
21
|
|
@@ -29,7 +27,6 @@ Gem::Specification.new do |spec|
|
|
29
27
|
spec.extra_rdoc_files = ['README.md', 'CHANGELOG.md', 'TemplateAPI.md']
|
30
28
|
spec.require_paths = ['lib']
|
31
29
|
|
32
|
-
spec.require_paths = ['lib']
|
33
30
|
spec.add_runtime_dependency 'em-http-request', '>= 1.1.5'
|
34
31
|
spec.add_runtime_dependency 'eventmachine', '>= 1.2.7'
|
35
32
|
spec.add_runtime_dependency 'parallel', '>= 1.2.7'
|
@@ -38,7 +35,7 @@ Gem::Specification.new do |spec|
|
|
38
35
|
spec.add_development_dependency 'rake', '~> 10.0'
|
39
36
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
40
37
|
spec.add_development_dependency 'rspec_junit_formatter'
|
41
|
-
spec.add_development_dependency 'rubocop', '0.
|
38
|
+
spec.add_development_dependency 'rubocop', '0.80.0'
|
42
39
|
spec.add_development_dependency 'rubocop-junit-formatter'
|
43
40
|
spec.add_development_dependency 'webmock'
|
44
41
|
end
|
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'consul/async/utilities'
|
2
2
|
require 'consul/async/stats'
|
3
3
|
require 'em-http'
|
4
|
-
require 'thread'
|
5
4
|
require 'json'
|
6
5
|
module Consul
|
7
6
|
module Async
|
7
|
+
# The class configuring Consul endpoints
|
8
|
+
# It allows to translate configuration options per specific endpoint/path
|
8
9
|
class ConsulConfiguration
|
9
10
|
attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
|
10
11
|
:missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression,
|
@@ -51,6 +52,7 @@ module Consul
|
|
51
52
|
|
52
53
|
def create(path)
|
53
54
|
return self unless @paths[path.to_sym]
|
55
|
+
|
54
56
|
ConsulConfiguration.new(base_url: ch(path, :base_url),
|
55
57
|
debug: ch(path, :debug),
|
56
58
|
token: ch(path, :token),
|
@@ -67,6 +69,9 @@ module Consul
|
|
67
69
|
fail_fast_errors: @fail_fast_errors)
|
68
70
|
end
|
69
71
|
end
|
72
|
+
|
73
|
+
# This keep track of answer from Consul
|
74
|
+
# It also keep statistics about result (x_consul_index, stats...)
|
70
75
|
class ConsulResult
|
71
76
|
attr_reader :data, :http, :x_consul_index, :last_update, :stats, :retry_in
|
72
77
|
def initialize(data, modified, http, x_consul_index, stats, retry_in, fake: false)
|
@@ -102,6 +107,8 @@ module Consul
|
|
102
107
|
next_retry + last_update
|
103
108
|
end
|
104
109
|
end
|
110
|
+
# Basic Encapsulation of HTTP response from Consul
|
111
|
+
# It supports empty responses to handle first call is an easy way
|
105
112
|
class HttpResponse
|
106
113
|
attr_reader :response_header, :response, :error
|
107
114
|
def initialize(http, override_nil_response = nil)
|
@@ -116,6 +123,9 @@ module Consul
|
|
116
123
|
end
|
117
124
|
end
|
118
125
|
end
|
126
|
+
# This class represents a specific path in Consul HTTP API
|
127
|
+
# It also stores x_consul_index and keep track on updates of API
|
128
|
+
# So, it basically performs all the optimizations to keep updated with Consul internal state.
|
119
129
|
class ConsulEndpoint
|
120
130
|
attr_reader :conf, :path, :x_consul_index, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
|
121
131
|
def initialize(conf, path, enforce_json_200 = true, query_params = {}, default_value = '[]')
|
@@ -144,12 +154,12 @@ module Consul
|
|
144
154
|
on_response do |result|
|
145
155
|
state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
|
146
156
|
stats = result.stats
|
147
|
-
|
157
|
+
warn "[DBUG]#{state}#{result.modified? ? '[MODFIED]' : '[NO DIFF]'}" \
|
148
158
|
"[s:#{stats.successes},err:#{stats.errors}]" \
|
149
159
|
"[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
|
150
160
|
" #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
|
151
161
|
end
|
152
|
-
on_error { |http|
|
162
|
+
on_error { |http| warn "[ERROR]: #{path}: #{http.error.inspect}" }
|
153
163
|
end
|
154
164
|
|
155
165
|
def on_response(&block)
|
@@ -173,7 +183,7 @@ module Consul
|
|
173
183
|
def build_request(consul_index)
|
174
184
|
res = {
|
175
185
|
head: {
|
176
|
-
'Accept'
|
186
|
+
'Accept' => 'application/json',
|
177
187
|
'X-Consul-Index' => consul_index,
|
178
188
|
'X-Consul-Token' => conf.token
|
179
189
|
},
|
@@ -216,7 +226,7 @@ module Consul
|
|
216
226
|
def fetch
|
217
227
|
options = {
|
218
228
|
connect_timeout: 5, # default connection setup timeout
|
219
|
-
inactivity_timeout: conf.wait_duration + 1 + (conf.wait_duration / 16)
|
229
|
+
inactivity_timeout: conf.wait_duration + 1 + (conf.wait_duration / 16) # default connection inactivity (post-setup) timeout
|
220
230
|
}
|
221
231
|
connection = {
|
222
232
|
conn: EventMachine::HttpRequest.new(conf.base_url, options)
|
@@ -228,7 +238,7 @@ module Consul
|
|
228
238
|
is_kv_empty = path.start_with?('/v1/kv') && http.response_header.status == 404
|
229
239
|
if !is_kv_empty && enforce_json_200 && http.response_header.status != 200 && http.response_header['Content-Type'] != 'application/json'
|
230
240
|
_handle_error(http, consul_index) do
|
231
|
-
|
241
|
+
warn "[RETRY][#{path}] (#{@consecutive_errors} errors)" if (@consecutive_errors % 10) == 1
|
232
242
|
end
|
233
243
|
else
|
234
244
|
n_consul_index = find_x_consul_index(http)
|
@@ -1,12 +1,13 @@
|
|
1
1
|
require 'consul/async/utilities'
|
2
2
|
require 'em-http'
|
3
|
-
require 'thread'
|
4
3
|
require 'forwardable'
|
5
4
|
require 'erb'
|
6
5
|
require 'digest'
|
7
6
|
|
8
7
|
module Consul
|
9
8
|
module Async
|
9
|
+
# Exception thrown when a template is invalid and cause a Runtime Exception during
|
10
|
+
# its rendering
|
10
11
|
class InvalidTemplateException < StandardError
|
11
12
|
attr_reader :cause
|
12
13
|
def initialize(cause)
|
@@ -14,6 +15,7 @@ module Consul
|
|
14
15
|
end
|
15
16
|
end
|
16
17
|
|
18
|
+
# Exception thrown when the template is Invalid due to a Syntax Error
|
17
19
|
class SyntaxErrorInTemplate < InvalidTemplateException
|
18
20
|
attr_reader :cause
|
19
21
|
def initialize(cause)
|
@@ -21,6 +23,7 @@ module Consul
|
|
21
23
|
end
|
22
24
|
end
|
23
25
|
|
26
|
+
# Class to handle the retrival of a Remote resource (such a JSON API)
|
24
27
|
class RemoteResource
|
25
28
|
def initialize(endpoints_manager)
|
26
29
|
@endp_manager = endpoints_manager
|
@@ -78,6 +81,10 @@ module Consul
|
|
78
81
|
end
|
79
82
|
end
|
80
83
|
|
84
|
+
# This class keep references over all endpoints (aka datasources) registered for all templates.
|
85
|
+
# This allows reusing those endpoints as well as performing listing and garbage collecting.
|
86
|
+
# This is also the main object visible from ERB files which contains all methods available
|
87
|
+
# to template writters.
|
81
88
|
class EndPointsManager
|
82
89
|
attr_reader :consul_conf, :vault_conf, :running, :net_info, :start_time, :coordinate, :remote_resource, :templates
|
83
90
|
def initialize(consul_configuration, vault_configuration, templates, trim_mode = nil)
|
@@ -100,9 +107,9 @@ module Consul
|
|
100
107
|
@context = {
|
101
108
|
current_erb_path: nil,
|
102
109
|
template_info: {
|
103
|
-
'source_root'
|
104
|
-
'source'
|
105
|
-
'destination'
|
110
|
+
'source_root' => nil,
|
111
|
+
'source' => nil,
|
112
|
+
'destination' => nil,
|
106
113
|
'was_rendered_once' => false
|
107
114
|
},
|
108
115
|
params: {}
|
@@ -119,6 +126,7 @@ module Consul
|
|
119
126
|
# https://www.consul.io/api/health.html#list-nodes-for-service
|
120
127
|
def service(name, dc: nil, passing: false, tag: nil)
|
121
128
|
raise 'You must specify a name for a service' if name.nil?
|
129
|
+
|
122
130
|
path = "/v1/health/service/#{name}"
|
123
131
|
query_params = {}
|
124
132
|
query_params[:dc] = dc if dc
|
@@ -130,6 +138,7 @@ module Consul
|
|
130
138
|
# https://www.consul.io/api/health.html#list-checks-for-service
|
131
139
|
def checks_for_service(name, dc: nil, passing: false)
|
132
140
|
raise 'You must specify a name for a service' if name.nil?
|
141
|
+
|
133
142
|
path = "/v1/health/checks/#{name}"
|
134
143
|
query_params = {}
|
135
144
|
query_params[:dc] = dc if dc
|
@@ -140,6 +149,7 @@ module Consul
|
|
140
149
|
# https://www.consul.io/api/health.html#list-checks-for-node
|
141
150
|
def checks_for_node(name, dc: nil, passing: false)
|
142
151
|
raise 'You must specify a name for a service' if name.nil?
|
152
|
+
|
143
153
|
path = "/v1/health/node/#{name}"
|
144
154
|
query_params = {}
|
145
155
|
query_params[:dc] = dc if dc
|
@@ -182,8 +192,8 @@ module Consul
|
|
182
192
|
# Return a param of template
|
183
193
|
def param(key, default_value = nil)
|
184
194
|
v = @context[:params][key]
|
185
|
-
v
|
186
|
-
v
|
195
|
+
v ||= @context[:params][key.to_sym]
|
196
|
+
v ||= default_value
|
187
197
|
v
|
188
198
|
end
|
189
199
|
|
@@ -222,6 +232,7 @@ module Consul
|
|
222
232
|
|
223
233
|
def secrets(path = '')
|
224
234
|
raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
|
235
|
+
|
225
236
|
path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
|
226
237
|
query_params = { list: 'true' }
|
227
238
|
create_if_missing(path, query_params, vault_conf.fail_fast_errors, vault_conf.max_consecutive_errors_on_endpoint) do
|
@@ -231,6 +242,7 @@ module Consul
|
|
231
242
|
|
232
243
|
def secret(path = '', post_data = nil)
|
233
244
|
raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
|
245
|
+
|
234
246
|
path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
|
235
247
|
query_params = {}
|
236
248
|
method = post_data ? 'POST' : 'GET'
|
@@ -243,12 +255,14 @@ module Consul
|
|
243
255
|
def render_file(path, params = {})
|
244
256
|
new_path = File.expand_path(path, File.dirname(@context[:current_erb_path]))
|
245
257
|
raise "render_file ERROR: #{path} is resolved as #{new_path}, but the file does not exists" unless File.exist? new_path
|
258
|
+
|
246
259
|
render(File.read(new_path), new_path, params, current_template_info: template_info)
|
247
260
|
end
|
248
261
|
|
249
262
|
# render a sub template from a string template
|
250
263
|
def render_from_string(template_content, params = {})
|
251
264
|
return unless template_content
|
265
|
+
|
252
266
|
sha1res = Digest::SHA1.hexdigest(template_content)
|
253
267
|
new_path = File.expand_path(":memory:sha1:#{sha1res}", File.dirname(@context[:current_erb_path]))
|
254
268
|
render(template_content, new_path, params, current_template_info: template_info)
|
@@ -256,6 +270,7 @@ module Consul
|
|
256
270
|
|
257
271
|
def find_line(e)
|
258
272
|
return e.message.dup[5..-1] if e.message.start_with? '(erb):'
|
273
|
+
|
259
274
|
e.backtrace.each do |line|
|
260
275
|
return line[5..-1] if line.start_with? '(erb):'
|
261
276
|
end
|
@@ -273,6 +288,7 @@ module Consul
|
|
273
288
|
}
|
274
289
|
result = ERB.new(tpl, nil, @trim_mode).result(binding)
|
275
290
|
raise "Result is not a string :='#{result}' for #{tpl_file_path}" unless result.is_a?(String)
|
291
|
+
|
276
292
|
@context = old_value
|
277
293
|
result
|
278
294
|
rescue StandardError => e
|
@@ -380,6 +396,7 @@ module Consul
|
|
380
396
|
end
|
381
397
|
end
|
382
398
|
|
399
|
+
# Abstract class that stores information about a result
|
383
400
|
class ConsulTemplateAbstract
|
384
401
|
attr_reader :result, :endpoint, :seen_at
|
385
402
|
def initialize(consul_endpoint)
|
@@ -421,12 +438,14 @@ module Consul
|
|
421
438
|
end
|
422
439
|
end
|
423
440
|
|
441
|
+
# Concrete class of a result when the result is a JSON Object
|
424
442
|
class ConsulTemplateAbstractMap < ConsulTemplateAbstract
|
425
443
|
def initialize(consul_endpoint)
|
426
444
|
super(consul_endpoint)
|
427
445
|
end
|
428
446
|
end
|
429
447
|
|
448
|
+
# Concrete class of a result when the result is a JSON Array
|
430
449
|
class ConsulTemplateAbstractArray < ConsulTemplateAbstract
|
431
450
|
def initialize(consul_endpoint)
|
432
451
|
super(consul_endpoint)
|
@@ -436,8 +455,11 @@ module Consul
|
|
436
455
|
# technically this class could be also an array, a simple string or any simple json object other than a hash.
|
437
456
|
class ConsulTemplateAbstractJSONObject < ConsulTemplateAbstractMap; end
|
438
457
|
|
458
|
+
# Just another name
|
439
459
|
class ConsulTemplateAbstractJSONArray < ConsulTemplateAbstractArray; end
|
440
460
|
|
461
|
+
# The ServiceInstance has shortcuts (such as service_address method), but is
|
462
|
+
# basically a Hash.
|
441
463
|
class ServiceInstance < Hash
|
442
464
|
def initialize(obj)
|
443
465
|
merge!(obj)
|
@@ -471,6 +493,7 @@ module Consul
|
|
471
493
|
ret = 'passing'
|
472
494
|
checks = self['Checks']
|
473
495
|
return ret unless checks
|
496
|
+
|
474
497
|
checks.each do |chk|
|
475
498
|
st = chk['Status']
|
476
499
|
if st == 'critical'
|
@@ -494,6 +517,7 @@ module Consul
|
|
494
517
|
end
|
495
518
|
end
|
496
519
|
|
520
|
+
# Representation as a Map of a Service (includes Service, Node, Checks)
|
497
521
|
class ConsulTemplateService < ConsulTemplateAbstractMap
|
498
522
|
def initialize(consul_endpoint)
|
499
523
|
super(consul_endpoint)
|
@@ -503,6 +527,7 @@ module Consul
|
|
503
527
|
|
504
528
|
def result_delegate
|
505
529
|
return @cached_result if @cached_json == result.json
|
530
|
+
|
506
531
|
new_res = []
|
507
532
|
result.json.each do |v|
|
508
533
|
new_res << ServiceInstance.new(v)
|
@@ -513,12 +538,14 @@ module Consul
|
|
513
538
|
end
|
514
539
|
end
|
515
540
|
|
541
|
+
# Object returned by datacenters(), basically a JSON Array
|
516
542
|
class ConsulTemplateDatacenters < ConsulTemplateAbstractArray
|
517
543
|
def initialize(consul_endpoint)
|
518
544
|
super(consul_endpoint)
|
519
545
|
end
|
520
546
|
end
|
521
547
|
|
548
|
+
# Object returned by services() an abstract map of service_name, tags
|
522
549
|
class ConsulTemplateServices < ConsulTemplateAbstractMap
|
523
550
|
def initialize(consul_endpoint)
|
524
551
|
super(consul_endpoint)
|
@@ -526,6 +553,7 @@ module Consul
|
|
526
553
|
|
527
554
|
def parse_result(res)
|
528
555
|
return res unless res.data == '{}' || endpoint.query_params[:tag]
|
556
|
+
|
529
557
|
res_json = JSON.parse(res.data)
|
530
558
|
result = {}
|
531
559
|
res_json.each do |name, tags|
|
@@ -536,33 +564,43 @@ module Consul
|
|
536
564
|
end
|
537
565
|
end
|
538
566
|
|
567
|
+
# Another name to handle backwards compatibility
|
539
568
|
class ConsulTemplateJSONObject < ConsulTemplateAbstractJSONObject; end
|
569
|
+
|
570
|
+
# Another name to handle backwards compatibility
|
540
571
|
class ConsulTemplateJSONArray < ConsulTemplateAbstractJSONArray; end
|
541
572
|
|
573
|
+
# Object returned by /v1/agent/self, a JSON Map
|
542
574
|
class ConsulAgentSelf < ConsulTemplateAbstractMap
|
543
575
|
def initialize(consul_endpoint)
|
544
576
|
super(consul_endpoint)
|
545
577
|
end
|
546
578
|
end
|
547
579
|
|
580
|
+
# Object returning metrics from Consul agent, a JSON Map
|
548
581
|
class ConsulAgentMetrics < ConsulTemplateAbstractMap
|
549
582
|
def initialize(consul_endpoint)
|
550
583
|
super(consul_endpoint)
|
551
584
|
end
|
552
585
|
end
|
553
586
|
|
587
|
+
# List of checks for agent
|
554
588
|
class ConsulTemplateChecks < ConsulTemplateAbstractArray
|
555
589
|
def initialize(consul_endpoint)
|
556
590
|
super(consul_endpoint)
|
557
591
|
end
|
558
592
|
end
|
559
593
|
|
594
|
+
# List of nodes of the whole cluster
|
560
595
|
class ConsulTemplateNodes < ConsulTemplateAbstractArray
|
561
596
|
def initialize(consul_endpoint)
|
562
597
|
super(consul_endpoint)
|
563
598
|
end
|
564
599
|
end
|
565
600
|
|
601
|
+
# Key/Values representations
|
602
|
+
# This is an array as it might contain several values
|
603
|
+
# Several helpers exist to handle nicely transformations
|
566
604
|
class ConsulTemplateKV < ConsulTemplateAbstractArray
|
567
605
|
attr_reader :root
|
568
606
|
def initialize(consul_endpoint, root)
|
@@ -584,6 +622,7 @@ module Consul
|
|
584
622
|
def get_value_decoded(name = root)
|
585
623
|
val = get_value(name)
|
586
624
|
return nil unless val
|
625
|
+
|
587
626
|
Base64.decode64(val)
|
588
627
|
end
|
589
628
|
|
@@ -591,10 +630,12 @@ module Consul
|
|
591
630
|
def get_value_json(name = root, catch_errors: true)
|
592
631
|
x = get_value_decoded(name)
|
593
632
|
return nil unless x
|
633
|
+
|
594
634
|
begin
|
595
635
|
JSON.parse(x)
|
596
636
|
rescue JSON::ParserError => e
|
597
637
|
return nil if catch_errors
|
638
|
+
|
598
639
|
raise StandardError.new(e), "get_value_json() cannot deserialize kv(#{name}) as JSON: #{e.message}", e.backtrace
|
599
640
|
end
|
600
641
|
end
|
@@ -603,23 +644,29 @@ module Consul
|
|
603
644
|
def get_value_yaml(name = root, catch_errors: true)
|
604
645
|
x = get_value_decoded(name)
|
605
646
|
return nil unless x
|
647
|
+
|
606
648
|
begin
|
607
649
|
YAML.safe_load(x)
|
608
650
|
rescue YAML::ParserError => e
|
609
651
|
return nil if catch_errors
|
652
|
+
|
610
653
|
raise StandardError.new(e), "get_value_yaml() cannot deserialize kv(#{name}) as YAML: #{e.message}", e.backtrace
|
611
654
|
end
|
612
655
|
end
|
613
656
|
end
|
614
657
|
|
658
|
+
# Vault Secrets is a Map of secrets properly decoded
|
615
659
|
class ConsulTemplateVaultSecret < ConsulTemplateAbstractMap
|
616
660
|
def initialize(vault_endpoint)
|
617
661
|
super(vault_endpoint)
|
618
662
|
end
|
619
663
|
end
|
664
|
+
|
665
|
+
# Array of available secrets
|
620
666
|
class ConsulTemplateVaultSecretList < ConsulTemplateAbstractArray
|
621
667
|
def parse_result(res)
|
622
668
|
return res if res.data.nil?
|
669
|
+
|
623
670
|
res.mutate(JSON.generate(res.json['data']['keys']))
|
624
671
|
res
|
625
672
|
end
|
@@ -5,10 +5,11 @@ require 'consul/async/vault_endpoint'
|
|
5
5
|
require 'consul/async/consul_template'
|
6
6
|
require 'consul/async/consul_template_render'
|
7
7
|
require 'em-http'
|
8
|
-
require 'thread'
|
9
8
|
require 'erb'
|
10
9
|
module Consul
|
11
10
|
module Async
|
11
|
+
# The Engine keeps tracks of all templates, handle hot-reload of files if needed
|
12
|
+
# as well as ticking the clock to compute state of templates periodically.
|
12
13
|
class ConsulTemplateEngine
|
13
14
|
attr_reader :template_manager, :hot_reload_failure, :template_frequency, :debug_memory, :result, :templates
|
14
15
|
attr_writer :hot_reload_failure, :template_frequency, :debug_memory
|
@@ -61,17 +62,17 @@ module Consul
|
|
61
62
|
@template_callbacks.each do |c|
|
62
63
|
c.call([all_ready, template_manager, results])
|
63
64
|
end
|
64
|
-
rescue StandardError =>
|
65
|
-
::Consul::Async::Debug.puts_error "callback error: #{
|
66
|
-
raise
|
65
|
+
rescue StandardError => e
|
66
|
+
::Consul::Async::Debug.puts_error "callback error: #{e.inspect}"
|
67
|
+
raise e
|
67
68
|
end
|
68
69
|
rescue Consul::Async::InvalidTemplateException => e
|
69
|
-
|
70
|
+
warn "[FATAL]#{e}"
|
70
71
|
template_manager.terminate
|
71
72
|
@result = 1
|
72
73
|
EventMachine.stop
|
73
74
|
rescue StandardError => e
|
74
|
-
|
75
|
+
warn "[FATAL] Error occured: #{e.inspect} - #{e.backtrace.join("\n\t")}"
|
75
76
|
template_manager.terminate
|
76
77
|
@result = 2
|
77
78
|
EventMachine.stop
|
@@ -82,6 +83,7 @@ module Consul
|
|
82
83
|
do_run(template_manager, template_renders)
|
83
84
|
|
84
85
|
return if @all_templates_rendered || @periodic_started
|
86
|
+
|
85
87
|
# We continue if rendering not done and periodic not started
|
86
88
|
EventMachine.next_tick do
|
87
89
|
do_run_fast(template_manager, template_renders)
|
@@ -111,7 +113,7 @@ module Consul
|
|
111
113
|
diff_num_objects = new_memory_state[:objects] - @last_memory_state[:objects]
|
112
114
|
if diff_allocated != 0 || diff_num_objects.abs > (@last_memory_state[:pages] / 3)
|
113
115
|
timediff = new_memory_state[:time] - @last_memory_state[:time]
|
114
|
-
|
116
|
+
warn "[MEMORY] #{new_memory_state[:time]} significant RAM Usage detected\n" \
|
115
117
|
"[MEMORY] #{new_memory_state[:time]} Pages : #{new_memory_state[:pages]}" \
|
116
118
|
" (diff #{diff_allocated} aka #{(diff_allocated / timediff).round(0)}/s) \n" \
|
117
119
|
"[MEMORY] #{new_memory_state[:time]} Objects: #{new_memory_state[:objects]}"\
|
@@ -1,9 +1,10 @@
|
|
1
1
|
require 'consul/async/utilities'
|
2
2
|
require 'em-http'
|
3
|
-
require 'thread'
|
4
3
|
require 'erb'
|
5
4
|
module Consul
|
6
5
|
module Async
|
6
|
+
# Result of the rendering of a template
|
7
|
+
# .ready? will tell if rendering is full or partial
|
7
8
|
class ConsulTemplateRenderedResult
|
8
9
|
attr_reader :template_file, :output_file, :hot_reloaded, :ready, :modified, :last_result
|
9
10
|
def initialize(template_file, output_file, hot_reloaded, was_success, modified, last_result)
|
@@ -19,6 +20,9 @@ module Consul
|
|
19
20
|
@ready
|
20
21
|
end
|
21
22
|
end
|
23
|
+
# Object handling the whole rendering of a template
|
24
|
+
# It stores the input, the output and flags about last result being modified or simply readiness
|
25
|
+
# information about whether the template did receive all data to be fully rendered
|
22
26
|
class ConsulTemplateRender
|
23
27
|
attr_reader :template_file, :output_file, :template_file_ctime, :hot_reload_failure, :params
|
24
28
|
def initialize(template_manager, template_file, output_file, hot_reload_failure: 'die', params: {})
|
@@ -53,6 +57,7 @@ module Consul
|
|
53
57
|
# Will throw Consul::Async::InvalidTemplateException if template invalid
|
54
58
|
def update_template(new_template, current_template_info)
|
55
59
|
return false unless new_template != @template
|
60
|
+
|
56
61
|
# We render to ensure the template is valid
|
57
62
|
render(new_template, current_template_info)
|
58
63
|
@template = new_template.freeze
|
@@ -61,9 +66,9 @@ module Consul
|
|
61
66
|
|
62
67
|
def _build_default_template_info
|
63
68
|
{
|
64
|
-
'destination'
|
65
|
-
'source_root'
|
66
|
-
'source'
|
69
|
+
'destination' => @output_file,
|
70
|
+
'source_root' => template_file.freeze,
|
71
|
+
'source' => template_file.freeze,
|
67
72
|
'was_rendered_once' => @was_rendered_once
|
68
73
|
}
|
69
74
|
end
|
@@ -72,7 +77,7 @@ module Consul
|
|
72
77
|
current_template_info = _build_default_template_info
|
73
78
|
success, modified, last_res = @template_manager.write(@output_file, @template, @last_result, template_file, params, current_template_info: current_template_info)
|
74
79
|
@last_result = last_res if last_res
|
75
|
-
@was_rendered_once
|
80
|
+
@was_rendered_once ||= success
|
76
81
|
[success, modified, @last_result]
|
77
82
|
end
|
78
83
|
|
@@ -84,9 +89,10 @@ module Consul
|
|
84
89
|
@template_file_ctime = new_time
|
85
90
|
return update_template(load_template, current_template_info: current_template_info)
|
86
91
|
rescue Consul::Async::InvalidTemplateException => e
|
87
|
-
|
92
|
+
warn "****\n[ERROR] HOT Reload of template #{template_file} did fail due to:\n #{e}\n****\n template_info: #{current_template_info}\n****"
|
88
93
|
raise e unless hot_reload_failure == 'keep'
|
89
|
-
|
94
|
+
|
95
|
+
warn "[WARN] Hot reload of #{template_file} was not taken into account, keep running with previous version"
|
90
96
|
end
|
91
97
|
end
|
92
98
|
false
|
data/lib/consul/async/debug.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
module Consul
|
2
2
|
module Async
|
3
|
+
# Toolbox for logs
|
3
4
|
class Debug
|
4
5
|
def self.level
|
5
6
|
@level || 2
|
@@ -12,15 +13,16 @@ module Consul
|
|
12
13
|
def self.level=(log_level)
|
13
14
|
lvl = levels.index(log_level)
|
14
15
|
raise "Log level #{log_level} unsupported, must be one of #{levels.inspect}" if lvl.nil?
|
16
|
+
|
15
17
|
@level = lvl
|
16
18
|
end
|
17
19
|
|
18
20
|
def self.puts_error(msg)
|
19
|
-
|
21
|
+
warn "[ERROR] #{msg}" if level.positive?
|
20
22
|
end
|
21
23
|
|
22
24
|
def self.puts_info(msg)
|
23
|
-
|
25
|
+
warn "[INFO] #{msg}" if level > 1
|
24
26
|
end
|
25
27
|
|
26
28
|
def self.print_info(msg)
|
@@ -28,7 +30,7 @@ module Consul
|
|
28
30
|
end
|
29
31
|
|
30
32
|
def self.puts_debug(msg)
|
31
|
-
|
33
|
+
warn "[DEBG] #{msg}" if level > 2
|
32
34
|
end
|
33
35
|
|
34
36
|
def self.print_debug(msg)
|
@@ -5,6 +5,7 @@ require 'json'
|
|
5
5
|
|
6
6
|
module Consul
|
7
7
|
module Async
|
8
|
+
# Configuration to apply to JSONEndpoints
|
8
9
|
class JSONConfiguration
|
9
10
|
attr_reader :url, :retry_duration, :min_duration, :retry_on_non_diff,
|
10
11
|
:debug, :enable_gzip_compression, :request_method, :json_body,
|
@@ -34,6 +35,7 @@ module Consul
|
|
34
35
|
self
|
35
36
|
end
|
36
37
|
end
|
38
|
+
# Result from call to a Remote JSON endpoint
|
37
39
|
class JSONResult
|
38
40
|
attr_reader :data, :http, :last_update, :stats, :retry_in
|
39
41
|
def initialize(data, modified, http, stats, retry_in, fake: false)
|
@@ -67,6 +69,7 @@ module Consul
|
|
67
69
|
next_retry + last_update
|
68
70
|
end
|
69
71
|
end
|
72
|
+
# Encapsulation of HTTP Response
|
70
73
|
class HttpResponse
|
71
74
|
attr_reader :response_header, :response, :error
|
72
75
|
def initialize(http, override_nil_response = nil)
|
@@ -81,6 +84,7 @@ module Consul
|
|
81
84
|
end
|
82
85
|
end
|
83
86
|
end
|
87
|
+
# Endpoint (aka URL) of a remote API that might be called
|
84
88
|
class JSONEndpoint
|
85
89
|
attr_reader :conf, :url, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
|
86
90
|
def initialize(conf, url, default_value, enforce_json_200 = true, query_params = {})
|
@@ -1,11 +1,16 @@
|
|
1
1
|
module Consul
|
2
2
|
module Async
|
3
|
+
# Exception thrown when process does not exists
|
3
4
|
class ProcessDoesNotExist < StandardError
|
4
5
|
end
|
6
|
+
|
7
|
+
# Handle the full lifecycle of a process and allows to forward
|
8
|
+
# Posix signals to child process when needed.
|
5
9
|
class ProcessHandler
|
6
10
|
attr_reader :command, :sig_reload, :sig_term, :pid, :exit_status
|
7
11
|
def initialize(command, sig_reload: 'HUP', sig_term: 'TERM')
|
8
12
|
raise 'empty sig_term is not supported' unless sig_term
|
13
|
+
|
9
14
|
@command = command
|
10
15
|
@sig_reload = sig_reload
|
11
16
|
@sig_term = sig_term
|
@@ -15,44 +20,49 @@ module Consul
|
|
15
20
|
|
16
21
|
def start
|
17
22
|
return pid unless pid.nil?
|
23
|
+
|
18
24
|
@pid = Process.spawn(command)
|
19
25
|
end
|
20
26
|
|
21
27
|
def reload
|
22
28
|
return if sig_reload.nil?
|
23
|
-
|
29
|
+
|
30
|
+
warn "Sending SIG #{sig_reload} to #{pid}..."
|
24
31
|
begin
|
25
32
|
Process.kill(sig_reload, pid)
|
26
33
|
rescue Errno::ESRCH => e
|
27
|
-
|
34
|
+
warn "*** Process #{pid} has already been killed: #{e.inspect}"
|
28
35
|
raise e
|
29
36
|
end
|
30
37
|
end
|
31
38
|
|
32
39
|
def kill
|
33
40
|
return exit_status if pid.nil?
|
41
|
+
|
34
42
|
the_pid = pid
|
35
43
|
@pid = nil
|
36
|
-
|
44
|
+
warn "[KILL] Sending SIG #{sig_term} to #{the_pid}..."
|
37
45
|
begin
|
38
|
-
|
46
|
+
warn "[KILL] waiting for #{the_pid}..."
|
39
47
|
Process.kill(sig_term, the_pid)
|
40
48
|
rescue Errno::ESRCH
|
41
|
-
|
49
|
+
warn "[KILL] *** Process #{the_pid} has already been killed"
|
42
50
|
end
|
43
51
|
begin
|
44
52
|
_pid, @exit_status = Process.waitpid2 the_pid
|
45
53
|
rescue SystemCallError
|
46
|
-
|
54
|
+
warn "[KILL] *** UNEXPECTED ERROR *** Failed to get return code for #{the_pid}"
|
47
55
|
end
|
48
56
|
exit_status
|
49
57
|
end
|
50
58
|
|
51
59
|
def process_status
|
52
60
|
raise ProcessDoesNotExist, 'No child process' if pid.nil?
|
61
|
+
|
53
62
|
begin
|
54
63
|
cpid, result = Process.waitpid2(pid, Process::WNOHANG)
|
55
64
|
raise ProcessDoesNotExist, "Unexpected PID: #{cpid}, was expecting #{pid}" unless cpid.nil? || cpid == pid
|
65
|
+
|
56
66
|
result
|
57
67
|
rescue Errno::ECHILD => e
|
58
68
|
e2 = ProcessDoesNotExist.new e
|
data/lib/consul/async/stats.rb
CHANGED
@@ -2,6 +2,8 @@ require 'consul/async/utilities'
|
|
2
2
|
|
3
3
|
module Consul
|
4
4
|
module Async
|
5
|
+
# Object keeping stats about a single Endpoint
|
6
|
+
# Accessible within the .stats of a EndPoint
|
5
7
|
class EndPointStats
|
6
8
|
attr_reader :successes, :errors, :consecutive_errors, :start, :body_bytes, :last_error, :last_success, :last_modified, :changes, :network_bytes
|
7
9
|
|
@@ -3,6 +3,7 @@ require 'yaml'
|
|
3
3
|
require 'json'
|
4
4
|
module Consul
|
5
5
|
module Async
|
6
|
+
# Various utilities
|
6
7
|
class Utilities
|
7
8
|
def self.bytes_to_h(bytes)
|
8
9
|
if bytes < 1024
|
@@ -25,6 +26,7 @@ module Consul
|
|
25
26
|
# Loads parameters from a file, supports JSON and YAML
|
26
27
|
def self.load_parameters_from_file(parameters_file)
|
27
28
|
raise "Parameters file #{parameters_file} does not exists" unless File.exist? parameters_file
|
29
|
+
|
28
30
|
if parameters_file.downcase.end_with?('.yaml', '.yml')
|
29
31
|
YAML.load_file(parameters_file)
|
30
32
|
elsif parameters_file.downcase.end_with?('.json')
|
@@ -35,7 +37,7 @@ module Consul
|
|
35
37
|
end
|
36
38
|
|
37
39
|
def self.random
|
38
|
-
@random
|
40
|
+
@random ||= Random.new
|
39
41
|
@random
|
40
42
|
end
|
41
43
|
end
|
@@ -3,11 +3,11 @@ require 'consul/async/stats'
|
|
3
3
|
require 'consul/async/debug'
|
4
4
|
require 'em-http'
|
5
5
|
require 'net/http'
|
6
|
-
require 'thread'
|
7
6
|
require 'json'
|
8
7
|
|
9
8
|
module Consul
|
10
9
|
module Async
|
10
|
+
# Configuration for Vault Endpoints
|
11
11
|
class VaultConfiguration
|
12
12
|
attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
|
13
13
|
:lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors
|
@@ -48,6 +48,7 @@ module Consul
|
|
48
48
|
|
49
49
|
def create(path)
|
50
50
|
return self unless @paths[path.to_sym]
|
51
|
+
|
51
52
|
VaultConfiguration.new(base_url: ch(path, :base_url),
|
52
53
|
debug: ch(path, :debug),
|
53
54
|
token: ch(path, :token),
|
@@ -60,6 +61,7 @@ module Consul
|
|
60
61
|
fail_fast_errors: @fail_fast_errors)
|
61
62
|
end
|
62
63
|
end
|
64
|
+
# Keep information about Vault result of a query
|
63
65
|
class VaultResult
|
64
66
|
attr_reader :data, :http, :stats, :retry_in
|
65
67
|
|
@@ -92,6 +94,8 @@ module Consul
|
|
92
94
|
@data_json
|
93
95
|
end
|
94
96
|
end
|
97
|
+
|
98
|
+
# VaultHttpResponse supports empty results (when no data has been received yet)
|
95
99
|
class VaultHttpResponse
|
96
100
|
attr_reader :response_header, :response, :error, :json
|
97
101
|
|
@@ -109,6 +113,7 @@ module Consul
|
|
109
113
|
end
|
110
114
|
end
|
111
115
|
|
116
|
+
# Endpoint in vault (a path in Vault)
|
112
117
|
class VaultEndpoint
|
113
118
|
attr_reader :conf, :path, :http_method, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
|
114
119
|
|
@@ -140,7 +145,7 @@ module Consul
|
|
140
145
|
on_response do |result|
|
141
146
|
state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
|
142
147
|
stats = result.stats
|
143
|
-
|
148
|
+
warn "[DBUG]#{state}#{result.modified? ? '[MODFIED]' : '[NO DIFF]'}" \
|
144
149
|
"[s:#{stats.successes},err:#{stats.errors}]" \
|
145
150
|
"[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
|
146
151
|
" #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
|
@@ -192,6 +197,7 @@ module Consul
|
|
192
197
|
|
193
198
|
def _get_errors(http)
|
194
199
|
return [http.error] if http.error
|
200
|
+
|
195
201
|
unless http.json.nil?
|
196
202
|
return http.json['errors'] if http.json.key?('errors')
|
197
203
|
end
|
@@ -213,7 +219,7 @@ module Consul
|
|
213
219
|
def fetch
|
214
220
|
options = {
|
215
221
|
connect_timeout: 5, # default connection setup timeout
|
216
|
-
inactivity_timeout: 1
|
222
|
+
inactivity_timeout: 1 # default connection inactivity (post-setup) timeout
|
217
223
|
}
|
218
224
|
connection = EventMachine::HttpRequest.new(conf.base_url, options)
|
219
225
|
cb = proc do |_|
|
@@ -241,9 +247,7 @@ module Consul
|
|
241
247
|
end
|
242
248
|
|
243
249
|
http.errback do
|
244
|
-
unless @stopping
|
245
|
-
_handle_error(http) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
|
246
|
-
end
|
250
|
+
_handle_error(http) { connection = EventMachine::HttpRequest.new(conf.base_url, options) } unless @stopping
|
247
251
|
end
|
248
252
|
queue.pop(&cb)
|
249
253
|
end
|
data/lib/consul/async/version.rb
CHANGED
@@ -153,7 +153,6 @@ function decorateIsoDate(value, formated) {
|
|
153
153
|
return document.createTextNode(value);
|
154
154
|
}
|
155
155
|
const d = new Date(parsed);
|
156
|
-
console.log(d);
|
157
156
|
const e = document.createElement('time');
|
158
157
|
e.setAttribute('datetime', formated);
|
159
158
|
e.setAttribute('title', value);
|
@@ -223,6 +222,64 @@ function service_meta_semantics_decorator(instance, key, value, serviceName) {
|
|
223
222
|
return fun(instance, key, value);
|
224
223
|
}
|
225
224
|
|
225
|
+
const node_meta_decorators = {
|
226
|
+
'os': function(k, v, nodeMetaTags, serviceMetaIfAny) {
|
227
|
+
var e = document.createElement('span');
|
228
|
+
e.setAttribute('class', 'badge badge-light mx-1 lookup');
|
229
|
+
e.appendChild(document.createTextNode(k + ':' + v + ' '));
|
230
|
+
var i = document.createElement('i');
|
231
|
+
i.setAttribute('class', 'fab fa-' + v);
|
232
|
+
e.appendChild(i);
|
233
|
+
return e;
|
234
|
+
}
|
235
|
+
}
|
236
|
+
|
237
|
+
function default_node_meta_decorator(k, value, nodeMetaTags, serviceMetaIfAny) {
|
238
|
+
if (httpRegexp.test(value)) {
|
239
|
+
var e = document.createElement('a');
|
240
|
+
e.setAttribute('href', value);
|
241
|
+
e.setAttribute('class', 'lookup');
|
242
|
+
e.appendChild(document.createTextNode(k + ':'+ value));
|
243
|
+
var metaTag = document.createElement('span');
|
244
|
+
metaTag.setAttribute('class', 'badge badge-secondary mx-1');
|
245
|
+
metaTag.appendChild(e);
|
246
|
+
return metaTag;
|
247
|
+
} else {
|
248
|
+
var metaTag = document.createElement('span');
|
249
|
+
metaTag.setAttribute('class', 'badge badge-primary mx-1 lookup');
|
250
|
+
metaTag.appendChild(document.createTextNode(k + ':' + value));
|
251
|
+
return metaTag;
|
252
|
+
}
|
253
|
+
}
|
254
|
+
|
255
|
+
function find_decorator(k, v) {
|
256
|
+
const nd = node_meta_decorators[k];
|
257
|
+
if (nd != null) {
|
258
|
+
return nd;
|
259
|
+
} else {
|
260
|
+
return default_node_meta_decorator;
|
261
|
+
}
|
262
|
+
}
|
263
|
+
|
264
|
+
function nodeMetaGenerator(nodeMetaTags, serviceMetaIfAny) {
|
265
|
+
var allMeta = document.createElement('div');
|
266
|
+
var metaTags = document.createElement('div');
|
267
|
+
metaTags.setAttribute('title', 'Node Meta')
|
268
|
+
metaTags.className = 'node-meta';
|
269
|
+
for (var tagKey in nodeMetaTags) {
|
270
|
+
const tagValue = nodeMetaTags[tagKey];
|
271
|
+
if (tagValue == ''){
|
272
|
+
continue;
|
273
|
+
}
|
274
|
+
const decorator = find_decorator(tagKey, tagValue);
|
275
|
+
var clazz = 'badge badge-primary mx-1 lookup';
|
276
|
+
const elem = decorator(tagKey, tagValue, nodeMetaTags, serviceMetaIfAny);
|
277
|
+
metaTags.appendChild(elem);
|
278
|
+
}
|
279
|
+
allMeta.appendChild(metaTags);
|
280
|
+
return allMeta;
|
281
|
+
}
|
282
|
+
|
226
283
|
/**
|
227
284
|
* navBarDecorator is called to modify to modify naviguation bar of all UI pages.
|
228
285
|
* it receives the nav bar div
|
@@ -122,22 +122,6 @@ function nodeAddressGenator(nodeaddr) {
|
|
122
122
|
return htmlAddress;
|
123
123
|
}
|
124
124
|
|
125
|
-
function nodeMetaGenerator(nodeMetaTags) {
|
126
|
-
var metaTags = document.createElement('div');
|
127
|
-
metaTags.setAttribute('title', 'Node Meta')
|
128
|
-
metaTags.className = 'node-meta';
|
129
|
-
for (var tagKey in nodeMetaTags) {
|
130
|
-
if (!nodeMetaTags[tagKey]) {
|
131
|
-
continue;
|
132
|
-
}
|
133
|
-
var metaTag = document.createElement('span');
|
134
|
-
metaTag.setAttribute('class', 'badge badge-primary mx-1 lookup');
|
135
|
-
metaTag.appendChild(document.createTextNode(tagKey + ':' + nodeMetaTags[tagKey]));
|
136
|
-
metaTags.appendChild(metaTag);
|
137
|
-
}
|
138
|
-
return metaTags;
|
139
|
-
}
|
140
|
-
|
141
125
|
function tagsGenerator(instanceTags) {
|
142
126
|
var tags = document.createElement('div');
|
143
127
|
|
@@ -1,4 +1,6 @@
|
|
1
1
|
module ConsulTimeline
|
2
|
+
# Node of a ringbuffer
|
3
|
+
# This contains entries to next and previous elemens as well as the value
|
2
4
|
class RingBufferNode
|
3
5
|
attr_reader :prev, :value, :next
|
4
6
|
attr_writer :prev, :next, :value
|
@@ -28,10 +30,15 @@ module ConsulTimeline
|
|
28
30
|
"[prev=#{@prev.object_id}, next=#{@next.object_id}, value=#{@value}]"
|
29
31
|
end
|
30
32
|
end
|
33
|
+
# A ringbuffer that supports inserting out of order elements at the right place
|
34
|
+
# This is needed as Consul might notify us from changes in any order, so we
|
35
|
+
# might receive a notification t-2 AFTER t-1, at t.
|
36
|
+
# The ringbuffer will discard old elements so it keeps the "n" most recent elements only.
|
31
37
|
class SortedRingBuffer
|
32
38
|
include Enumerable
|
33
39
|
def initialize(max_size, sort_func)
|
34
40
|
raise "Invalid size #{max_size}" unless max_size.positive?
|
41
|
+
|
35
42
|
@head = RingBufferNode.new(nil, nil, nil)
|
36
43
|
@sort_func = sort_func
|
37
44
|
@tail = @head
|
@@ -43,11 +50,11 @@ module ConsulTimeline
|
|
43
50
|
|
44
51
|
def push(obj)
|
45
52
|
return unless obj
|
53
|
+
|
46
54
|
cur = @tail
|
47
55
|
raise "No head.next found in #{@head}" unless @head.next
|
48
|
-
|
49
|
-
|
50
|
-
end
|
56
|
+
|
57
|
+
cur = cur.prev while cur&.value && @sort_func.call(cur.value, obj).positive?
|
51
58
|
if cur.nil?
|
52
59
|
# The value we try to insert is before @head
|
53
60
|
# no need to do anything
|
@@ -64,6 +71,7 @@ module ConsulTimeline
|
|
64
71
|
|
65
72
|
def each
|
66
73
|
return enum_for(:each) unless block_given? # Sparkling magic!
|
74
|
+
|
67
75
|
cur = @head
|
68
76
|
until cur.nil?
|
69
77
|
yield cur.value if cur.value
|
@@ -103,5 +111,6 @@ if ARGV.count.positive? && ARGV[0] == 'debug'
|
|
103
111
|
ringbuff.push 99_999_999_999_999
|
104
112
|
arr = ringbuff.to_a
|
105
113
|
raise "OOPS wrong size := #{arr.count} instead of #{size}" unless arr.count == size
|
114
|
+
|
106
115
|
STDOUT.puts JSON.generate(arr)
|
107
116
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: consul-templaterb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.24.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SRE Core Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-02-
|
11
|
+
date: 2020-02-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: em-http-request
|
@@ -114,14 +114,14 @@ dependencies:
|
|
114
114
|
requirements:
|
115
115
|
- - '='
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
117
|
+
version: 0.80.0
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - '='
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
124
|
+
version: 0.80.0
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rubocop-junit-formatter
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|