consul-templaterb 1.22.0 → 1.23.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -3
- data/CHANGELOG.md +10 -0
- data/README.md +5 -0
- data/bin/consul-templaterb +21 -0
- data/lib/consul/async/consul_endpoint.rb +10 -3
- data/lib/consul/async/consul_template.rb +21 -5
- data/lib/consul/async/consul_template_engine.rb +6 -0
- data/lib/consul/async/stats.rb +4 -1
- data/lib/consul/async/vault_endpoint.rb +9 -3
- data/lib/consul/async/version.rb +1 -1
- data/samples/metrics.erb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1322b0762c3a9ea41ae648227b38f42e21166193fbae07cc5b1b450b7814ce6b
|
4
|
+
data.tar.gz: e656d9e1b4cff345704bfe1e32ce71fc9be6b9433f554a0c6184ef506423e453
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 89029f97088f851c676263d79d584e0b6cfa22b72f2008690857f5341b7af3f8b7b627d082ada09b057010c97358cd75846f18ed8b825e4e212a0c07c0998abf
|
7
|
+
data.tar.gz: e2581df1a03190bb75a1f40838282ab49c9184e32a5fcf7807b1f86febbb1b0a88ea1deb93cddfe256c3aeccfeaf3874d4ad99f8b53935200fd15c9c8d71d821
|
data/.rubocop.yml
CHANGED
@@ -7,13 +7,13 @@ Metrics/AbcSize:
|
|
7
7
|
Max: 82
|
8
8
|
|
9
9
|
Metrics/BlockLength:
|
10
|
-
Max:
|
10
|
+
Max: 160
|
11
11
|
|
12
12
|
Metrics/BlockNesting:
|
13
13
|
Max: 4
|
14
14
|
|
15
15
|
Metrics/ClassLength:
|
16
|
-
Max:
|
16
|
+
Max: 275
|
17
17
|
|
18
18
|
Metrics/CyclomaticComplexity:
|
19
19
|
Max: 20
|
@@ -25,7 +25,7 @@ Metrics/MethodLength:
|
|
25
25
|
Max: 65
|
26
26
|
|
27
27
|
Metrics/ParameterLists:
|
28
|
-
Max:
|
28
|
+
Max: 14
|
29
29
|
|
30
30
|
Metrics/PerceivedComplexity:
|
31
31
|
Max: 23
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,16 @@
|
|
2
2
|
|
3
3
|
## (UNRELEASED)
|
4
4
|
|
5
|
+
## 1.23.0 (February 18, 2020)
|
6
|
+
|
7
|
+
NEW FEATURES:
|
8
|
+
|
9
|
+
* Implementation of #59 - implementation of `--retry` and `--vault-retry` new flags
|
10
|
+
Those flags work in a similar way as in consul-template: stop program after X failures
|
11
|
+
of consul or vault endpoints
|
12
|
+
* Added --fail-fast that stop the programs immediately if vault or consul are not available
|
13
|
+
at startup (also works with `--once`)
|
14
|
+
|
5
15
|
## 1.22.0 (January 17, 2020)
|
6
16
|
|
7
17
|
NEW FEATURES:
|
data/README.md
CHANGED
@@ -160,6 +160,9 @@ $ consul-templaterb --help
|
|
160
160
|
USAGE: consul-templaterb [[options]]
|
161
161
|
-h, --help Show help
|
162
162
|
-v, --version Show Version
|
163
|
+
--retry, --consul-retry-attempts [RETRIES]
|
164
|
+
If consul fails after n retries, stop the program, default=10
|
165
|
+
-f, --[no-]fail-fast If consul/vault endpoints fail at startup, fail immediately
|
163
166
|
-g, --no-gzip-compression Disable GZIP compression in HTTP requests
|
164
167
|
-c, --consul-addr=<address> Address of Consul, eg: http://localhost:8500
|
165
168
|
-l, --log-level=<log_level> Log level, default=info, any of none|error|info|debug
|
@@ -167,6 +170,8 @@ USAGE: consul-templaterb [[options]]
|
|
167
170
|
-V, --vault-addr=<address> Address of Vault, eg: http://localhost:8200
|
168
171
|
--vault-token=<token> Token used to authenticate against vault.
|
169
172
|
--[no-]vault-renew Control auto-renewal of the Vault token. Default: activated
|
173
|
+
--vault-retry, --vault-retry-attempts [RETRIES]
|
174
|
+
If vault fails after n retries, stop the program, default=10
|
170
175
|
--vault-lease-duration-factor=<factor>
|
171
176
|
Wait at least <factor> * lease time before updating a Vault secret. Default: 0.5
|
172
177
|
-w, --wait=<min_duration> Wait at least n seconds before each template generation
|
data/bin/consul-templaterb
CHANGED
@@ -27,6 +27,8 @@ options = {
|
|
27
27
|
},
|
28
28
|
base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
|
29
29
|
token: ENV['VAULT_TOKEN'] || nil,
|
30
|
+
max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
|
31
|
+
fail_fast_errors: nil, # fail fast the program if endpoint was never success
|
30
32
|
token_renew: true,
|
31
33
|
retry_duration: 10,
|
32
34
|
lease_duration_factor: 0.5, # The time it waits before actualizing data based on the lease time: 2h lease * 0.5 = Fetch data every 1h
|
@@ -45,6 +47,8 @@ options = {
|
|
45
47
|
},
|
46
48
|
base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500',
|
47
49
|
token: ENV['CONSUL_HTTP_TOKEN'] || nil,
|
50
|
+
max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
|
51
|
+
fail_fast_errors: nil, # fail fast the program if endpoint was never success
|
48
52
|
retry_duration: 10, # On error, retry after n seconds
|
49
53
|
min_duration: 5, # On sucess and when differences are found
|
50
54
|
retry_on_non_diff: 3, # On success but when there are not differences
|
@@ -94,6 +98,17 @@ optparse = OptionParser.new do |opts|
|
|
94
98
|
exit 0
|
95
99
|
end
|
96
100
|
|
101
|
+
opts.on('--retry [RETRIES]', '--consul-retry-attempts [RETRIES]', Integer,
|
102
|
+
"If consul fails after n retries, stop the program, default=#{options[:consul][:max_consecutive_errors_on_endpoint]}") do |value|
|
103
|
+
raise "#{value} Must be greater or equal to 0" unless value >= 0
|
104
|
+
options[:consul][:max_consecutive_errors_on_endpoint] = value
|
105
|
+
end
|
106
|
+
|
107
|
+
opts.on('-f', '--[no-]fail-fast', 'If consul/vault endpoints fail at startup, fail immediately') do |v|
|
108
|
+
options[:consul][:fail_fast_errors] = v
|
109
|
+
options[:vault][:fail_fast_errors] = v
|
110
|
+
end
|
111
|
+
|
97
112
|
opts.on('-g', '--no-gzip-compression', 'Disable GZIP compression in HTTP requests') do
|
98
113
|
options[:consul][:enable_gzip_compression] = false
|
99
114
|
end
|
@@ -123,6 +138,12 @@ optparse = OptionParser.new do |opts|
|
|
123
138
|
options[:vault][:token_renew] = vault_renew
|
124
139
|
end
|
125
140
|
|
141
|
+
opts.on('--vault-retry [RETRIES]', '--vault-retry-attempts [RETRIES]', Integer,
|
142
|
+
"If vault fails after n retries, stop the program, default=#{options[:vault][:max_consecutive_errors_on_endpoint]}") do |value|
|
143
|
+
raise "#{value} Must be greater or equal to 0" unless value >= 0
|
144
|
+
options[:vault][:max_consecutive_errors_on_endpoint] = value
|
145
|
+
end
|
146
|
+
|
126
147
|
opts.on('--vault-lease-duration-factor=<factor>', Float, 'Wait at least <factor> * lease time before updating a Vault secret. Default: 0.5') do |factor|
|
127
148
|
options[:vault][:lease_duration_factor] = factor
|
128
149
|
end
|
@@ -7,7 +7,8 @@ module Consul
|
|
7
7
|
module Async
|
8
8
|
class ConsulConfiguration
|
9
9
|
attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
|
10
|
-
:missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression
|
10
|
+
:missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression,
|
11
|
+
:fail_fast_errors, :max_consecutive_errors_on_endpoint
|
11
12
|
def initialize(base_url: 'http://localhost:8500',
|
12
13
|
debug: { network: false },
|
13
14
|
token: nil,
|
@@ -19,7 +20,9 @@ module Consul
|
|
19
20
|
missing_index_retry_time_on_diff: 15,
|
20
21
|
missing_index_retry_time_on_unchanged: 60,
|
21
22
|
enable_gzip_compression: true,
|
22
|
-
paths: {}
|
23
|
+
paths: {},
|
24
|
+
max_consecutive_errors_on_endpoint: 10,
|
25
|
+
fail_fast_errors: 1)
|
23
26
|
@base_url = base_url
|
24
27
|
@token = token
|
25
28
|
@debug = debug
|
@@ -32,6 +35,8 @@ module Consul
|
|
32
35
|
@missing_index_retry_time_on_diff = missing_index_retry_time_on_diff
|
33
36
|
@missing_index_retry_time_on_unchanged = missing_index_retry_time_on_unchanged
|
34
37
|
@paths = paths
|
38
|
+
@max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
|
39
|
+
@fail_fast_errors = fail_fast_errors
|
35
40
|
end
|
36
41
|
|
37
42
|
def ch(path, symbol)
|
@@ -57,7 +62,9 @@ module Consul
|
|
57
62
|
missing_index_retry_time_on_diff: ch(path, :missing_index_retry_time_on_diff),
|
58
63
|
missing_index_retry_time_on_unchanged: ch(path, :missing_index_retry_time_on_unchanged),
|
59
64
|
enable_gzip_compression: enable_gzip_compression,
|
60
|
-
paths: @paths
|
65
|
+
paths: @paths,
|
66
|
+
max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
|
67
|
+
fail_fast_errors: @fail_fast_errors)
|
61
68
|
end
|
62
69
|
end
|
63
70
|
class ConsulResult
|
@@ -79,8 +79,9 @@ module Consul
|
|
79
79
|
end
|
80
80
|
|
81
81
|
class EndPointsManager
|
82
|
-
attr_reader :consul_conf, :vault_conf, :net_info, :start_time, :coordinate, :remote_resource, :templates
|
82
|
+
attr_reader :consul_conf, :vault_conf, :running, :net_info, :start_time, :coordinate, :remote_resource, :templates
|
83
83
|
def initialize(consul_configuration, vault_configuration, templates, trim_mode = nil)
|
84
|
+
@running = true
|
84
85
|
@consul_conf = consul_configuration
|
85
86
|
@vault_conf = vault_configuration
|
86
87
|
@trim_mode = trim_mode
|
@@ -106,6 +107,8 @@ module Consul
|
|
106
107
|
},
|
107
108
|
params: {}
|
108
109
|
}
|
110
|
+
@max_consecutive_errors_on_endpoint = consul_configuration.max_consecutive_errors_on_endpoint || 10
|
111
|
+
@fail_fast_errors = consul_configuration.fail_fast_errors
|
109
112
|
@coordinate = Coordinate.new(self)
|
110
113
|
@remote_resource = RemoteResource.new(self)
|
111
114
|
|
@@ -221,7 +224,7 @@ module Consul
|
|
221
224
|
raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
|
222
225
|
path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
|
223
226
|
query_params = { list: 'true' }
|
224
|
-
create_if_missing(path, query_params) do
|
227
|
+
create_if_missing(path, query_params, vault_conf.fail_fast_errors, vault_conf.max_consecutive_errors_on_endpoint) do
|
225
228
|
ConsulTemplateVaultSecretList.new(VaultEndpoint.new(vault_conf, path, 'GET', true, query_params, JSON.generate(data: { keys: [] })))
|
226
229
|
end
|
227
230
|
end
|
@@ -231,7 +234,9 @@ module Consul
|
|
231
234
|
path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
|
232
235
|
query_params = {}
|
233
236
|
method = post_data ? 'POST' : 'GET'
|
234
|
-
create_if_missing(path, query_params
|
237
|
+
create_if_missing(path, query_params, vault_conf.fail_fast_errors, vault_conf.max_consecutive_errors_on_endpoint) do
|
238
|
+
ConsulTemplateVaultSecret.new(VaultEndpoint.new(vault_conf, path, method, true, query_params, JSON.generate(data: {})))
|
239
|
+
end
|
235
240
|
end
|
236
241
|
|
237
242
|
# render a relative file with the given params accessible from template
|
@@ -329,6 +334,7 @@ module Consul
|
|
329
334
|
end
|
330
335
|
|
331
336
|
def terminate
|
337
|
+
@running = false
|
332
338
|
@endpoints.each_value do |v|
|
333
339
|
v.endpoint.terminate
|
334
340
|
end
|
@@ -341,7 +347,7 @@ module Consul
|
|
341
347
|
VaultEndpoint.new(vault_conf, path, :POST, {}, {})
|
342
348
|
end
|
343
349
|
|
344
|
-
def create_if_missing(path, query_params)
|
350
|
+
def create_if_missing(path, query_params, fail_fast_errors = @fail_fast_errors, max_consecutive_errors_on_endpoint = @max_consecutive_errors_on_endpoint)
|
345
351
|
fqdn = path.dup
|
346
352
|
query_params.each_pair do |k, v|
|
347
353
|
fqdn = "#{fqdn}&#{k}=#{v}"
|
@@ -357,7 +363,17 @@ module Consul
|
|
357
363
|
@net_info[:changes] += 1 if result.modified?
|
358
364
|
@net_info[:network_bytes] += result.http.response_header['Content-Length'].to_i
|
359
365
|
end
|
360
|
-
tpl.endpoint.on_error
|
366
|
+
tpl.endpoint.on_error do |_err|
|
367
|
+
@net_info[:errors] = @net_info[:errors] + 1
|
368
|
+
if tpl.endpoint.stats.successes.zero? && fail_fast_errors
|
369
|
+
::Consul::Async::Debug.puts_error "Endpoint #{path} is failing at first call with fail fast activated, terminating..."
|
370
|
+
terminate
|
371
|
+
end
|
372
|
+
if tpl.endpoint.stats.consecutive_errors > max_consecutive_errors_on_endpoint
|
373
|
+
::Consul::Async::Debug.puts_error "Endpoint #{path} has too many consecutive errors: #{tpl.endpoint.stats.consecutive_errors}, terminating..."
|
374
|
+
terminate
|
375
|
+
end
|
376
|
+
end
|
361
377
|
end
|
362
378
|
tpl._seen_at(@iteration)
|
363
379
|
tpl
|
@@ -44,6 +44,12 @@ module Consul
|
|
44
44
|
|
45
45
|
# Run templating engine once
|
46
46
|
def do_run(template_manager, template_renders)
|
47
|
+
unless template_manager.running
|
48
|
+
::Consul::Async::Debug.puts_info '[FATAL] TemplateManager has been stopped, stopping everything'
|
49
|
+
@result = 3
|
50
|
+
EventMachine.stop
|
51
|
+
return
|
52
|
+
end
|
47
53
|
results = template_renders.map(&:run)
|
48
54
|
all_ready = results.all?(&:ready?)
|
49
55
|
if !@all_templates_rendered && all_ready
|
data/lib/consul/async/stats.rb
CHANGED
@@ -3,7 +3,7 @@ require 'consul/async/utilities'
|
|
3
3
|
module Consul
|
4
4
|
module Async
|
5
5
|
class EndPointStats
|
6
|
-
attr_reader :successes, :errors, :start, :body_bytes, :last_error, :last_success, :last_modified, :changes, :network_bytes
|
6
|
+
attr_reader :successes, :errors, :consecutive_errors, :start, :body_bytes, :last_error, :last_success, :last_modified, :changes, :network_bytes
|
7
7
|
|
8
8
|
def initialize
|
9
9
|
@start = Time.now.utc
|
@@ -15,6 +15,7 @@ module Consul
|
|
15
15
|
@last_error = @start
|
16
16
|
@last_success = @start
|
17
17
|
@last_modified = @start
|
18
|
+
@consecutive_errors = 0
|
18
19
|
end
|
19
20
|
|
20
21
|
def on_response(res)
|
@@ -23,12 +24,14 @@ module Consul
|
|
23
24
|
@body_bytes += res.http.response.bytesize
|
24
25
|
@changes += 1 if res.modified?
|
25
26
|
@last_modified = @last_success if res.modified?
|
27
|
+
@consecutive_errors = 0
|
26
28
|
@network_bytes += res.http.response_header['Content-Length'].to_i
|
27
29
|
end
|
28
30
|
|
29
31
|
def on_error(_http)
|
30
32
|
@last_error = Time.now.utc
|
31
33
|
@errors += 1
|
34
|
+
@consecutive_errors += 1
|
32
35
|
end
|
33
36
|
|
34
37
|
def bytes_per_sec(now = Time.now.utc)
|
@@ -10,7 +10,7 @@ module Consul
|
|
10
10
|
module Async
|
11
11
|
class VaultConfiguration
|
12
12
|
attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
|
13
|
-
:lease_duration_factor, :debug
|
13
|
+
:lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors
|
14
14
|
|
15
15
|
def initialize(base_url: 'http://localhost:8200',
|
16
16
|
debug: { network: false },
|
@@ -20,7 +20,9 @@ module Consul
|
|
20
20
|
min_duration: 0.1,
|
21
21
|
lease_duration_factor: 0.5,
|
22
22
|
max_retry_duration: 600,
|
23
|
-
paths: {}
|
23
|
+
paths: {},
|
24
|
+
max_consecutive_errors_on_endpoint: 10,
|
25
|
+
fail_fast_errors: false)
|
24
26
|
@base_url = base_url
|
25
27
|
@token_renew = token_renew
|
26
28
|
@debug = debug
|
@@ -30,6 +32,8 @@ module Consul
|
|
30
32
|
@lease_duration_factor = lease_duration_factor
|
31
33
|
@paths = paths
|
32
34
|
@token = token
|
35
|
+
@max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
|
36
|
+
@fail_fast_errors = fail_fast_errors
|
33
37
|
end
|
34
38
|
|
35
39
|
def ch(path, symbol)
|
@@ -51,7 +55,9 @@ module Consul
|
|
51
55
|
min_duration: ch(path, :min_duration),
|
52
56
|
max_retry_duration: ch(path, :max_retry_duration),
|
53
57
|
lease_duration_factor: ch(path, :lease_duration_factor),
|
54
|
-
paths: @paths
|
58
|
+
paths: @paths,
|
59
|
+
max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
|
60
|
+
fail_fast_errors: @fail_fast_errors)
|
55
61
|
end
|
56
62
|
end
|
57
63
|
class VaultResult
|
data/lib/consul/async/version.rb
CHANGED
data/samples/metrics.erb
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# EXCLUDE_SERVICES: comma-separated services regexps to exclude (example: lbl7.*,netsvc-probe.*,consul-probed.*)
|
8
8
|
# PROMETHEUS_EXPORTED_SERVICE_META: comma-separated list of meta to export into metrics of service
|
9
9
|
<%
|
10
|
-
service_metas_to_export = (ENV['PROMETHEUS_EXPORTED_SERVICE_META'] || 'version,os,criteo_flavor,node_group').split(',')
|
10
|
+
service_metas_to_export = (ENV['PROMETHEUS_EXPORTED_SERVICE_META'] || 'OWNERS,version,os,criteo_flavor,node_group').split(',')
|
11
11
|
|
12
12
|
service_tag_filter = ENV['SERVICES_TAG_FILTER'] || nil
|
13
13
|
instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: consul-templaterb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.23.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SRE Core Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: em-http-request
|