consul-templaterb 1.5.3 → 1.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +3 -3
- data/CHANGELOG.md +14 -2
- data/README.md +2 -0
- data/Rakefile +3 -1
- data/lib/consul/async/consul_template.rb +10 -10
- data/lib/consul/async/endpoint.rb +5 -5
- data/lib/consul/async/stats.rb +0 -1
- data/lib/consul/async/utilities.rb +11 -5
- data/lib/consul/async/vault_endpoint.rb +4 -6
- data/lib/consul/async/version.rb +1 -1
- data/samples/criteo/haproxy.cfg.erb +1 -22
- data/samples/metrics.erb +71 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 29433f0be694b9d9358498c00ac3f4c0cf2112fdabd86801d94c2afe899243f9
|
4
|
+
data.tar.gz: df4dcdd2a3862122ffb122e3abaad2da1691366fd672c4a111b703586161ad37
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b20f9e3549f5a886da874cf48b567a264269573c8fdb0204c0d6e3627adc68297411a55de04fa9ec9e8946e5141b8a7dce548d90448d843792b29b8df533d680
|
7
|
+
data.tar.gz: 06722bbfaf0149c3fb3ee5b071de9950fb950ebaa53167926ad8f7d24f0bb86ea43a07e0d08f4809d19e15d7607e6d4c3a5bb6ea5365c0dd3905c403b3b1d664
|
data/.gitignore
CHANGED
@@ -18,6 +18,7 @@ samples/consul-ui/*.json
|
|
18
18
|
samples/consul-ui/common/*.html
|
19
19
|
/samples/criteo/haproxy.cfg
|
20
20
|
/samples/consul_template
|
21
|
+
/samples/metrics
|
21
22
|
/samples/find_blocked_choregraphies
|
22
23
|
/spec/consul/async/resources/templates/*.txt
|
23
24
|
/spec/consul/async/resources/templates/*.result
|
data/.rubocop.yml
CHANGED
@@ -7,19 +7,19 @@ Metrics/AbcSize:
|
|
7
7
|
Max: 80
|
8
8
|
|
9
9
|
Metrics/BlockLength:
|
10
|
-
Max:
|
10
|
+
Max: 135
|
11
11
|
|
12
12
|
Metrics/BlockNesting:
|
13
13
|
Max: 4
|
14
14
|
|
15
15
|
Metrics/ClassLength:
|
16
|
-
Max:
|
16
|
+
Max: 200
|
17
17
|
|
18
18
|
Metrics/CyclomaticComplexity:
|
19
19
|
Max: 15
|
20
20
|
|
21
21
|
Metrics/LineLength:
|
22
|
-
Max:
|
22
|
+
Max: 175
|
23
23
|
|
24
24
|
Metrics/MethodLength:
|
25
25
|
Max: 50
|
data/CHANGELOG.md
CHANGED
@@ -2,9 +2,21 @@
|
|
2
2
|
|
3
3
|
## (UNRELEASED)
|
4
4
|
|
5
|
-
## 1.5.
|
5
|
+
## 1.5.4 (September 26, 2018)
|
6
6
|
|
7
|
-
|
7
|
+
IMPROVEMENTS:
|
8
|
+
|
9
|
+
* [Prometheus template](samples/metrics.erb) to export easily Consul
|
10
|
+
informations about nodes, datacenters and all services states
|
11
|
+
* Code style cleanup + travis now enforces Rubocop
|
12
|
+
* Remove criteo references in spec files thanks to @pierrecdn
|
13
|
+
* Bitrate display more consistent thanks to @pierrecdn
|
14
|
+
|
15
|
+
## 1.5.3 (September 24, 2018)
|
16
|
+
|
17
|
+
IMPROVEMENTS:
|
18
|
+
|
19
|
+
* added to_a method to Arrays objects (ex: datacenters)
|
8
20
|
* Improved Consul-UI templates to have proper list of DCs
|
9
21
|
|
10
22
|
## 1.5.2 (September 12, 2018)
|
data/README.md
CHANGED
@@ -225,6 +225,8 @@ examples:
|
|
225
225
|
6. [Services in XML](samples/consul_template.xml.erb)
|
226
226
|
7. [Services in JSON](samples/consul_template.json.erb)
|
227
227
|
8. [Generate HaProxy Configuration](samples/ha_proxy.cfg.erb)
|
228
|
+
9. [Export Consul Statistics to Prometheus](samples/metrics.erb) : count all services, their state,
|
229
|
+
datacenters and nodes and export it to prometheus easily to trigger alerts.
|
228
230
|
|
229
231
|
If you want to test it quickly, you might try with (assuming your consul agent is listening on
|
230
232
|
`http://localhost:8500`):
|
data/Rakefile
CHANGED
@@ -38,10 +38,8 @@ module Consul
|
|
38
38
|
params: {}
|
39
39
|
}
|
40
40
|
|
41
|
-
|
42
|
-
|
43
|
-
vault_setup_token_renew
|
44
|
-
end
|
41
|
+
# Setup token renewal
|
42
|
+
vault_setup_token_renew unless @vault_conf.token.nil? || !@vault_conf.token_renew
|
45
43
|
end
|
46
44
|
|
47
45
|
# https://www.consul.io/api/health.html#list-nodes-for-service
|
@@ -135,16 +133,18 @@ module Consul
|
|
135
133
|
|
136
134
|
def secrets(path = '')
|
137
135
|
raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
|
138
|
-
path = "/v1/#{path}".gsub(
|
139
|
-
query_params = {list:
|
140
|
-
create_if_missing(path, query_params)
|
136
|
+
path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
|
137
|
+
query_params = { list: 'true' }
|
138
|
+
create_if_missing(path, query_params) do
|
139
|
+
ConsulTemplateVaultSecretList.new(VaultEndpoint.new(vault_conf, path, 'GET', true, query_params, JSON.generate(data: { keys: [] })))
|
140
|
+
end
|
141
141
|
end
|
142
142
|
|
143
143
|
def secret(path = '', post_data = nil)
|
144
144
|
raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
|
145
|
-
path = "/v1/#{path}".gsub(
|
145
|
+
path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
|
146
146
|
query_params = {}
|
147
|
-
method = post_data ?
|
147
|
+
method = post_data ? 'POST' : 'GET'
|
148
148
|
create_if_missing(path, query_params) { ConsulTemplateVaultSecret.new(VaultEndpoint.new(vault_conf, path, method, true, query_params, JSON.generate(data: {}))) }
|
149
149
|
end
|
150
150
|
|
@@ -228,7 +228,7 @@ module Consul
|
|
228
228
|
|
229
229
|
def vault_setup_token_renew
|
230
230
|
path = 'v1/auth/token/renew-self'
|
231
|
-
STDERR.print
|
231
|
+
STDERR.print '[INFO] Setting up vault token renewal'
|
232
232
|
VaultEndpoint.new(vault_conf, path, :POST, {}, {})
|
233
233
|
end
|
234
234
|
|
@@ -55,11 +55,11 @@ class Endpoint
|
|
55
55
|
|
56
56
|
def build_request(headers = {}, query_params = {})
|
57
57
|
req = {
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
58
|
+
head: headers,
|
59
|
+
path: path,
|
60
|
+
query: query_params,
|
61
|
+
keepalive: true,
|
62
|
+
callback: method(:on_response)
|
63
63
|
}
|
64
64
|
@query_params.each_pair do |k, v|
|
65
65
|
req[:query][k] = v
|
data/lib/consul/async/stats.rb
CHANGED
@@ -7,12 +7,18 @@ module Consul
|
|
7
7
|
def self.bytes_to_h(bytes)
|
8
8
|
if bytes < 1024
|
9
9
|
"#{bytes} b"
|
10
|
-
elsif bytes < 1_048_576
|
11
|
-
"#{(bytes / 1024).round(2)} Kb"
|
12
|
-
elsif bytes < 1_073_741_824
|
13
|
-
"#{(bytes / 1_048_576.0).round(2)} Mb"
|
14
10
|
else
|
15
|
-
|
11
|
+
if bytes < 1_048_576
|
12
|
+
bytes_h = bytes / 1024.0
|
13
|
+
unit_prefix = 'K'
|
14
|
+
elsif bytes < 1_073_741_824
|
15
|
+
bytes_h = bytes / 1_048_576.0
|
16
|
+
unit_prefix = 'M'
|
17
|
+
else
|
18
|
+
bytes_h = bytes / 1_073_741_824.0
|
19
|
+
unit_prefix = 'G'
|
20
|
+
end
|
21
|
+
"#{bytes_h.round(2)} #{unit_prefix}b"
|
16
22
|
end
|
17
23
|
end
|
18
24
|
|
@@ -29,7 +29,6 @@ module Consul
|
|
29
29
|
@lease_duration_factor = lease_duration_factor
|
30
30
|
@paths = paths
|
31
31
|
@token = token
|
32
|
-
|
33
32
|
end
|
34
33
|
|
35
34
|
def ch(path, symbol)
|
@@ -85,7 +84,6 @@ module Consul
|
|
85
84
|
@data_json = JSON.parse(data) if @data_json.nil?
|
86
85
|
@data_json
|
87
86
|
end
|
88
|
-
|
89
87
|
end
|
90
88
|
class VaultHttpResponse
|
91
89
|
attr_reader :response_header, :response, :error, :json
|
@@ -107,7 +105,7 @@ module Consul
|
|
107
105
|
class VaultEndpoint
|
108
106
|
attr_reader :conf, :path, :http_method, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
|
109
107
|
|
110
|
-
def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data ={})
|
108
|
+
def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data = {})
|
111
109
|
@conf = conf.create(path)
|
112
110
|
@default_value = default_value
|
113
111
|
@path = path
|
@@ -123,7 +121,7 @@ module Consul
|
|
123
121
|
@post_data = post_data
|
124
122
|
@stopping = false
|
125
123
|
@stats = EndPointStats.new
|
126
|
-
@last_result = VaultResult.new(VaultHttpResponse.new(nil, default_value), false, stats
|
124
|
+
@last_result = VaultResult.new(VaultHttpResponse.new(nil, default_value), false, stats, 1)
|
127
125
|
on_response { |result| @stats.on_response result }
|
128
126
|
on_error { |http| @stats.on_error http }
|
129
127
|
_enable_network_debug if conf.debug && conf.debug[:network]
|
@@ -161,7 +159,7 @@ module Consul
|
|
161
159
|
|
162
160
|
private
|
163
161
|
|
164
|
-
def build_request
|
162
|
+
def build_request
|
165
163
|
res = {
|
166
164
|
head: {
|
167
165
|
'Accept' => 'application/json',
|
@@ -200,7 +198,7 @@ module Consul
|
|
200
198
|
http_result = VaultHttpResponse.new(http, default_value)
|
201
199
|
EventMachine.add_timer(retry_in) do
|
202
200
|
yield
|
203
|
-
queue.push
|
201
|
+
queue.push
|
204
202
|
end
|
205
203
|
@e_callbacks.each { |c| c.call(http_result) }
|
206
204
|
end
|
data/lib/consul/async/version.rb
CHANGED
@@ -116,26 +116,9 @@ listen stats
|
|
116
116
|
|
117
117
|
# ==== Managed by consul-templaterb ====
|
118
118
|
|
119
|
-
frontend fe_main
|
119
|
+
frontend fe_main
|
120
120
|
bind *:80 name http
|
121
121
|
|
122
|
-
# Monitoring
|
123
|
-
monitor-uri /delivery/monitor/mesos-haproxy/lb-check
|
124
|
-
errorfile 200 /etc/haproxy/200-criteo-ok.txt
|
125
|
-
|
126
|
-
# Adding a header indicating client side is using HTTPS
|
127
|
-
acl http ssl_fc,not
|
128
|
-
acl https ssl_fc
|
129
|
-
http-request set-header X-Forwarded-Protocol https if https
|
130
|
-
|
131
|
-
# Disable CONNECT globally (identified as vulnerability by audit tools such as Qualys)
|
132
|
-
acl ::disabled_http_methods method CONNECT
|
133
|
-
http-request block if ::disabled_http_methods
|
134
|
-
|
135
|
-
# Consul-agent ACL
|
136
|
-
acl consul-agent-http-acl hdr_dom(host) -i consul-agent-http.lb-ty5.crto.in consul-agent-http.lb-ty5.central.criteo.prod
|
137
|
-
use_backend bestatic_consul-agent-http if consul-agent-http-acl
|
138
|
-
|
139
122
|
# ACLs, HTTP Host header based, following the pattern <consul_app_name>.<fqdn_suffix>
|
140
123
|
<% backends.each_pair do |service_name, nodes|
|
141
124
|
%>
|
@@ -144,10 +127,6 @@ frontend fe_main # HTTP(S) Service
|
|
144
127
|
<% end %>
|
145
128
|
|
146
129
|
# Backends
|
147
|
-
backend bestatic_consul-agent-http
|
148
|
-
http-request deny if !METH_GET # Deny writes
|
149
|
-
server srv0 127.0.0.1:8500
|
150
|
-
|
151
130
|
<% backends.each_pair do |service_name, nodes|
|
152
131
|
%>
|
153
132
|
backend be_<%= service_name %>
|
data/samples/metrics.erb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
# A template for exporting metrics for prometheus using consul-templaterb
|
2
|
+
# This template can be configure the following way with environment variables
|
3
|
+
# Environment variables to filter services/instances
|
4
|
+
# SERVICES_TAG_FILTER: basic tag filter for service (default HTTP)
|
5
|
+
# INSTANCE_MUST_TAG: Second level of filtering (optional, default to SERVICES_TAG_FILTER)
|
6
|
+
# INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag
|
7
|
+
# EXCLUDE_SERVICES: comma-separated services to exclude (example: consul-agent-http,mesos-slave,mesos-agent-watcher)
|
8
|
+
<%
|
9
|
+
|
10
|
+
service_tag_filter = ENV['SERVICES_TAG_FILTER'] || nil
|
11
|
+
instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
|
12
|
+
instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG']
|
13
|
+
|
14
|
+
# Services to hide
|
15
|
+
services_blacklist = (ENV['EXCLUDE_SERVICES'] || '').split(',')
|
16
|
+
# Compute the health of a Service
|
17
|
+
def compute_state(snode)
|
18
|
+
states = ['passing', []]
|
19
|
+
snode['Checks'].each do |chk|
|
20
|
+
st = chk['Status']
|
21
|
+
states[1] << st
|
22
|
+
if st == 'critical'
|
23
|
+
states[0] = st
|
24
|
+
elsif st == 'warning' && states[0] == 'passing'
|
25
|
+
states[0] = st
|
26
|
+
end
|
27
|
+
end
|
28
|
+
states
|
29
|
+
end
|
30
|
+
backends = {}
|
31
|
+
service_count = 0
|
32
|
+
services(tag: service_tag_filter).each do |service_name, tags|
|
33
|
+
if !services_blacklist.include?(service_name) && (instance_must_tag.nil? || tags.include?(instance_must_tag))
|
34
|
+
service_count += 1
|
35
|
+
res = {
|
36
|
+
'passing' => 0,
|
37
|
+
'warning' => 0,
|
38
|
+
'critical' => 0,
|
39
|
+
}
|
40
|
+
service(service_name).each do |snode|
|
41
|
+
state = compute_state(snode)[0]
|
42
|
+
res[state] += 1
|
43
|
+
end
|
44
|
+
backends[service_name] = res
|
45
|
+
end
|
46
|
+
end
|
47
|
+
%>
|
48
|
+
# HELP consul_datacenters_count A gauge of number of datacenters available
|
49
|
+
# TYPE consul_nodes_count gauge
|
50
|
+
consul_datacenters_count <%= datacenters().count %>
|
51
|
+
|
52
|
+
# HELP consul_nodes_count A gauge of number of nodes in the cluster
|
53
|
+
# TYPE consul_nodes_count gauge
|
54
|
+
consul_nodes_count <%= nodes().count %>
|
55
|
+
|
56
|
+
# HELP consul_services_count A gauge of the number of services
|
57
|
+
# TYPE consul_services_count gauge
|
58
|
+
consul_services_count <%= service_count %>
|
59
|
+
|
60
|
+
# HELP consul_service_count A gauge of number instances of service with their current state
|
61
|
+
# TYPE consul_service_count gauge
|
62
|
+
<%
|
63
|
+
json_backends = {}
|
64
|
+
backends.each_pair do |service_name, stats|
|
65
|
+
stats.each_pair do |state_name, state_count|
|
66
|
+
%>consul_service_count{service="<%= service_name %>",state="<%= state_name %>"} <%= state_count %>
|
67
|
+
<%
|
68
|
+
end
|
69
|
+
end
|
70
|
+
%>
|
71
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: consul-templaterb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.5.
|
4
|
+
version: 1.5.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SRE Core Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-09-
|
11
|
+
date: 2018-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: em-http-request
|
@@ -199,6 +199,7 @@ files:
|
|
199
199
|
- samples/debug/compare_connect_services.txt.erb
|
200
200
|
- samples/ha_proxy.cfg.erb
|
201
201
|
- samples/keys.html.erb
|
202
|
+
- samples/metrics.erb
|
202
203
|
- samples/nodes.html.erb
|
203
204
|
- samples/sample_keys.html.erb
|
204
205
|
- samples/services.html.erb
|