consul-templaterb 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,70 @@
1
+ <%
2
+ require 'rexml/document'
3
+
4
+ def build_node(e, name, attributes, text = nil)
5
+ xe = REXML::Element.new(name)
6
+ attributes.each do |name|
7
+ xe.add_attribute name, e[name]
8
+ end
9
+ xe.text = text if text
10
+ xe
11
+ end
12
+
13
+ def create_datacenters
14
+ xdatacenters = REXML::Element.new 'datacenters'
15
+ datacenters.each do |dc|
16
+ xdc = REXML::Element.new 'datacenter'
17
+ xdc.add_attribute 'name', dc
18
+ xdc.add_attribute 'services', services(dc:dc).keys.count
19
+ xdc.add_attribute 'nodes', nodes(dc:dc).count
20
+ xdatacenters.add_element xdc
21
+ end
22
+ xdatacenters
23
+ end
24
+
25
+ def create_services
26
+ xservices = REXML::Element.new 'services'
27
+ xservices.add_attribute 'count', services.count
28
+ services.each do |service_name, tags|
29
+ xserv = REXML::Element.new 'service'
30
+ xserv.add_attribute 'name', service_name
31
+ xserv.add_attribute 'tags', tags.sort.join(',')
32
+ xnodes = REXML::Element.new 'instances'
33
+ xnodes.add_attribute 'count', service(service_name).count
34
+ service(service_name).sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |snode|
35
+ xnode = REXML::Element.new 'service-instance'
36
+ xnode.add_attribute 'id', snode['Node']['ID'] if snode['Node']['ID']
37
+ xnode.add_attribute 'address', snode['Node']['Address']
38
+ xnode.add_attribute 'name', snode['Node']['Name']
39
+ xnode.add_attribute 'port', snode['Service']['Port'].to_i
40
+ xnode.add_attribute 'tags', snode['Service']['Tags'].sort.join(',')
41
+ xnode.add_attribute 'statuses', (snode['Checks'].map { |c| c['Status'] }.join(' '))
42
+ xnodes.add_element xnode
43
+ end
44
+ xserv.add_element xnodes
45
+ xservices.add_element xserv
46
+ end
47
+ xservices
48
+ end
49
+
50
+ def create_nodes
51
+ xnodes = REXML::Element.new 'nodes'
52
+ xnodes.add_attribute 'count', nodes.count
53
+ nodes.sort {|a,b| a['Node'] <=> b['Node'] }.each do |snode|
54
+ xnode = REXML::Element.new 'node'
55
+ xnode.add_attribute 'id', snode['ID']
56
+ xnode.add_attribute 'address', snode['Address']
57
+ xnode.add_attribute 'name', snode['Node']
58
+ xnodes.add xnode
59
+ end
60
+ xnodes
61
+ end
62
+ doc = REXML::Document.new '<consul-info/>'
63
+ doc << REXML::XMLDecl.new(version='1.0', 'UTF-8')
64
+
65
+ doc.root.add_element create_datacenters
66
+ doc.root.add_element create_services
67
+ doc.root.add_element create_nodes
68
+ result = ''
69
+ doc.write(result, 2)
70
+ %><%= result %>
@@ -0,0 +1,163 @@
1
+ <%
2
+ # This template can be configure the following way with environment variables
3
+ # Environment variables to filter services/instances
4
+ # SERVICES_TAG_FILTER: basic tag filter for service (default HTTP)
5
+ # INSTANCE_MUST_TAG: Second level of filtering (optional, default to SERVICES_TAG_FILTER)
6
+ # INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag (default: canary)
7
+ # EXCLUDE_SERVICES: comma-separated services to exclude (default: consul-agent-http,mesos-slave,mesos-agent-watcher)
8
+ # Environment variables to configure HaProxy
9
+ # PORT0 : HaProxy main port (default to 8000)
10
+ # PORT1 : HaProxy admin port (default to 8001)
11
+ # HAPROXY_ADMIN_PORT: Port to listen
12
+ # HAPROXY_ADMIN_USER_PASSWORD: User:Password to access to stats (default: none)
13
+ # HAPROXY_SOCKET_FILE: HaProxy socket to control HaProxy
14
+
15
+ service_tag_filter = ENV['SERVICES_TAG_FILTER'] || 'http'
16
+ instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
17
+ instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG'] || 'canary'
18
+ kv_root = ENV['KV_ROOT'] || "services-data"
19
+ optional_dns_suffix = ENV['DNS_SUFFIX'] || ''
20
+
21
+ # Services to hide
22
+ services_blacklist = (ENV['EXCLUDE_SERVICES'] || 'consul-agent-http,mesos-slave,mesos-agent-watcher,mesos-exporter-slave').split(',')
23
+ # Compute the health of a Service
24
+ def compute_state(node)
25
+ states = ['passing', []]
26
+ node['Checks'].each do |chk|
27
+ st = chk['Status']
28
+ states[1] << st
29
+ states[0] = st if st == 'critical' || (st == 'warning' && states[0] == 'passing')
30
+ end
31
+ states
32
+ end
33
+ def compute_attributes(node)
34
+ w = 100
35
+ node['Service']['Tags'].each do |tag|
36
+ match = /^weight-([1-9][0-9])*$/.match(tag)
37
+ w = match[1].to_i if match
38
+ end
39
+ attributes = ""
40
+ states = compute_state(node)
41
+ attributes = "#{attributes} disabled" if states[0] == 'critical'
42
+ if states[0] == 'warning'
43
+ w = w / 8
44
+ end
45
+ attributes = "#{attributes} weight #{w}" if w.positive?
46
+ end
47
+ backends = {}
48
+ all_kv = {}
49
+ services(tag: service_tag_filter).each do |service_name, tags|
50
+ if !services_blacklist.include?(service_name) && tags.include?(instance_must_tag)
51
+ the_backends = []
52
+ kv_data = kv("#{kv_root}/#{service_name}/network-service").get_value_json
53
+ all_kv[service_name] = kv_data if kv_data
54
+ service(service_name, tag:'http').sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |node|
55
+ tags_of_instance = node['Service']['Tags']
56
+ if tags_of_instance.include?(instance_must_tag) && !tags_of_instance.include?(instance_exclude_tag)
57
+ the_backends << node if node['Service']['Port']
58
+ end
59
+ end
60
+ # We add the backend ONLY if at least one valid instance does exists
61
+ backends[service_name] = the_backends
62
+ end
63
+ end
64
+
65
+ @all_kv = all_kv
66
+ def find_alternative_fqdns(service_name)
67
+ json_kv = @all_kv[service_name]
68
+ return nil unless json_kv
69
+ return nil unless json_kv['alternative_fqdn']
70
+ # Add spaces before
71
+ " #{json_kv['alternative_fqdn'].join(' ')}"
72
+ end
73
+ %>global
74
+ log 127.0.0.1 local0 info
75
+ maxconn 262144
76
+ user haproxy
77
+ group haproxy
78
+ nbproc 1
79
+ daemon
80
+ stats socket /var/lib/haproxy/stats level admin mode 644 expose-fd listeners
81
+ stats timeout 2m
82
+ tune.bufsize 33792
83
+ ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
84
+ ssl-default-bind-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
85
+ ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
86
+ ssl-default-server-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
87
+ ssl-server-verify required
88
+
89
+ defaults
90
+ log global
91
+ mode http
92
+ retries 3
93
+ timeout connect 10s
94
+ timeout client 30s
95
+ timeout server 30s
96
+ timeout http-keep-alive 10s
97
+ timeout http-request 10s
98
+ timeout queue 1s
99
+ timeout check 5s
100
+ option httplog
101
+ option dontlognull
102
+ option redispatch
103
+ option prefer-last-server
104
+ option dontlog-normal
105
+ option http-keep-alive
106
+ option forwardfor except 127.0.0.0/8
107
+ balance roundrobin
108
+ maxconn 262134
109
+ http-reuse safe
110
+ default-server inter 5s fastinter 1s fall 3 slowstart 20s
111
+
112
+ listen stats
113
+ bind *:8080
114
+ stats enable
115
+ stats uri /haproxy_stats
116
+
117
+ # ==== Managed by consul-templaterb ====
118
+
119
+ frontend fe_main # HTTP(S) Service
120
+ bind *:80 name http
121
+
122
+ # Monitoring
123
+ monitor-uri /delivery/monitor/mesos-haproxy/lb-check
124
+ errorfile 200 /etc/haproxy/200-criteo-ok.txt
125
+
126
+ # Adding a header indicating client side is using HTTPS
127
+ acl http ssl_fc,not
128
+ acl https ssl_fc
129
+ http-request set-header X-Forwarded-Protocol https if https
130
+
131
+ # Disable CONNECT globally (identified as vulnerability by audit tools such as Qualys)
132
+ acl ::disabled_http_methods method CONNECT
133
+ http-request block if ::disabled_http_methods
134
+
135
+ # Consul-agent ACL
136
+ acl consul-agent-http-acl hdr_dom(host) -i consul-agent-http.lb-ty5.crto.in consul-agent-http.lb-ty5.central.criteo.prod
137
+ use_backend bestatic_consul-agent-http if consul-agent-http-acl
138
+
139
+ # ACLs, HTTP Host header based, following the pattern <consul_app_name>.<fqdn_suffix>
140
+ <% backends.each_pair do |service_name, nodes|
141
+ %>
142
+ acl <%= service_name %>-acl hdr_beg(host) -i <%= service_name %>.<%= optional_dns_suffix %><%= find_alternative_fqdns(service_name)%>
143
+ use_backend be_<%= service_name %> if <%= service_name%>-acl
144
+ <% end %>
145
+
146
+ # Backends
147
+ backend bestatic_consul-agent-http
148
+ http-request deny if !METH_GET # Deny writes
149
+ server srv0 127.0.0.1:8500
150
+
151
+ <% backends.each_pair do |service_name, nodes|
152
+ %>
153
+ backend be_<%= service_name %>
154
+ <% if all_kv[service_name] %>
155
+ # data2: <%= all_kv[service_name] %>
156
+ <% end %>
157
+ mode http
158
+ balance leastconn
159
+ <% nodes.each_with_index do |node, idx|
160
+ %> server srv<%= idx %> <%= node['Node']['Address']%>:<%= node['Service']['Port'] %><%= compute_attributes(node) %>
161
+ <% end
162
+ end
163
+ %>
@@ -0,0 +1,91 @@
1
+ <% require 'base64'
2
+ require 'json'
3
+ require 'date'
4
+ @current_time = Time.now.utc
5
+ %>
6
+ <%= render_file 'common/header.html.erb' %>
7
+ <main role="main" class="container">
8
+ <div>
9
+ <h1>Show all choregraphie information</h1>
10
+ <div>This page only show choregraphie when at least one holder exists.</div>
11
+ <%
12
+ def display_holder(holder, value)
13
+ begin
14
+ holder_date = Time.parse(value)
15
+ status = 'success'
16
+ diff = (@current_time - holder_date).round(0)
17
+ status = 'warning' if diff > 3600
18
+ status = 'danger' if diff > 7200
19
+ diff_txt = "#{diff % 3600} seconds"
20
+ diff_txt = "#{(diff % 86400) / 3600} hours, #{diff_txt}" if diff > 3600
21
+ diff_txt = "#{diff / 86400} days, #{diff_txt}" if diff > 86400
22
+ rescue StandardError => e
23
+ status = 'info'
24
+ holder_date = "Cannot parse date: #{e}"
25
+ diff_txt = "Error Parsing date #{value}"
26
+ end
27
+ ["<li class=\"list-group-item list-group-item-#{status}\">#{holder}: #{holder_date} : #{diff_txt} ago", status]
28
+ end
29
+ %>
30
+
31
+ <div id="accordion">
32
+ <% kv('choregraphie', recurse:true).each do |tuple| %>
33
+ <!-- <%= tuple['Key'] %> -->
34
+ <%
35
+ key = tuple['Key'].gsub('/', '-')
36
+ if tuple['Value'].nil?
37
+ json = []
38
+ holders = []
39
+ %>
40
+ <div class="alert alert-warning" role="alert">
41
+ Invalid Choregraphie data: <%= tuple %>
42
+ </div>
43
+ <%
44
+ else
45
+ json = JSON.parse(Base64.decode64(tuple['Value']))
46
+ holders = json['holders']
47
+ end
48
+ %>
49
+ <% if holders.count > 0 %>
50
+
51
+ <div class="card">
52
+ <div class="card-header" id="heading-<%= key %>">
53
+ <span class="badge badge-pill badge-primary float-right"><%= holders.count%>/<%= json['concurrency'] %></span>
54
+ <h5 class="mb-0">
55
+ <a href="#<%= key %>" class="btn btn-link collapsed" data-toggle="collapse" data-target="#collapse-<%= key %>" aria-expanded="true" aria-controls="collapse-<%= key %>">
56
+ <%= tuple['Key'] %>
57
+ </a>
58
+ </h5>
59
+ </div>
60
+ <%
61
+ text_result = ''
62
+ clazz = 'collapse'
63
+ holders.each_pair do |key,value|
64
+ if value.is_a?(Hash)
65
+ text_result+= "<li><u>#{key}:</u><ul class=\"list-group\">"
66
+ value.each_pair do |k, v|
67
+ res, status = display_holder(k, v)
68
+ clazz = 'collapse show' unless status == 'success'
69
+ text_result += res
70
+ end
71
+ text_result += '</ul></li>'
72
+ else
73
+ res, status = display_holder(key, value)
74
+ clazz = 'collapse show' unless status == 'success'
75
+ text_result << res
76
+ end
77
+ end
78
+ %>
79
+ <div id="collapse-<%= key %>" class="<%= clazz %>" aria-labelledby="heading-<%= key %>" data-parent="#accordion">
80
+ <div class="card-body">
81
+ <pre class="pre-scrollable"><code><%= JSON.pretty_generate(json) %></code></pre>
82
+ <ul class="list-group">
83
+ <%= text_result %>
84
+ </ul>
85
+ </div>
86
+ </div>
87
+ </div>
88
+ <% end %>
89
+ <% end %>
90
+ </div>
91
+ <%= render_file 'common/footer.html.erb' %>
@@ -0,0 +1,127 @@
1
+ <%
2
+ # This template can be configure the following way with environment variables
3
+ # Environment variables to filter services/instances
4
+ # SERVICES_TAG_FILTER: basic tag filter for service (default HTTP)
5
+ # INSTANCE_MUST_TAG: Second level of filtering (optional, default to SERVICES_TAG_FILTER)
6
+ # INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag (default: canary)
7
+ # EXCLUDE_SERVICES: comma-separated services to exclude (default: consul-agent-http,mesos-slave,mesos-agent-watcher)
8
+ # Environment variables to configure HaProxy
9
+ # PORT0 : HaProxy main port (default to 8000)
10
+ # PORT1 : HaProxy admin port (default to 8001)
11
+ # HAPROXY_ADMIN_PORT: Port to listen
12
+ # HAPROXY_ADMIN_USER_PASSWORD: User:Password to access to stats (default: none)
13
+ # HAPROXY_SOCKET_FILE: HaProxy socket to control HaProxy
14
+
15
+ service_tag_filter = ENV['SERVICES_TAG_FILTER'] || 'http'
16
+ instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
17
+ instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG'] || 'canary'
18
+
19
+ # Services to hide
20
+ services_blacklist = (ENV['EXCLUDE_SERVICES'] || 'consul-agent-http,mesos-slave,mesos-agent-watcher,mesos-exporter-slave').split(',')
21
+ # Compute the health of a Service
22
+ def compute_state(node)
23
+ states = ['passing', []]
24
+ node['Checks'].each do |chk|
25
+ st = chk['Status']
26
+ states[1] << st
27
+ if st == 'critical'
28
+ states[0] = st
29
+ elsif st == 'warning' && states[0] == 'passing'
30
+ states[0] = st
31
+ end
32
+ end
33
+ states
34
+ end
35
+ def compute_attributes(node)
36
+ w = 100
37
+ node['Service']['Tags'].each do |tag|
38
+ match = /^weight-([1-9][0-9])*$/.match(tag)
39
+ w = match[1].to_i if match
40
+ end
41
+ attributes = ""
42
+ states = compute_state(node)
43
+ attributes = "#{attributes} disabled" if states[0] == 'critical'
44
+ if states[0] == 'warning'
45
+ w = w / 8
46
+ end
47
+ attributes = "#{attributes} weight #{w}" if w.positive?
48
+ end
49
+ backends = {}
50
+ services(tag: service_tag_filter).each do |service_name, tags|
51
+ if !services_blacklist.include?(service_name) && tags.include?(instance_must_tag)
52
+ the_backends = []
53
+ service(service_name, tag:'http').sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |node|
54
+ tags_of_instance = node['Service']['Tags']
55
+ if tags_of_instance.include?(instance_must_tag) && !tags_of_instance.include?(instance_exclude_tag)
56
+ the_backends << node if node['Service']['Port']
57
+ end
58
+ end
59
+ # We add the backend ONLY if at least one valid instance does exists
60
+ backends[service_name] = the_backends
61
+ end
62
+ end
63
+ %>
64
+ # This HaProxy configuration is generated by consul-templaterb
65
+ # Service/Instances filtering
66
+ # Service tag MUST be <%= service_tag_filter %>
67
+ # Instance tag MUST be <%= instance_must_tag %>
68
+ # Instance tag excluded: <%= instance_exclude_tag %>
69
+
70
+ # Found <%= backends.keys.count %> different services with <%= backends.values.flatten.count %> nodes
71
+
72
+ global
73
+ maxconn 1024
74
+ <%
75
+ socket_file = ENV['HAPROXY_SOCKET_FILE']
76
+ if socket_file
77
+ %>
78
+ stats socket <%= socket_file %> mode 600 level admin
79
+ <% end %>
80
+
81
+ defaults
82
+ timeout connect 1000
83
+ timeout client 5000
84
+ timeout server 15000
85
+ <% backends.each_pair do |service_name, nodes|
86
+ %>
87
+ backend backend_http__<%= service_name %>
88
+ mode http
89
+ balance leastconn
90
+ <% nodes.each do |node|
91
+ %> server <%= node['Node']['Node']%>_<%= node['Service']['Port'] %> <%= node['Node']['Address']%>:<%= node['Service']['Port'] %><%= compute_attributes(node) %>
92
+ <% end
93
+ end
94
+ %>
95
+ frontend fontend_http
96
+ mode http
97
+ bind *:<%= ENV['PORT0'] || 8000 %>
98
+ option httplog
99
+ log 127.0.0.1:534 local5
100
+ option dontlognull
101
+ option forwardfor
102
+ <% backends.each_pair do |service_name, nodes|
103
+ %>
104
+ acl host__<%= service_name %> hdr_beg(host) -i <%= service_name %>.
105
+ use_backend backend_http__<%= service_name %> if host__<%= service_name%>
106
+ <% end %>
107
+
108
+ listen stats
109
+ bind :<%= ENV['PORT1'] || 8001 %>
110
+ mode http
111
+ log global
112
+ maxconn 10
113
+ timeout client 100s
114
+ timeout server 100s
115
+ timeout connect 100s
116
+ timeout queue 100s
117
+ stats enable
118
+ stats hide-version
119
+ stats refresh 30s
120
+ stats show-node
121
+ stats uri /haproxy?stats
122
+ <%
123
+ user_password = ENV['HAPROXY_ADMIN_USER_PASSWORD']
124
+ if user_password
125
+ %>
126
+ stats auth <%= user_password %>
127
+ <% end %>