consul-templaterb 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +25 -0
- data/.gitreview +5 -0
- data/.rspec +2 -0
- data/.rubocop.yml +43 -0
- data/.ruby_app +0 -0
- data/.travis.yml +13 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +201 -0
- data/README.md +270 -0
- data/Rakefile +6 -0
- data/bin/consul-templaterb +246 -0
- data/consul-templaterb.gemspec +37 -0
- data/lib/consul/async/consul_endpoint.rb +279 -0
- data/lib/consul/async/consul_template.rb +323 -0
- data/lib/consul/async/consul_template_engine.rb +57 -0
- data/lib/consul/async/consul_template_render.rb +80 -0
- data/lib/consul/async/process_handler.rb +64 -0
- data/lib/consul/async/utilities.rb +17 -0
- data/lib/consul/async/version.rb +5 -0
- data/samples/checks.html.erb +96 -0
- data/samples/common/footer.html.erb +10 -0
- data/samples/common/header.html.erb +51 -0
- data/samples/consul_template.html.erb +94 -0
- data/samples/consul_template.json.erb +77 -0
- data/samples/consul_template.txt.erb +45 -0
- data/samples/consul_template.xml.erb +70 -0
- data/samples/criteo/haproxy.cfg.erb +163 -0
- data/samples/criteo_choregraphies.html.erb +91 -0
- data/samples/ha_proxy.cfg.erb +127 -0
- data/samples/keys.html.erb +38 -0
- data/samples/nodes.html.erb +17 -0
- data/samples/services.html.erb +89 -0
- metadata +189 -0
@@ -0,0 +1,323 @@
|
|
1
|
+
require 'consul/async/utilities'
|
2
|
+
require 'em-http'
|
3
|
+
require 'thread'
|
4
|
+
require 'forwardable'
|
5
|
+
require 'erb'
|
6
|
+
module Consul
|
7
|
+
module Async
|
8
|
+
class InvalidTemplateException < StandardError
|
9
|
+
attr_reader :cause
|
10
|
+
def initialize(cause)
|
11
|
+
@cause = cause
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class ConsulEndPointsManager
|
16
|
+
attr_reader :conf, :net_info, :start_time
|
17
|
+
def initialize(consul_configuration)
|
18
|
+
@conf = consul_configuration
|
19
|
+
@endpoints = {}
|
20
|
+
@iteration = 1
|
21
|
+
@start_time = Time.now.utc
|
22
|
+
@net_info = {
|
23
|
+
success: 0,
|
24
|
+
errors: 0,
|
25
|
+
bytes_read: 0
|
26
|
+
}
|
27
|
+
end
|
28
|
+
|
29
|
+
# https://www.consul.io/api/health.html#list-nodes-for-service
|
30
|
+
def service(name, dc: nil, passing: false, tag: nil)
|
31
|
+
raise 'You must specify a name for a service' if name.nil?
|
32
|
+
path = "/v1/health/service/#{name}"
|
33
|
+
query_params = {}
|
34
|
+
query_params[:dc] = dc if dc
|
35
|
+
query_params[:passing] = passing if passing
|
36
|
+
query_params[:tag] = tag if tag
|
37
|
+
create_if_missing(path, query_params) { ConsulTemplateService.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
|
38
|
+
end
|
39
|
+
|
40
|
+
# https://www.consul.io/api/health.html#list-checks-for-service
|
41
|
+
def checks_for_service(name, dc: nil, passing: false)
|
42
|
+
raise 'You must specify a name for a service' if name.nil?
|
43
|
+
path = "/v1/health/checks/#{name}"
|
44
|
+
query_params = {}
|
45
|
+
query_params[:dc] = dc if dc
|
46
|
+
query_params[:passing] = passing if passing
|
47
|
+
create_if_missing(path, query_params) { ConsulTemplateChecks.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
|
48
|
+
end
|
49
|
+
|
50
|
+
# https://www.consul.io/api/catalog.html#list-nodes
|
51
|
+
def nodes(dc: nil)
|
52
|
+
path = '/v1/catalog/nodes'
|
53
|
+
query_params = {}
|
54
|
+
query_params[:dc] = dc if dc
|
55
|
+
create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
|
56
|
+
end
|
57
|
+
|
58
|
+
# https://www.consul.io/api/catalog.html#list-services-for-node
|
59
|
+
def node(name_or_id, dc: nil)
|
60
|
+
path = "/v1/catalog/node/#{name_or_id}"
|
61
|
+
query_params = {}
|
62
|
+
query_params[:dc] = dc if dc
|
63
|
+
create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(conf, path, true, query_params, '{}')) }
|
64
|
+
end
|
65
|
+
|
66
|
+
# https://www.consul.io/api/agent.html#read-configuration
|
67
|
+
def agent_self
|
68
|
+
path = '/v1/agent/self'
|
69
|
+
query_params = {}
|
70
|
+
default_value = '{"Config":{}, "Coord":{}, "Member":{}, "Meta":{}, "Stats":{}}'
|
71
|
+
create_if_missing(path, query_params) { ConsulAgentSelf.new(ConsulEndpoint.new(conf, path, true, query_params, default_value)) }
|
72
|
+
end
|
73
|
+
|
74
|
+
# https://www.consul.io/api/agent.html#view-metrics
|
75
|
+
def agent_metrics
|
76
|
+
path = '/v1/agent/metrics'
|
77
|
+
query_params = {}
|
78
|
+
default_value = '{"Gauges":[], "Points":[], "Member":{}, "Counters":[], "Samples":{}}'
|
79
|
+
create_if_missing(path, query_params) { ConsulAgentMetrics.new(ConsulEndpoint.new(conf, path, true, query_params, default_value)) }
|
80
|
+
end
|
81
|
+
|
82
|
+
# https://www.consul.io/api/catalog.html#list-services
|
83
|
+
def services(dc: nil, tag: nil)
|
84
|
+
path = '/v1/catalog/services'
|
85
|
+
query_params = {}
|
86
|
+
query_params[:dc] = dc if dc
|
87
|
+
# Tag filtering is performed on client side
|
88
|
+
query_params[:tag] = tag if tag
|
89
|
+
create_if_missing(path, query_params) { ConsulTemplateServices.new(ConsulEndpoint.new(conf, path, true, query_params, '{}')) }
|
90
|
+
end
|
91
|
+
|
92
|
+
# https://www.consul.io/api/catalog.html#list-datacenters
|
93
|
+
def datacenters
|
94
|
+
path = '/v1/catalog/datacenters'
|
95
|
+
query_params = {}
|
96
|
+
create_if_missing(path, query_params) { ConsulTemplateDatacenters.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
|
97
|
+
end
|
98
|
+
|
99
|
+
# https://www.consul.io/api/kv.html#read-key
|
100
|
+
def kv(name = nil, dc: nil, keys: nil, recurse: false)
|
101
|
+
path = "/v1/kv/#{name}"
|
102
|
+
query_params = {}
|
103
|
+
query_params[:dc] = dc if dc
|
104
|
+
query_params[:recurse] = recurse if recurse
|
105
|
+
query_params[:keys] = keys if keys
|
106
|
+
default_value = '[]'
|
107
|
+
create_if_missing(path, query_params) { ConsulTemplateKV.new(ConsulEndpoint.new(conf, path, true, query_params, default_value), name) }
|
108
|
+
end
|
109
|
+
|
110
|
+
def render_file(path)
|
111
|
+
new_path = File.expand_path(path, File.dirname(@current_erb_path))
|
112
|
+
raise "render_file ERROR: #{path} is resolved as #{new_path}, but the file does not exists" unless File.exist? new_path
|
113
|
+
render(File.read(new_path), new_path)
|
114
|
+
end
|
115
|
+
|
116
|
+
def render(tpl, tpl_file_path)
|
117
|
+
# Ugly, but allow to use render_file well to support stack of calls
|
118
|
+
old_value = @current_erb_path
|
119
|
+
@current_erb_path = tpl_file_path
|
120
|
+
result = ERB.new(tpl).result(binding)
|
121
|
+
@current_erb_path = old_value
|
122
|
+
result
|
123
|
+
rescue StandardError => e
|
124
|
+
e2 = InvalidTemplateException.new e
|
125
|
+
raise e2, "Template contains errors: #{e.message}", e.backtrace
|
126
|
+
end
|
127
|
+
|
128
|
+
def write(file, tpl, last_result, tpl_file_path)
|
129
|
+
data = render(tpl, tpl_file_path)
|
130
|
+
not_ready = []
|
131
|
+
ready = 0
|
132
|
+
@iteration = Time.now.utc - @start_time
|
133
|
+
to_cleanup = []
|
134
|
+
@endpoints.each_pair do |endpoint_key, endpt|
|
135
|
+
if endpt.ready?
|
136
|
+
ready += 1
|
137
|
+
else
|
138
|
+
not_ready << endpt.endpoint.path
|
139
|
+
end
|
140
|
+
to_cleanup << endpoint_key if (@iteration - endpt.seen_at) > 10
|
141
|
+
end
|
142
|
+
if not_ready.count.positive?
|
143
|
+
STDERR.print "[INFO] Waiting for data from #{not_ready.count}/#{not_ready.count + ready} endpoints: #{not_ready[0..2]}..."
|
144
|
+
return [false, false, '']
|
145
|
+
end
|
146
|
+
if to_cleanup.count > 1
|
147
|
+
STDERR.puts "[INFO] Cleaned up #{to_cleanup.count} endpoints: #{to_cleanup}"
|
148
|
+
to_cleanup.each do |to_remove|
|
149
|
+
x = @endpoints.delete(to_remove)
|
150
|
+
x.endpoint.terminate
|
151
|
+
end
|
152
|
+
end
|
153
|
+
if last_result != data
|
154
|
+
STDERR.print "[INFO] Write #{Utilities.bytes_to_h data.bytesize} bytes to #{file}, "\
|
155
|
+
"netinfo=#{@net_info} aka "\
|
156
|
+
"#{Utilities.bytes_to_h((net_info[:bytes_read] / (Time.now.utc - @start_time)).round(1))}/s ...\n"
|
157
|
+
tmp_file = "#{file}.tmp"
|
158
|
+
File.open(tmp_file, 'w') do |f|
|
159
|
+
f.write data
|
160
|
+
end
|
161
|
+
File.rename(tmp_file, file)
|
162
|
+
end
|
163
|
+
[true, data != last_result, data]
|
164
|
+
end
|
165
|
+
|
166
|
+
def terminate
|
167
|
+
@endpoints.each_value do |v|
|
168
|
+
v.endpoint.terminate
|
169
|
+
end
|
170
|
+
@endpoints = {}
|
171
|
+
end
|
172
|
+
|
173
|
+
def create_if_missing(path, query_params)
|
174
|
+
fqdn = path.dup
|
175
|
+
query_params.each_pair do |k, v|
|
176
|
+
fqdn = "#{fqdn}&#{k}=#{v}"
|
177
|
+
end
|
178
|
+
tpl = @endpoints[fqdn]
|
179
|
+
unless tpl
|
180
|
+
tpl = yield
|
181
|
+
STDERR.print "[INFO] path #{path.ljust(64)} #{query_params.inspect}\r"
|
182
|
+
@endpoints[fqdn] = tpl
|
183
|
+
tpl.endpoint.on_response do |result|
|
184
|
+
@net_info[:success] = @net_info[:success] + 1
|
185
|
+
@net_info[:bytes_read] = @net_info[:bytes_read] + result.data.bytesize
|
186
|
+
end
|
187
|
+
tpl.endpoint.on_error { @net_info[:errors] = @net_info[:errors] + 1 }
|
188
|
+
end
|
189
|
+
tpl._seen_at(@iteration)
|
190
|
+
tpl
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
class ConsulTemplateAbstract
|
195
|
+
extend Forwardable
|
196
|
+
def_delegators :result_delegate, :each, :[], :sort, :each_value, :count, :empty?
|
197
|
+
attr_reader :result, :endpoint, :seen_at
|
198
|
+
def initialize(consul_endpoint)
|
199
|
+
@endpoint = consul_endpoint
|
200
|
+
consul_endpoint.on_response do |res|
|
201
|
+
@result = parse_result(res)
|
202
|
+
end
|
203
|
+
@result = parse_result(consul_endpoint.last_result)
|
204
|
+
end
|
205
|
+
|
206
|
+
def _seen_at(val)
|
207
|
+
@seen_at = val
|
208
|
+
end
|
209
|
+
|
210
|
+
def ready?
|
211
|
+
@endpoint.ready?
|
212
|
+
end
|
213
|
+
|
214
|
+
protected
|
215
|
+
|
216
|
+
def result_delegate
|
217
|
+
result.json
|
218
|
+
end
|
219
|
+
|
220
|
+
def parse_result(res)
|
221
|
+
res
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
class ConsulTemplateAbstractMap < ConsulTemplateAbstract
|
226
|
+
def_delegators :result_delegate, :each, :[], :keys, :sort, :values, :each_pair, :each_value
|
227
|
+
def initialize(consul_endpoint)
|
228
|
+
super(consul_endpoint)
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
232
|
+
class ConsulTemplateAbstractArray < ConsulTemplateAbstract
|
233
|
+
def initialize(consul_endpoint)
|
234
|
+
super(consul_endpoint)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
class ConsulTemplateService < ConsulTemplateAbstractMap
|
239
|
+
def initialize(consul_endpoint)
|
240
|
+
super(consul_endpoint)
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
class ConsulTemplateDatacenters < ConsulTemplateAbstractArray
|
245
|
+
def initialize(consul_endpoint)
|
246
|
+
super(consul_endpoint)
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
class ConsulTemplateServices < ConsulTemplateAbstractMap
|
251
|
+
def initialize(consul_endpoint)
|
252
|
+
super(consul_endpoint)
|
253
|
+
end
|
254
|
+
|
255
|
+
def parse_result(res)
|
256
|
+
return res unless res.data == '{}' || endpoint.query_params[:tag]
|
257
|
+
res_json = JSON.parse(res.data)
|
258
|
+
result = {}
|
259
|
+
res_json.each do |name, tags|
|
260
|
+
result[name] = tags if tags.include? endpoint.query_params[:tag]
|
261
|
+
end
|
262
|
+
res.mutate(JSON.generate(result))
|
263
|
+
res
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
class ConsulAgentSelf < ConsulTemplateAbstractMap
|
268
|
+
def initialize(consul_endpoint)
|
269
|
+
super(consul_endpoint)
|
270
|
+
end
|
271
|
+
end
|
272
|
+
|
273
|
+
class ConsulAgentMetrics < ConsulTemplateAbstractMap
|
274
|
+
def initialize(consul_endpoint)
|
275
|
+
super(consul_endpoint)
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
class ConsulTemplateChecks < ConsulTemplateAbstractArray
|
280
|
+
def initialize(consul_endpoint)
|
281
|
+
super(consul_endpoint)
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
class ConsulTemplateNodes < ConsulTemplateAbstractArray
|
286
|
+
def initialize(consul_endpoint)
|
287
|
+
super(consul_endpoint)
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
class ConsulTemplateKV < ConsulTemplateAbstractArray
|
292
|
+
attr_reader :root
|
293
|
+
def initialize(consul_endpoint, root)
|
294
|
+
@root = root
|
295
|
+
super(consul_endpoint)
|
296
|
+
end
|
297
|
+
|
298
|
+
def find(name = root)
|
299
|
+
res = result_delegate.find { |k| name == k['Key'] }
|
300
|
+
res || {}
|
301
|
+
end
|
302
|
+
|
303
|
+
# Get the raw value (might be base64 encoded)
|
304
|
+
def get_value(name = root)
|
305
|
+
find(name)['Value']
|
306
|
+
end
|
307
|
+
|
308
|
+
# Get the Base64 Decoded value
|
309
|
+
def get_value_decoded(name = root)
|
310
|
+
val = get_value(name)
|
311
|
+
return nil unless val
|
312
|
+
Base64.decode64(val)
|
313
|
+
end
|
314
|
+
|
315
|
+
# Helper to get the value decoded as JSON
|
316
|
+
def get_value_json(name = root)
|
317
|
+
x = get_value_decoded(name)
|
318
|
+
return nil unless x
|
319
|
+
JSON.parse(x)
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'consul/async/utilities'
|
2
|
+
require 'consul/async/consul_endpoint'
|
3
|
+
require 'consul/async/consul_template'
|
4
|
+
require 'consul/async/consul_template_render'
|
5
|
+
require 'em-http'
|
6
|
+
require 'thread'
|
7
|
+
require 'erb'
|
8
|
+
module Consul
|
9
|
+
module Async
|
10
|
+
class ConsulTemplateEngine
|
11
|
+
attr_reader :template_manager, :hot_reload_failure
|
12
|
+
attr_writer :hot_reload_failure
|
13
|
+
def initialize
|
14
|
+
@templates = []
|
15
|
+
@template_callbacks = []
|
16
|
+
@hot_reload_failure = 'die'
|
17
|
+
end
|
18
|
+
|
19
|
+
def add_template_callback(&block)
|
20
|
+
@template_callbacks << block
|
21
|
+
end
|
22
|
+
|
23
|
+
def add_template(source, dest)
|
24
|
+
@templates.push([source, dest])
|
25
|
+
end
|
26
|
+
|
27
|
+
def run(template_manager)
|
28
|
+
@template_manager = template_manager
|
29
|
+
EventMachine.run do
|
30
|
+
template_renders = []
|
31
|
+
@templates.each do |template_file, output_file|
|
32
|
+
template_renders << Consul::Async::ConsulTemplateRender.new(template_manager, template_file, output_file,
|
33
|
+
hot_reload_failure: hot_reload_failure)
|
34
|
+
end
|
35
|
+
EventMachine.add_periodic_timer(1) do
|
36
|
+
begin
|
37
|
+
results = template_renders.map(&:run)
|
38
|
+
all_ready = results.reduce(true) { |a, e| a && e.ready? }
|
39
|
+
begin
|
40
|
+
@template_callbacks.each do |c|
|
41
|
+
c.call([all_ready, template_manager, results])
|
42
|
+
end
|
43
|
+
rescue StandardError => cbk_error
|
44
|
+
STDERR.puts "Error in callback: #{cbk_error.inspect}"
|
45
|
+
raise cbk_error
|
46
|
+
end
|
47
|
+
rescue StandardError => e
|
48
|
+
STDERR.puts "[ERROR] Fatal error occured: #{e.inspect} - #{e.backtrace}"
|
49
|
+
template_manager.terminate
|
50
|
+
EventMachine.stop
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'consul/async/utilities'
|
2
|
+
require 'em-http'
|
3
|
+
require 'thread'
|
4
|
+
require 'erb'
|
5
|
+
module Consul
|
6
|
+
module Async
|
7
|
+
class ConsulTemplateRenderedResult
|
8
|
+
attr_reader :template_file, :output_file, :hot_reloaded, :ready, :modified, :last_result
|
9
|
+
def initialize(template_file, output_file, hot_reloaded, was_success, modified, last_result)
|
10
|
+
@template_file = template_file
|
11
|
+
@output_file = output_file
|
12
|
+
@hot_reloaded = hot_reloaded
|
13
|
+
@ready = was_success
|
14
|
+
@modified = modified
|
15
|
+
@last_result = last_result
|
16
|
+
end
|
17
|
+
|
18
|
+
def ready?
|
19
|
+
@ready
|
20
|
+
end
|
21
|
+
end
|
22
|
+
class ConsulTemplateRender
|
23
|
+
attr_reader :template_file, :output_file, :template_file_ctime, :hot_reload_failure
|
24
|
+
def initialize(template_manager, template_file, output_file, hot_reload_failure: 'die')
|
25
|
+
@hot_reload_failure = hot_reload_failure
|
26
|
+
@template_file = template_file
|
27
|
+
@output_file = output_file
|
28
|
+
@template_manager = template_manager
|
29
|
+
@last_result = ''
|
30
|
+
@template = load_template
|
31
|
+
end
|
32
|
+
|
33
|
+
def render(tpl = @template)
|
34
|
+
@template_manager.render(tpl, template_file)
|
35
|
+
end
|
36
|
+
|
37
|
+
def run
|
38
|
+
hot_reloaded = hot_reload_if_needed
|
39
|
+
was_success, modified, last_result = write
|
40
|
+
ConsulTemplateRenderedResult.new(template_file, output_file, hot_reloaded, was_success, modified, last_result)
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def load_template
|
46
|
+
@template_file_ctime = File.ctime(template_file)
|
47
|
+
File.read(template_file)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Will throw Consul::Async::InvalidTemplateException if template invalid
|
51
|
+
def update_template(new_template)
|
52
|
+
return false unless new_template != @template
|
53
|
+
# We render to ensure the template is valid
|
54
|
+
render(new_template)
|
55
|
+
@template = new_template.freeze
|
56
|
+
true
|
57
|
+
end
|
58
|
+
|
59
|
+
def write
|
60
|
+
success, modified, @last_result = @template_manager.write(@output_file, @template, @last_result, template_file)
|
61
|
+
[success, modified, @last_result]
|
62
|
+
end
|
63
|
+
|
64
|
+
def hot_reload_if_needed
|
65
|
+
new_time = File.ctime(template_file)
|
66
|
+
if template_file_ctime != new_time
|
67
|
+
begin
|
68
|
+
@template_file_ctime = new_time
|
69
|
+
return update_template(load_template)
|
70
|
+
rescue Consul::Async::InvalidTemplateException => e
|
71
|
+
STDERR.puts "****\n[ERROR] HOT Reload of template #{template_file} did fail due to #{e}\n****\n"
|
72
|
+
raise e unless hot_reload_failure == 'keep'
|
73
|
+
STDERR.puts "[WARN] Hot reload of #{template_file} was not taken into account, keep running with previous version"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
false
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|