consul-templaterb 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,323 @@
1
+ require 'consul/async/utilities'
2
+ require 'em-http'
3
+ require 'thread'
4
+ require 'forwardable'
5
+ require 'erb'
6
+ module Consul
7
+ module Async
8
+ class InvalidTemplateException < StandardError
9
+ attr_reader :cause
10
+ def initialize(cause)
11
+ @cause = cause
12
+ end
13
+ end
14
+
15
+ class ConsulEndPointsManager
16
+ attr_reader :conf, :net_info, :start_time
17
+ def initialize(consul_configuration)
18
+ @conf = consul_configuration
19
+ @endpoints = {}
20
+ @iteration = 1
21
+ @start_time = Time.now.utc
22
+ @net_info = {
23
+ success: 0,
24
+ errors: 0,
25
+ bytes_read: 0
26
+ }
27
+ end
28
+
29
+ # https://www.consul.io/api/health.html#list-nodes-for-service
30
+ def service(name, dc: nil, passing: false, tag: nil)
31
+ raise 'You must specify a name for a service' if name.nil?
32
+ path = "/v1/health/service/#{name}"
33
+ query_params = {}
34
+ query_params[:dc] = dc if dc
35
+ query_params[:passing] = passing if passing
36
+ query_params[:tag] = tag if tag
37
+ create_if_missing(path, query_params) { ConsulTemplateService.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
38
+ end
39
+
40
+ # https://www.consul.io/api/health.html#list-checks-for-service
41
+ def checks_for_service(name, dc: nil, passing: false)
42
+ raise 'You must specify a name for a service' if name.nil?
43
+ path = "/v1/health/checks/#{name}"
44
+ query_params = {}
45
+ query_params[:dc] = dc if dc
46
+ query_params[:passing] = passing if passing
47
+ create_if_missing(path, query_params) { ConsulTemplateChecks.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
48
+ end
49
+
50
+ # https://www.consul.io/api/catalog.html#list-nodes
51
+ def nodes(dc: nil)
52
+ path = '/v1/catalog/nodes'
53
+ query_params = {}
54
+ query_params[:dc] = dc if dc
55
+ create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
56
+ end
57
+
58
+ # https://www.consul.io/api/catalog.html#list-services-for-node
59
+ def node(name_or_id, dc: nil)
60
+ path = "/v1/catalog/node/#{name_or_id}"
61
+ query_params = {}
62
+ query_params[:dc] = dc if dc
63
+ create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(conf, path, true, query_params, '{}')) }
64
+ end
65
+
66
+ # https://www.consul.io/api/agent.html#read-configuration
67
+ def agent_self
68
+ path = '/v1/agent/self'
69
+ query_params = {}
70
+ default_value = '{"Config":{}, "Coord":{}, "Member":{}, "Meta":{}, "Stats":{}}'
71
+ create_if_missing(path, query_params) { ConsulAgentSelf.new(ConsulEndpoint.new(conf, path, true, query_params, default_value)) }
72
+ end
73
+
74
+ # https://www.consul.io/api/agent.html#view-metrics
75
+ def agent_metrics
76
+ path = '/v1/agent/metrics'
77
+ query_params = {}
78
+ default_value = '{"Gauges":[], "Points":[], "Member":{}, "Counters":[], "Samples":{}}'
79
+ create_if_missing(path, query_params) { ConsulAgentMetrics.new(ConsulEndpoint.new(conf, path, true, query_params, default_value)) }
80
+ end
81
+
82
+ # https://www.consul.io/api/catalog.html#list-services
83
+ def services(dc: nil, tag: nil)
84
+ path = '/v1/catalog/services'
85
+ query_params = {}
86
+ query_params[:dc] = dc if dc
87
+ # Tag filtering is performed on client side
88
+ query_params[:tag] = tag if tag
89
+ create_if_missing(path, query_params) { ConsulTemplateServices.new(ConsulEndpoint.new(conf, path, true, query_params, '{}')) }
90
+ end
91
+
92
+ # https://www.consul.io/api/catalog.html#list-datacenters
93
+ def datacenters
94
+ path = '/v1/catalog/datacenters'
95
+ query_params = {}
96
+ create_if_missing(path, query_params) { ConsulTemplateDatacenters.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
97
+ end
98
+
99
+ # https://www.consul.io/api/kv.html#read-key
100
+ def kv(name = nil, dc: nil, keys: nil, recurse: false)
101
+ path = "/v1/kv/#{name}"
102
+ query_params = {}
103
+ query_params[:dc] = dc if dc
104
+ query_params[:recurse] = recurse if recurse
105
+ query_params[:keys] = keys if keys
106
+ default_value = '[]'
107
+ create_if_missing(path, query_params) { ConsulTemplateKV.new(ConsulEndpoint.new(conf, path, true, query_params, default_value), name) }
108
+ end
109
+
110
+ def render_file(path)
111
+ new_path = File.expand_path(path, File.dirname(@current_erb_path))
112
+ raise "render_file ERROR: #{path} is resolved as #{new_path}, but the file does not exists" unless File.exist? new_path
113
+ render(File.read(new_path), new_path)
114
+ end
115
+
116
+ def render(tpl, tpl_file_path)
117
+ # Ugly, but allow to use render_file well to support stack of calls
118
+ old_value = @current_erb_path
119
+ @current_erb_path = tpl_file_path
120
+ result = ERB.new(tpl).result(binding)
121
+ @current_erb_path = old_value
122
+ result
123
+ rescue StandardError => e
124
+ e2 = InvalidTemplateException.new e
125
+ raise e2, "Template contains errors: #{e.message}", e.backtrace
126
+ end
127
+
128
+ def write(file, tpl, last_result, tpl_file_path)
129
+ data = render(tpl, tpl_file_path)
130
+ not_ready = []
131
+ ready = 0
132
+ @iteration = Time.now.utc - @start_time
133
+ to_cleanup = []
134
+ @endpoints.each_pair do |endpoint_key, endpt|
135
+ if endpt.ready?
136
+ ready += 1
137
+ else
138
+ not_ready << endpt.endpoint.path
139
+ end
140
+ to_cleanup << endpoint_key if (@iteration - endpt.seen_at) > 10
141
+ end
142
+ if not_ready.count.positive?
143
+ STDERR.print "[INFO] Waiting for data from #{not_ready.count}/#{not_ready.count + ready} endpoints: #{not_ready[0..2]}..."
144
+ return [false, false, '']
145
+ end
146
+ if to_cleanup.count > 1
147
+ STDERR.puts "[INFO] Cleaned up #{to_cleanup.count} endpoints: #{to_cleanup}"
148
+ to_cleanup.each do |to_remove|
149
+ x = @endpoints.delete(to_remove)
150
+ x.endpoint.terminate
151
+ end
152
+ end
153
+ if last_result != data
154
+ STDERR.print "[INFO] Write #{Utilities.bytes_to_h data.bytesize} bytes to #{file}, "\
155
+ "netinfo=#{@net_info} aka "\
156
+ "#{Utilities.bytes_to_h((net_info[:bytes_read] / (Time.now.utc - @start_time)).round(1))}/s ...\n"
157
+ tmp_file = "#{file}.tmp"
158
+ File.open(tmp_file, 'w') do |f|
159
+ f.write data
160
+ end
161
+ File.rename(tmp_file, file)
162
+ end
163
+ [true, data != last_result, data]
164
+ end
165
+
166
+ def terminate
167
+ @endpoints.each_value do |v|
168
+ v.endpoint.terminate
169
+ end
170
+ @endpoints = {}
171
+ end
172
+
173
+ def create_if_missing(path, query_params)
174
+ fqdn = path.dup
175
+ query_params.each_pair do |k, v|
176
+ fqdn = "#{fqdn}&#{k}=#{v}"
177
+ end
178
+ tpl = @endpoints[fqdn]
179
+ unless tpl
180
+ tpl = yield
181
+ STDERR.print "[INFO] path #{path.ljust(64)} #{query_params.inspect}\r"
182
+ @endpoints[fqdn] = tpl
183
+ tpl.endpoint.on_response do |result|
184
+ @net_info[:success] = @net_info[:success] + 1
185
+ @net_info[:bytes_read] = @net_info[:bytes_read] + result.data.bytesize
186
+ end
187
+ tpl.endpoint.on_error { @net_info[:errors] = @net_info[:errors] + 1 }
188
+ end
189
+ tpl._seen_at(@iteration)
190
+ tpl
191
+ end
192
+ end
193
+
194
+ class ConsulTemplateAbstract
195
+ extend Forwardable
196
+ def_delegators :result_delegate, :each, :[], :sort, :each_value, :count, :empty?
197
+ attr_reader :result, :endpoint, :seen_at
198
+ def initialize(consul_endpoint)
199
+ @endpoint = consul_endpoint
200
+ consul_endpoint.on_response do |res|
201
+ @result = parse_result(res)
202
+ end
203
+ @result = parse_result(consul_endpoint.last_result)
204
+ end
205
+
206
+ def _seen_at(val)
207
+ @seen_at = val
208
+ end
209
+
210
+ def ready?
211
+ @endpoint.ready?
212
+ end
213
+
214
+ protected
215
+
216
+ def result_delegate
217
+ result.json
218
+ end
219
+
220
+ def parse_result(res)
221
+ res
222
+ end
223
+ end
224
+
225
+ class ConsulTemplateAbstractMap < ConsulTemplateAbstract
226
+ def_delegators :result_delegate, :each, :[], :keys, :sort, :values, :each_pair, :each_value
227
+ def initialize(consul_endpoint)
228
+ super(consul_endpoint)
229
+ end
230
+ end
231
+
232
+ class ConsulTemplateAbstractArray < ConsulTemplateAbstract
233
+ def initialize(consul_endpoint)
234
+ super(consul_endpoint)
235
+ end
236
+ end
237
+
238
+ class ConsulTemplateService < ConsulTemplateAbstractMap
239
+ def initialize(consul_endpoint)
240
+ super(consul_endpoint)
241
+ end
242
+ end
243
+
244
+ class ConsulTemplateDatacenters < ConsulTemplateAbstractArray
245
+ def initialize(consul_endpoint)
246
+ super(consul_endpoint)
247
+ end
248
+ end
249
+
250
+ class ConsulTemplateServices < ConsulTemplateAbstractMap
251
+ def initialize(consul_endpoint)
252
+ super(consul_endpoint)
253
+ end
254
+
255
+ def parse_result(res)
256
+ return res unless res.data == '{}' || endpoint.query_params[:tag]
257
+ res_json = JSON.parse(res.data)
258
+ result = {}
259
+ res_json.each do |name, tags|
260
+ result[name] = tags if tags.include? endpoint.query_params[:tag]
261
+ end
262
+ res.mutate(JSON.generate(result))
263
+ res
264
+ end
265
+ end
266
+
267
+ class ConsulAgentSelf < ConsulTemplateAbstractMap
268
+ def initialize(consul_endpoint)
269
+ super(consul_endpoint)
270
+ end
271
+ end
272
+
273
+ class ConsulAgentMetrics < ConsulTemplateAbstractMap
274
+ def initialize(consul_endpoint)
275
+ super(consul_endpoint)
276
+ end
277
+ end
278
+
279
+ class ConsulTemplateChecks < ConsulTemplateAbstractArray
280
+ def initialize(consul_endpoint)
281
+ super(consul_endpoint)
282
+ end
283
+ end
284
+
285
+ class ConsulTemplateNodes < ConsulTemplateAbstractArray
286
+ def initialize(consul_endpoint)
287
+ super(consul_endpoint)
288
+ end
289
+ end
290
+
291
+ class ConsulTemplateKV < ConsulTemplateAbstractArray
292
+ attr_reader :root
293
+ def initialize(consul_endpoint, root)
294
+ @root = root
295
+ super(consul_endpoint)
296
+ end
297
+
298
+ def find(name = root)
299
+ res = result_delegate.find { |k| name == k['Key'] }
300
+ res || {}
301
+ end
302
+
303
+ # Get the raw value (might be base64 encoded)
304
+ def get_value(name = root)
305
+ find(name)['Value']
306
+ end
307
+
308
+ # Get the Base64 Decoded value
309
+ def get_value_decoded(name = root)
310
+ val = get_value(name)
311
+ return nil unless val
312
+ Base64.decode64(val)
313
+ end
314
+
315
+ # Helper to get the value decoded as JSON
316
+ def get_value_json(name = root)
317
+ x = get_value_decoded(name)
318
+ return nil unless x
319
+ JSON.parse(x)
320
+ end
321
+ end
322
+ end
323
+ end
@@ -0,0 +1,57 @@
1
+ require 'consul/async/utilities'
2
+ require 'consul/async/consul_endpoint'
3
+ require 'consul/async/consul_template'
4
+ require 'consul/async/consul_template_render'
5
+ require 'em-http'
6
+ require 'thread'
7
+ require 'erb'
8
+ module Consul
9
+ module Async
10
+ class ConsulTemplateEngine
11
+ attr_reader :template_manager, :hot_reload_failure
12
+ attr_writer :hot_reload_failure
13
+ def initialize
14
+ @templates = []
15
+ @template_callbacks = []
16
+ @hot_reload_failure = 'die'
17
+ end
18
+
19
+ def add_template_callback(&block)
20
+ @template_callbacks << block
21
+ end
22
+
23
+ def add_template(source, dest)
24
+ @templates.push([source, dest])
25
+ end
26
+
27
+ def run(template_manager)
28
+ @template_manager = template_manager
29
+ EventMachine.run do
30
+ template_renders = []
31
+ @templates.each do |template_file, output_file|
32
+ template_renders << Consul::Async::ConsulTemplateRender.new(template_manager, template_file, output_file,
33
+ hot_reload_failure: hot_reload_failure)
34
+ end
35
+ EventMachine.add_periodic_timer(1) do
36
+ begin
37
+ results = template_renders.map(&:run)
38
+ all_ready = results.reduce(true) { |a, e| a && e.ready? }
39
+ begin
40
+ @template_callbacks.each do |c|
41
+ c.call([all_ready, template_manager, results])
42
+ end
43
+ rescue StandardError => cbk_error
44
+ STDERR.puts "Error in callback: #{cbk_error.inspect}"
45
+ raise cbk_error
46
+ end
47
+ rescue StandardError => e
48
+ STDERR.puts "[ERROR] Fatal error occured: #{e.inspect} - #{e.backtrace}"
49
+ template_manager.terminate
50
+ EventMachine.stop
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,80 @@
1
+ require 'consul/async/utilities'
2
+ require 'em-http'
3
+ require 'thread'
4
+ require 'erb'
5
+ module Consul
6
+ module Async
7
+ class ConsulTemplateRenderedResult
8
+ attr_reader :template_file, :output_file, :hot_reloaded, :ready, :modified, :last_result
9
+ def initialize(template_file, output_file, hot_reloaded, was_success, modified, last_result)
10
+ @template_file = template_file
11
+ @output_file = output_file
12
+ @hot_reloaded = hot_reloaded
13
+ @ready = was_success
14
+ @modified = modified
15
+ @last_result = last_result
16
+ end
17
+
18
+ def ready?
19
+ @ready
20
+ end
21
+ end
22
+ class ConsulTemplateRender
23
+ attr_reader :template_file, :output_file, :template_file_ctime, :hot_reload_failure
24
+ def initialize(template_manager, template_file, output_file, hot_reload_failure: 'die')
25
+ @hot_reload_failure = hot_reload_failure
26
+ @template_file = template_file
27
+ @output_file = output_file
28
+ @template_manager = template_manager
29
+ @last_result = ''
30
+ @template = load_template
31
+ end
32
+
33
+ def render(tpl = @template)
34
+ @template_manager.render(tpl, template_file)
35
+ end
36
+
37
+ def run
38
+ hot_reloaded = hot_reload_if_needed
39
+ was_success, modified, last_result = write
40
+ ConsulTemplateRenderedResult.new(template_file, output_file, hot_reloaded, was_success, modified, last_result)
41
+ end
42
+
43
+ private
44
+
45
+ def load_template
46
+ @template_file_ctime = File.ctime(template_file)
47
+ File.read(template_file)
48
+ end
49
+
50
+ # Will throw Consul::Async::InvalidTemplateException if template invalid
51
+ def update_template(new_template)
52
+ return false unless new_template != @template
53
+ # We render to ensure the template is valid
54
+ render(new_template)
55
+ @template = new_template.freeze
56
+ true
57
+ end
58
+
59
+ def write
60
+ success, modified, @last_result = @template_manager.write(@output_file, @template, @last_result, template_file)
61
+ [success, modified, @last_result]
62
+ end
63
+
64
+ def hot_reload_if_needed
65
+ new_time = File.ctime(template_file)
66
+ if template_file_ctime != new_time
67
+ begin
68
+ @template_file_ctime = new_time
69
+ return update_template(load_template)
70
+ rescue Consul::Async::InvalidTemplateException => e
71
+ STDERR.puts "****\n[ERROR] HOT Reload of template #{template_file} did fail due to #{e}\n****\n"
72
+ raise e unless hot_reload_failure == 'keep'
73
+ STDERR.puts "[WARN] Hot reload of #{template_file} was not taken into account, keep running with previous version"
74
+ end
75
+ end
76
+ false
77
+ end
78
+ end
79
+ end
80
+ end