consul-templaterb 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,40 @@
1
+ require 'consul/async/utilities'
2
+
3
+ module Consul
4
+ module Async
5
+ class EndPointStats
6
+ attr_reader :successes, :errors, :start, :body_bytes
7
+
8
+ def initialize
9
+ @start = Time.now.utc
10
+ @successes = 0
11
+ @errors = 0
12
+ @body_bytes = 0
13
+ end
14
+
15
+ def on_response(res)
16
+ @successes += 1
17
+ @body_bytes = body_bytes + res.http.response.bytesize
18
+ end
19
+
20
+ def on_error(_http)
21
+ @errors += 1
22
+ end
23
+
24
+ def bytes_per_sec
25
+ diff = (Time.now.utc - start)
26
+ diff = 1 if diff < 1
27
+ (body_bytes / diff).round(0)
28
+ end
29
+
30
+ def bytes_per_sec_human
31
+ "#{Utilities.bytes_to_h(bytes_per_sec)}/s"
32
+ end
33
+
34
+ def body_bytes_human
35
+ Utilities.bytes_to_h(body_bytes)
36
+ end
37
+ end
38
+ end
39
+ end
40
+
@@ -0,0 +1,249 @@
1
+ require 'consul/async/utilities'
2
+ require 'consul/async/stats'
3
+ require 'em-http'
4
+ require 'net/http'
5
+ require 'thread'
6
+ require 'json'
7
+
8
+ module Consul
9
+ module Async
10
+ class VaultConfiguration
11
+ attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
12
+ :lease_duration_factor, :debug
13
+
14
+ def initialize(base_url: 'http://localhost:8200',
15
+ debug: { network: false },
16
+ token: nil,
17
+ token_renew: true,
18
+ retry_duration: 10,
19
+ min_duration: 0.1,
20
+ lease_duration_factor: 0.5,
21
+ max_retry_duration: 600,
22
+ paths: {})
23
+ @base_url = base_url
24
+ @token_renew = token_renew
25
+ @debug = debug
26
+ @retry_duration = retry_duration
27
+ @min_duration = min_duration
28
+ @max_retry_duration = max_retry_duration
29
+ @lease_duration_factor = lease_duration_factor
30
+ @paths = paths
31
+ @token = token
32
+
33
+ end
34
+
35
+ def ch(path, symbol)
36
+ sub = @paths[path.to_sym]
37
+ if sub && sub[symbol]
38
+ STDERR.puts "[INFO] Overriding #{symbol}:=#{sub[symbol]} for #{path}"
39
+ sub[symbol]
40
+ else
41
+ method(symbol).call
42
+ end
43
+ end
44
+
45
+ def create(path)
46
+ return self unless @paths[path.to_sym]
47
+ VaultConfiguration.new(base_url: ch(path, :base_url),
48
+ debug: ch(path, :debug),
49
+ token: ch(path, :token),
50
+ retry_duration: ch(path, :retry_duration),
51
+ min_duration: ch(path, :min_duration),
52
+ max_retry_duration: ch(path, :max_retry_duration),
53
+ lease_duration_factor: ch(path, :lease_duration_factor),
54
+ paths: @paths)
55
+ end
56
+ end
57
+ class VaultResult
58
+ attr_reader :data, :http, :stats, :retry_in
59
+
60
+ def initialize(result, modified, stats, retry_in)
61
+ @data = result.response
62
+ @modified = modified
63
+ @http = result
64
+ @data_json = result.json
65
+ @last_update = Time.now.utc
66
+ @next_update = Time.now.utc + retry_in
67
+ @stats = stats
68
+ @retry_in = retry_in
69
+ end
70
+
71
+ def modified?
72
+ @modified
73
+ end
74
+
75
+ def mutate(new_data)
76
+ @data = new_data.dup
77
+ @data_json = nil
78
+ end
79
+
80
+ def [](path)
81
+ json[path]
82
+ end
83
+
84
+ def json
85
+ @data_json = JSON.parse(data) if @data_json.nil?
86
+ @data_json
87
+ end
88
+
89
+ end
90
+ class VaultHttpResponse
91
+ attr_reader :response_header, :response, :error, :json
92
+
93
+ def initialize(http, override_nil_response = nil)
94
+ if http.nil?
95
+ @response_header = nil
96
+ @response = override_nil_response
97
+ @error = 'Not initialized yet'
98
+ else
99
+ @response_header = http.response_header.nil? ? nil : http.response_header.dup.freeze
100
+ @response = http.response.nil? || http.response.empty? ? override_nil_response : http.response.dup.freeze
101
+ @error = http.error.nil? ? nil : http.error.dup.freeze
102
+ end
103
+ @json = JSON[response]
104
+ end
105
+ end
106
+
107
+ class VaultEndpoint
108
+ attr_reader :conf, :path, :http_method, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
109
+
110
+ def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data ={})
111
+ @conf = conf.create(path)
112
+ @default_value = default_value
113
+ @path = path
114
+ @http_method = http_method
115
+ @queue = EM::Queue.new
116
+ @x_consul_index = 0
117
+ @s_callbacks = []
118
+ @e_callbacks = []
119
+ @enforce_json_200 = enforce_json_200
120
+ @start_time = Time.now.utc
121
+ @consecutive_errors = 0
122
+ @query_params = query_params
123
+ @post_data = post_data
124
+ @stopping = false
125
+ @stats = EndPointStats.new
126
+ @last_result = VaultResult.new(VaultHttpResponse.new(nil, default_value), false, stats ,1)
127
+ on_response { |result| @stats.on_response result }
128
+ on_error { |http| @stats.on_error http }
129
+ _enable_network_debug if conf.debug && conf.debug[:network]
130
+ fetch
131
+ queue.push 0
132
+ end
133
+
134
+ def _enable_network_debug
135
+ on_response do |result|
136
+ state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
137
+ stats = result.stats
138
+ STDERR.puts "[DBUG]#{state}#{result.modified? ? '[MODFIED]' : '[NO DIFF]'}" \
139
+ "[s:#{stats.successes},err:#{stats.errors}]" \
140
+ "[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
141
+ " #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
142
+ end
143
+ on_error { |http| STDERR.puts "[ERROR]: #{path}: #{http.error}" }
144
+ end
145
+
146
+ def on_response(&block)
147
+ @s_callbacks << block
148
+ end
149
+
150
+ def on_error(&block)
151
+ @e_callbacks << block
152
+ end
153
+
154
+ def ready?
155
+ @ready
156
+ end
157
+
158
+ def terminate
159
+ @stopping = true
160
+ end
161
+
162
+ private
163
+
164
+ def build_request()
165
+ res = {
166
+ head: {
167
+ 'Accept' => 'application/json',
168
+ 'X-Vault-Token' => conf.token
169
+ },
170
+ query: {},
171
+ path: path,
172
+ keepalive: true,
173
+ callback: method(:on_response)
174
+ }
175
+ # if @post_data
176
+ # res[:body] = JSON.generate(@post_data)
177
+ # end
178
+ @query_params.each_pair do |k, v|
179
+ res[:query][k] = v
180
+ end
181
+ res
182
+ end
183
+
184
+ def get_lease_duration(result)
185
+ result.json['lease_duration'] || conf.min_duration
186
+ end
187
+
188
+ def _get_errors(http)
189
+ return [http.error] if http.error
190
+ unless http.json.nil?
191
+ return http.json['errors'] if http.json.key?('errors')
192
+ end
193
+ ['unknown error']
194
+ end
195
+
196
+ def _handle_error(http)
197
+ retry_in = [conf.max_retry_duration, conf.retry_duration + 2**@consecutive_errors].min
198
+ STDERR.puts "[ERROR][#{path}][#{http_method}] Code: #{http.response_header.status} #{_get_errors(http).join(' - ')} - Retry in #{retry_in}s #{stats.body_bytes_human}"
199
+ @consecutive_errors += 1
200
+ http_result = VaultHttpResponse.new(http, default_value)
201
+ EventMachine.add_timer(retry_in) do
202
+ yield
203
+ queue.push()
204
+ end
205
+ @e_callbacks.each { |c| c.call(http_result) }
206
+ end
207
+
208
+ def fetch
209
+ options = {
210
+ connect_timeout: 5, # default connection setup timeout
211
+ inactivity_timeout: 1, # default connection inactivity (post-setup) timeout
212
+ }
213
+ connection = EventMachine::HttpRequest.new(conf.base_url, options)
214
+ cb = proc do |_|
215
+ http = connection.send(http_method.downcase, build_request) # Under the hood: c.send('get', {stuff}) === c.get({stuff})
216
+ http.callback do
217
+ http_result = VaultHttpResponse.new(http.dup.freeze, default_value)
218
+ if enforce_json_200 && http.response_header.status != 200
219
+ _handle_error(http_result) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
220
+ else
221
+ @consecutive_errors = 0
222
+ modified = @last_result.nil? ? true : @last_result.data != http_result.response # Leaving it do to stats with this later
223
+ retry_in = get_lease_duration(http_result) * conf.lease_duration_factor
224
+ retry_in = [retry_in, conf.max_retry_duration].min
225
+ retry_in = [retry_in, conf.min_duration].max
226
+ result = VaultResult.new(http_result, modified, stats, retry_in)
227
+ unless @stopping
228
+ EventMachine.add_timer(retry_in) do
229
+ queue.push(0)
230
+ end
231
+ end
232
+ @last_result = result
233
+ @ready = true
234
+ @s_callbacks.each { |c| c.call(result) }
235
+ end
236
+ end
237
+
238
+ http.errback do
239
+ unless @stopping
240
+ _handle_error(http) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
241
+ end
242
+ end
243
+ queue.pop(&cb)
244
+ end
245
+ queue.pop(&cb)
246
+ end
247
+ end
248
+ end
249
+ end
@@ -1,5 +1,5 @@
1
1
  module Consul
2
2
  module Async
3
- VERSION = '1.2.1'.freeze
3
+ VERSION = '1.3.0'.freeze
4
4
  end
5
5
  end
@@ -3,13 +3,26 @@
3
3
  # CONSUL_TOOLS_SUFFIX: suffix for the address of consul tools
4
4
  # CONSUL_TOOLS_PREFIX: prefix for the address of consul tools
5
5
  # CONSUL_TOOLS: comma sperated list of consul tools
6
- tools = (ENV['CONSUL_TOOLS'] || 'services').split(",")
6
+ tools = (ENV['CONSUL_TOOLS'] || 'services,nodes,keys').split(",")
7
7
  tools_suffix = ENV['CONSUL_TOOLS_PREFIX'] || '-ui.html'
8
8
  tools_prefix = ENV['CONSUL_TOOLS_SUFFIX'] || 'consul-'
9
9
 
10
10
  dc_suffix = ENV['CONSUL_DC_SUFFIX'] || ''
11
11
  dc_prefix = ENV['CONSUL_DC_PREFIX'] || '#'
12
12
 
13
+ # CUSTOM_LINKS: A comma separated list of URLs to add to the header in the format: [linkname](actuallink)
14
+ # Example: [Consul template erb](https://github.com/criteo/consul-templaterb),[anotherlink](#)
15
+ custom_links = (ENV['CUSTOM_LINKS'] || '').split(",")
16
+ links_to_display = {}
17
+ if !custom_links.empty?
18
+ custom_links.each do |link|
19
+ if /\[.*\]\(.*\)/ =~ link
20
+ link_data = /\[(.*)\]\((.*)\)/.match(link)
21
+ links_to_display[link_data[1]] = link_data[2]
22
+ end
23
+ end
24
+ end
25
+
13
26
  %><!DOCTYPE html>
14
27
  <html lang="en">
15
28
  <head>
@@ -21,6 +34,7 @@
21
34
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
22
35
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.11/css/all.css" integrity="sha384-p2jx59pefphTFIpeqCcISO9MdVfIm4pNnsL08A6v5vaQc4owkQqxMV8kg4Yvhaw/" crossorigin="anonymous">
23
36
  <link rel="stylesheet" href="css/style.css">
37
+ <link rel="stylesheet" href="vendors/highlight/atom-one-dark.css">
24
38
  <style id="css-states">
25
39
  .service-tags { display: none; }
26
40
  </style>
@@ -44,6 +58,11 @@
44
58
  <a class="nav-link" href="<%= tools_prefix + tool + tools_suffix %>"><%= tool.gsub('_', ' ').capitalize %></a>
45
59
  </li>
46
60
  <% end %>
61
+ <% links_to_display.each do |linkname,link| %>
62
+ <li class="nav-item">
63
+ <a class="nav-link" target="_blank" href="<%= link %>"><%= linkname %></a>
64
+ </li>
65
+ <% end %>
47
66
  </ul>
48
67
  </div>
49
68
  </nav>
@@ -0,0 +1,39 @@
1
+ <% datasource = ENV['SERVICE_DATASOURCE'] || 'consul_keys.json'
2
+ # Time to wait before reloading configuration again in seconds (0 = never)
3
+ refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Keys') %>
4
+ <div class="main">
5
+ <div class="row mx-0">
6
+ <div id="filter-menu" class="col-4 col-m-3 px-4 pt-4">
7
+ <div class="form-group">
8
+ <div class="input-group">
9
+ <input id="keys-filter" type="text" placeholder="filter keys by name" class="form-control" />
10
+ <div class="input-group-append">
11
+ <span class="input-group-text" id="keys-counter"></span>
12
+ </div>
13
+ </div>
14
+ </div>
15
+ <div id="keys-wrapper" >
16
+ <ul id="keys-list" class="list-group">
17
+ </ul>
18
+ </div>
19
+ </div>
20
+ <div class="col-8 col-m-9">
21
+ <h2 class="text-center" id="kv-title"></h2>
22
+ <div id='data-wrapper'>
23
+ <pre><code id="kv-data">...</code></pre>
24
+ </div>
25
+ </div>
26
+ </div>
27
+ </div>
28
+ <!-- Optional JavaScript -->
29
+ <!-- JavaScript Dependencies: jQuery, Popper.js, Bootstrap JS, Shards JS -->
30
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
31
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
32
+ <script src="js/utils.js"></script>
33
+ <script src="js/keys.js"></script>
34
+ <script src="vendors/highlight/highlight.pack.js"></script>
35
+ <script type="text/javascript">
36
+ consulKeys = new ConsulKeys('<%= datasource %>','<%= refresh %>');
37
+ </script>
38
+ </body>
39
+ </html>
@@ -0,0 +1,35 @@
1
+ <% datasource = ENV['SERVICE_DATASOURCE'] || 'consul_nodes.json'
2
+ # Time to wait before reloading configuration again in seconds (0 = never)
3
+ refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Nodes') %>
4
+ <div id='nodes' class="main">
5
+ <div class="row mx-0">
6
+ <div class="col-12 col-m-12">
7
+ <h2 class="text-center" id="service-title"></h2>
8
+ <div class="row mb-2">
9
+ <div class="input-group float-left col-10">
10
+ <input id="instance-filter" type="text" placeholder="filter nodes by name, service or tags" class="form-control" />
11
+ </div>
12
+ <div id="instance-statuses" class="col-2">
13
+ <span id="service-status-passing" status="passing" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-success passing service-status">0</span>
14
+ <span id="service-status-warning" status="warning" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-warning warning service-status">0</span>
15
+ <span id="service-status-critical" status="critical" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-danger critical service-status">0</span>
16
+ <span id="service-status-total" status="total" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-dark service-status">0</span>
17
+ </div>
18
+ </div>
19
+ <div id="instances-wrapper">
20
+ <div id="instances-list" class="list-group"></div>
21
+ </div>
22
+ </div>
23
+ </div>
24
+ </div>
25
+ <!-- Optional JavaScript -->
26
+ <!-- JavaScript Dependencies: jQuery, Popper.js, Bootstrap JS, Shards JS -->
27
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
28
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
29
+ <script src="js/utils.js"></script>
30
+ <script src="js/nodes.js"></script>
31
+ <script type="text/javascript">
32
+ consulNodes = new ConsulNodes('<%= datasource %>','<%= refresh %>');
33
+ </script>
34
+ </body>
35
+ </html>
@@ -1,6 +1,6 @@
1
- <% datasource = ENV['SERVICE_DATASOURCE'] || 'consul-template.json'
1
+ <% datasource = ENV['SERVICE_DATASOURCE'] || 'consul_services.json'
2
2
  # Time to wait before reloading configuration again in seconds (0 = never)
3
- refresh = ENV['REFRESH'] || '600' %><%= render_file('common/header.html.erb', title: 'Services') %>
3
+ refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Services') %>
4
4
  <div class="main">
5
5
  <div class="row mx-0">
6
6
  <div id="filter-menu" class="col-4 col-m-3 px-4 pt-4">
@@ -12,7 +12,7 @@
12
12
  </label>
13
13
  </div>
14
14
  <div class="input-group">
15
- <input id="service-filter" type="text" placeholder="filter by name or tags" class="form-control" />
15
+ <input id="service-filter" type="text" placeholder="filter services by name or tags" class="form-control" />
16
16
  <div class="input-group-append">
17
17
  <span class="input-group-text" id="service-counter"></span>
18
18
  </div>
@@ -25,6 +25,11 @@
25
25
  </div>
26
26
  <div class="col-8 col-m-9">
27
27
  <h2 class="text-center" id="service-title"></h2>
28
+ <div class="row mb-2">
29
+ <div class="input-group float-left col-12">
30
+ <input id="instance-filter" type="text" placeholder="filter nodes by name or tags" class="form-control" />
31
+ </div>
32
+ </div>
28
33
  <div class="progress">
29
34
  <div id="service-progress-passing" status="passing" onclick="consulService.onClickFilter(this)" class="progress-status progress-bar bg-success" role="progressbar" style="width: 100%">passing</div>
30
35
  <div id="service-progress-warning" status="warning" onclick="consulService.onClickFilter(this)" class="progress-status progress-bar bg-warning" role="progressbar" style="width: 0%">warning</div>
@@ -40,6 +45,7 @@
40
45
  <!-- JavaScript Dependencies: jQuery, Popper.js, Bootstrap JS, Shards JS -->
41
46
  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
42
47
  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
48
+ <script src="js/utils.js"></script>
43
49
  <script src="js/service.js"></script>
44
50
  <script type="text/javascript">
45
51
  consulService = new ConsulService('<%= datasource %>','<%= refresh %>');