consul-templaterb 1.26.2 → 1.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc35b5803af2e05001323f77e18be90ec85baa05dd621030a83127cf1b5fd9f1
-  data.tar.gz: f7d3e25054dcd6c1b022e5c4bf58ea902c5c95676aafa936512659eaec3de9a1
+  metadata.gz: c15ee78520048d7d091882b9d3b1d43b4cc5f5658cf39d782f06e1dc617d8cd9
+  data.tar.gz: 6f77a9daab557420f2640a314a97f78af68e26fec91c1a66f3ee575d9bf150f8
 SHA512:
-  metadata.gz: e1e053e41716e64177917ef8a3ef1360ba82995758bb7dacfafd3bc7fb7a8f5774f107c62cbb7c4340189ea59cab15db936fc27854f6fab1ba487466ad0add3f
-  data.tar.gz: 95c0584f456239478395ceb0d2678e2ec1c7165f2148371f272ab23780148ed6498f419bc8104933d14cd155fcf20403c2b66521e604a3d2f4232bb8373f69c2
+  metadata.gz: 8c2b374a9400061f3ffd673e418771bb5685ec3d13a631c3839660121d86575be1fbbd1cd99f7ee1a4ddedf850ec72fa984ca084575da587b7a239d2e4654b2e
+  data.tar.gz: '0933c0a2f1f58c617c184679ab68c1e90d55f1d9ab796e45336ddbbf49ae2fd73cf1aafbfea6e921254a520c8be3e5f0f380cfbdacd694ba2de6c5f8e4f5a074'

data/.rubocop.yml

@@ -7,28 +7,28 @@ Layout/LineLength:
   Max: 175
 
 Metrics/AbcSize:
-  Max: 82
+  Max: 87
 
 Metrics/BlockLength:
-  Max: 160
+  Max: 188
 
 Metrics/BlockNesting:
   Max: 4
 
 Metrics/ClassLength:
-  Max: 275
+  Max: 285
 
 Metrics/CyclomaticComplexity:
-  Max: 20
+  Max: 21
 
 Metrics/MethodLength:
-  Max: 65
+  Max: 68
 
 Metrics/ParameterLists:
-  Max: 14
+  Max: 18
 
 Metrics/PerceivedComplexity:
-  Max: 23
+  Max: 24
 
 # We use `dc` as a parameter in many methods
 Naming/MethodParameterName:

data/.travis.yml

@@ -1,13 +1,13 @@
 language: ruby
 rvm:
-- 2.4.9
-- 2.5.7
-- 2.6.5
-- 2.7.0
+- 2.4.10
+- 2.5.8
+- 2.6.6
+- 2.7.1
 jobs:
   include:
     - stage: Gem release
-      rvm: 2.5.7
+      rvm: 2.5.8
       script: echo "Publishing consul-templaterb on rubygems.org ..."
       deploy:
         provider: rubygems

data/CHANGELOG.md

@@ -2,8 +2,46 @@
 
 ## (UNRELEASED)
 
+## 1.28.0 (Sept 25, 2020)
+
+NEW FEATURES:
+
+ * Added `-W` or `--wait-between-reload-signal` to avoid sending too many signals
+   to a process executed. This avoids for instance reloading too much a HAProxy configuration
+   without having to play with `-w` as described in [#69](https://github.com/criteo/consul-templaterb/issues/69)
+
+BUGFIX:
+
+ * Removed warnings at runtime with Ruby 2.7+
+ * Minor JS fix in Consul-UI (Added missing unused parameter to function `serviceTitleGenerator`)
+
+## 1.27.2 (Sept 4, 2020)
+
+IMPROVEMENTS:
+
+ * Consul-UI now supports navigation between nodes and services in both ways
+
+## 1.27.1 (July 28, 2020)
+
+BUGIX:
+
+ * Fix collision in JSON queries when using payload in requests #68
+
+## 1.27.0 (June 5, 2020)
+
 NEW FEATURES:
 
+ * For Consul 1.7+, now support `checks_in_state(check_state, dc: nil, [agent: consul_agent_address])`,
+   fixes feature [#65](https://github.com/criteo/consul-templaterb/issues/65)
+ * New options to support/disable TLS validation thanks to [@jeromegn](https://github.com/jeromegn)
+   [#66](https://github.com/criteo/consul-templaterb/pull/66)
+
+## 1.26.3 (April 15, 2020)
+
+BUGFIX:
+
+ * Removed all Criteo specific decorators from Consul-UI
+
 ## 1.26.2 (April 15, 2020)
 
 BUGFIX:

data/README.md

@@ -165,9 +165,19 @@ USAGE: consul-templaterb [[options]]
     -f, --[no-]fail-fast             If consul/vault endpoints fail at startup, fail immediately
     -g, --no-gzip-compression        Disable GZIP compression in HTTP requests
     -c, --consul-addr=<address>      Address of Consul, eg: http://localhost:8500
+        --consul-cert-chain=<path/to/cert_chain>
+                                     Path to Consul TLS client certificate chain to use
+        --consul-private-key=<path/to/private_key>
+                                     Path to Consul TLS client private key to use
+        --skip-consul-verify-tls     Skip verifying Consul TLS via certificate authority (DANGEROUS)
     -l, --log-level=<log_level>      Log level, default=info, any of none|error|info|debug
         --consul-token=<token>       Use a token to connect to Consul
     -V, --vault-addr=<address>       Address of Vault, eg: http://localhost:8200
+        --vault-cert-chain=<path/to/cert_chain>
+                                     Path to Vault TLS client certificate chain to use
+        --vault-private-key=<path/to/private_key>
+                                     Path to Vault TLS client private key to use
+        --skip-vault-verify-tls      Skip verifying Vault TLS via certificate authority (DANGEROUS)
         --vault-token=<token>        Token used to authenticate against vault.
         --[no-]vault-renew           Control auto-renewal of the Vault token. Default: activated
         --vault-retry, --vault-retry-attempts [RETRIES]
@@ -178,10 +188,11 @@ USAGE: consul-templaterb [[options]]
     -r, --retry-delay=<min_duration> Min Retry delay on Error/Missing Consul Index
     -k, --hot-reload=<behavior>      Control hot reload behaviour, one of :[die (kill daemon on hot reload failure), keep (on error, keep running), disable (hot reload disabled)]
     -K, --sig-term=kill_signal       Signal to send to next --exec command on kill, default=TERM
+    -M, --debug-memory-usage         Display messages when RAM grows
     -T, --trim-mode=trim_mode        ERB Trim mode to use (- by default)
     -R, --sig-reload=reload_signal   Signal to send to next --exec command on reload (NONE supported), default=HUP
-    -M, --debug-memory-usage         Display messages when RAM grows
-    -e, --exec=<command>             Execute the following command
+    -W, --wait-signal=min_duration   Wait at least n seconds before each reload signal being sent to next --exec process
+    -e, --exec=<command>             Execute the following command in as a subprocess when all templates are ready
     -d, --debug-network-usage        Debug the network usage
     -t erb_file:[output]:[command]:[params_file],
         --template                   Add a erb template, its output and optional reload command
@@ -235,7 +246,11 @@ nor write the file.
 Signals can be customized per process. Two signals are supported with options `--sig-reload` and
 `--sig-term`. When the option is added, the next `--exec` options to start a process will use the
 given signal. By default, HUP will be sent to reload events (you can use NONE to avoid sending any
-reload signal), TERM will be used when leaving consul-templaterb.
+reload signal), TERM will be used when leaving consul-templaterb. A minimum duration between reload
+signals can be specified for each sub process by prepending `--wait-signal=min_duration` to `--exec`
+command.
+In such case, the signal will be sent every `min_duration` as a maximum (very useful for templates
+changing a lot, but you don't want to trigger too many reloads, for instance for a load-balancer).
 
 ### Bandwidth limitation
 
@@ -332,7 +347,7 @@ Please consult [CHANGELOG.md](CHANGELOG.md) for fixed bugs.
 
 ## TODO
 
-* [x] Hashi's Vault support (EXPERIMENTAL)
+* [x] Hashi's Vault support
 * [ ] Implement automatic dynamic rate limit
 * [x] More samples: apache, nginx, a full website displaying consul information...
 * [x] Optimize rendering speed at start-up: an iteration is done every second by default, but it would be possible to speed

data/TemplateAPI.md

@@ -388,6 +388,15 @@ name or its ID. If DC is specified, will lookup for given node in another datace
 
 [Find all the checks](https://www.consul.io/api/health.html#list-checks-for-service) of a given service.
 
+## def checks_in_state(check_state, dc: nil, [agent: consul_agent_address])
+
+[Find all the checks in a given state](https://www.consul.io/api-docs/health#list-checks-in-state) in the whole cluster.
+
+The filter check_state must be one of any|critical|warning|passing.
+
+Warning: this endpoint might be very frequently updated in a
+large cluster if you are using `any` value. This endpoint is supported with Consul 1.7+.
+
 ## kv(name, [dc: nil], [keys: false], [recurse: false], [agent: consul_agent_address])
 
 [Read keys from KV Store](https://www.consul.io/api/kv.html#read-key). It can be used for both listing the keys and

data/bin/consul-templaterb

@@ -28,6 +28,9 @@ options = {
     },
     base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
     token: ENV['VAULT_TOKEN'] || nil,
+    tls_cert_chain: ENV['VAULT_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['VAULT_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     token_renew: true,
@@ -48,6 +51,9 @@ options = {
     },
     base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500',
     token: ENV['CONSUL_HTTP_TOKEN'] || nil,
+    tls_cert_chain: ENV['CONSUL_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['CONSUL_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     retry_duration: 10,      # On error, retry after n seconds
@@ -88,6 +94,7 @@ consul_engine = Consul::Async::ConsulTemplateEngine.new
 @programs = {}
 cur_sig_reload = 'HUP'.freeze
 cur_sig_term = 'TERM'.freeze
+cur_min_duration_between_signals = 1
 
 optparse = OptionParser.new do |opts|
   opts.banner = usage_text
@@ -122,6 +129,18 @@ optparse = OptionParser.new do |opts|
     options[:consul][:base_url] = consul_url
   end
 
+  opts.on('--consul-cert-chain=<path/to/cert_chain>', String, 'Path to Consul TLS client certificate chain to use') do |consul_client_cert|
+    options[:consul][:tls_cert_chain] = consul_client_cert
+  end
+
+  opts.on('--consul-private-key=<path/to/private_key>', String, 'Path to Consul TLS client private key to use') do |consul_client_key|
+    options[:consul][:tls_private_key] = consul_client_key
+  end
+
+  opts.on('--skip-consul-verify-tls', 'Skip verifying Consul TLS via certificate authority (DANGEROUS)') do
+    options[:consul][:tls_verify_peer] = false
+  end
+
   opts.on('-l', '--log-level=<log_level>', String, "Log level, default=info, any of #{::Consul::Async::Debug.levels.join('|')}") do |log_level|
     ::Consul::Async::Debug.level = log_level
   end
@@ -134,6 +153,18 @@ optparse = OptionParser.new do |opts|
     options[:vault][:base_url] = vault_url
   end
 
+  opts.on('--vault-cert-chain=<path/to/cert_chain>', String, 'Path to Vault TLS client certificate chain to use') do |vault_client_cert|
+    options[:vault][:tls_cert_chain] = vault_client_cert
+  end
+
+  opts.on('--vault-private-key=<path/to/private_key>', String, 'Path to Vault TLS client private key to use') do |vault_client_key|
+    options[:vault][:tls_private_key] = vault_client_key
+  end
+
+  opts.on('--skip-vault-verify-tls', 'Skip verifying Vault TLS via certificate authority (DANGEROUS)') do
+    options[:vault][:tls_verify_peer] = false
+  end
+
   opts.on('-T', '--vault-token=<token>', String, 'Token used to authenticate against vault.') do |vault_token|
     options[:vault][:token] = vault_token
   end
@@ -185,6 +216,10 @@ optparse = OptionParser.new do |opts|
     cur_sig_term = compute_signal(sig, nil)
   end
 
+  opts.on('-M', '--debug-memory-usage', 'Display messages when RAM grows') do
+    consul_engine.debug_memory = true
+  end
+
   opts.on('-T', '--trim-mode=trim_mode', String,
           "ERB Trim mode to use (#{options[:erb][:trim_mode]} by default)") do |trim_mode|
     options[:erb][:trim_mode] = trim_mode
@@ -195,25 +230,36 @@ optparse = OptionParser.new do |opts|
     cur_sig_reload = compute_signal(sig, 'NONE')
   end
 
-  opts.on('-M', '--debug-memory-usage', 'Display messages when RAM grows') do
-    consul_engine.debug_memory = true
+  opts.on('-W', '--wait-signal=min_duration', Float, 'Wait at least n seconds before each reload signal being sent to next --exec process') do |min_duration|
+    raise "-wait-between-reload-signal=#{min_duration} must be greater than 0" unless min_duration.positive?
+
+    cur_min_duration_between_signals = min_duration
   end
 
-  opts.on('-e', '--exec=<command>', String, 'Execute the following command') do |cmd|
+  opts.on('-e', '--exec=<command>', String, 'Execute the following command in as a subprocess when all templates are ready') do |cmd|
     sig_reload = cur_sig_reload
     sig_term = cur_sig_term
+    sig_min_interval = cur_min_duration_between_signals
     consul_engine.add_template_callback do |all_ready, template_manager, results|
       if all_ready
         modified = results.any?(&:modified)
         if @programs[cmd].nil?
-          warn "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload || 'NONE'} on_term=#{sig_term}"
+          warn "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload || 'NONE'} on_term=#{sig_term}, delay between reloads=#{sig_min_interval}s"
           @programs[cmd] = Consul::Async::ProcessHandler.new(cmd, sig_reload: sig_reload, sig_term: sig_term)
           @programs[cmd].start
         else
+
           # At least one template has been modified
-          @programs[cmd].reload if modified
+          process_to_reload = @programs[cmd]
+          if modified && !process_to_reload.reload_scheduled
+            process_to_reload.reload_scheduled = true
+            now = Time.now
+            delay = sig_min_interval - (now - @programs[cmd].last_signal_sent)
+            delay = 0 if delay.negative?
+            EventMachine.add_timer(delay) { process_to_reload.reload }
+          end
           begin
-            @programs[cmd].process_status
+            process_to_reload.process_status
           rescue Consul::Async::ProcessDoesNotExist => e
             warn "[FATAL] The process is dead, aborting run: #{e.inspect}"
             template_manager.terminate

data/lib/consul/async/consul_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class ConsulConfiguration
       attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
                   :missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression,
-                  :fail_fast_errors, :max_consecutive_errors_on_endpoint
+                  :fail_fast_errors, :max_consecutive_errors_on_endpoint, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(base_url: 'http://localhost:8500',
                      debug: { network: false },
                      token: nil,
@@ -23,7 +23,10 @@ module Consul
                      enable_gzip_compression: true,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: 1)
+                     fail_fast_errors: 1,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token = token
         @debug = debug
@@ -38,6 +41,9 @@ module Consul
         @paths = paths
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -71,7 +77,10 @@ module Consul
                                 enable_gzip_compression: enable_gzip_compression,
                                 paths: @paths,
                                 max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
-                                fail_fast_errors: @fail_fast_errors)
+                                fail_fast_errors: @fail_fast_errors,
+                                tls_cert_chain: ch(path, :tls_cert_chain),
+                                tls_private_key: ch(path, :tls_private_key),
+                                tls_verify_peer: ch(path, :tls_verify_peer))
       end
     end
 
@@ -233,6 +242,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: conf.wait_duration + 1 + (conf.wait_duration / 16) # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.base_url, options)
         }

data/lib/consul/async/consul_template.rb

@@ -31,7 +31,8 @@ module Consul
 
       def as_json(url, default_value, refresh_delay_secs: 10, **opts)
         conf = JSONConfiguration.new(url: url, min_duration: refresh_delay_secs, retry_on_non_diff: refresh_delay_secs, **opts)
-        @endp_manager.create_if_missing(url, {}) do
+        endpoint_id = url + opts.to_json
+        @endp_manager.create_if_missing(url, {}, endpoint_id: endpoint_id) do
           if default_value.is_a?(Array)
             ConsulTemplateJSONArray.new(JSONEndpoint.new(conf, url, default_value))
           else
@@ -161,6 +162,18 @@ module Consul
         create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
+      # https://www.consul.io/api-docs/health#list-checks-in-state
+      # Supported in Consul 1.7+
+      def checks_in_state(check_state, dc: nil, agent: nil)
+        valid_checks_states = %w[any critical passing warning]
+        raise "checks_in_state('#{check_state}'...) must be one of #{valid_checks_states}" unless valid_checks_states.include?(check_state)
+
+        path = "/v1/health/state/#{check_state}"
+        query_params = {}
+        query_params[:dc] = dc if dc
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
+      end
+
       # https://www.consul.io/api/catalog.html#list-nodes
       def nodes(dc: nil, agent: nil)
         path = '/v1/catalog/nodes'
@@ -382,16 +395,21 @@ module Consul
         VaultEndpoint.new(vault_conf, path, :POST, {}, {})
       end
 
-      def create_if_missing(path, query_params, fail_fast_errors: @fail_fast_errors, max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint, agent: nil)
-        fqdn = path.dup
-        query_params.each_pair do |k, v|
-          fqdn = "#{agent}#{fqdn}&#{k}=#{v}"
-        end
-        tpl = @endpoints[fqdn]
+      def create_if_missing(path, query_params, fail_fast_errors: @fail_fast_errors,
+                            max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
+                            agent: nil, endpoint_id: nil)
+        endpoint_id ||= begin
+                          fqdn = path.dup
+                          query_params.each_pair do |k, v|
+                            fqdn = "#{agent}#{fqdn}&#{k}=#{v}"
+                          end
+                          fqdn
+                        end
+        tpl = @endpoints[endpoint_id]
         unless tpl
           tpl = yield
           ::Consul::Async::Debug.print_debug "path #{path.ljust(64)} #{query_params.inspect}\r"
-          @endpoints[fqdn] = tpl
+          @endpoints[endpoint_id] = tpl
           tpl.endpoint.on_response do |result|
             @net_info[:success] += 1
             @net_info[:bytes_read] += result.data.bytesize

data/lib/consul/async/consul_template_render.rb

@@ -59,7 +59,7 @@ module Consul
         return false unless new_template != @template
 
         # We render to ensure the template is valid
-        render(new_template, current_template_info)
+        render(new_template, current_template_info: current_template_info)
         @template = new_template.freeze
         true
       end

data/lib/consul/async/json_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class JSONConfiguration
       attr_reader :url, :retry_duration, :min_duration, :retry_on_non_diff,
                   :debug, :enable_gzip_compression, :request_method, :json_body,
-                  :headers
+                  :headers, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(url:,
                      debug: { network: false },
                      retry_duration: 10,
@@ -18,7 +18,10 @@ module Consul
                      request_method: :get,
                      json_body: nil,
                      headers: {},
-                     enable_gzip_compression: true)
+                     enable_gzip_compression: true,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @url = url
         @debug = debug
         @enable_gzip_compression = enable_gzip_compression
@@ -28,6 +31,9 @@ module Consul
         @request_method = request_method
         @json_body = json_body
         @headers = headers
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def create(_url)
@@ -181,6 +187,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 60 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.url, options)
         }

data/lib/consul/async/process_handler.rb

@@ -7,7 +7,8 @@ module Consul
     # Handle the full lifecycle of a process and allows to forward
     # Posix signals to child process when needed.
     class ProcessHandler
-      attr_reader :command, :sig_reload, :sig_term, :pid, :exit_status
+      attr_reader :command, :sig_reload, :sig_term, :pid, :exit_status, :last_signal_sent, :reload_scheduled
+      attr_writer :reload_scheduled
       def initialize(command, sig_reload: 'HUP', sig_term: 'TERM')
         raise 'empty sig_term is not supported' unless sig_term
 
@@ -16,18 +17,23 @@ module Consul
         @sig_term = sig_term
         @pid = nil
         @exit_status = nil
+        @last_signal_sent = Time.now
+        @reload_scheduled = false
       end
 
       def start
         return pid unless pid.nil?
 
         @pid = Process.spawn(command)
+        @last_signal_sent = Time.now
       end
 
       def reload
         return if sig_reload.nil?
 
+        @last_signal_sent = Time.now
         warn "Sending SIG #{sig_reload} to #{pid}..."
+        @reload_scheduled = false
         begin
           Process.kill(sig_reload, pid)
         rescue Errno::ESRCH => e

data/lib/consul/async/vault_endpoint.rb

@@ -10,7 +10,8 @@ module Consul
     # Configuration for Vault Endpoints
     class VaultConfiguration
       attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
-                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors
+                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors, :tls_cert_chain, :tls_private_key,
+                  :tls_verify_peer
 
       def initialize(base_url: 'http://localhost:8200',
                      debug: { network: false },
@@ -22,7 +23,10 @@ module Consul
                      max_retry_duration: 600,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: false)
+                     fail_fast_errors: false,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token_renew = token_renew
         @debug = debug
@@ -34,6 +38,9 @@ module Consul
         @token = token
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -226,6 +233,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 1 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = EventMachine::HttpRequest.new(conf.base_url, options)
         cb = proc do |_|
           http = connection.send(http_method.downcase, build_request) # Under the hood: c.send('get', {stuff}) === c.get({stuff})

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.26.2'.freeze
+    VERSION = '1.28.0'.freeze
   end
 end

data/samples/checks_in_warning_or_critical_state.yaml.erb

@@ -0,0 +1,13 @@
+<%=
+  # This sample displays checks for the whole cluster
+  # in warning or critical state
+  # API available with Consul 1.7+
+  res = []
+  checks_in_state('warning').each do |c|
+    res << c
+  end
+  checks_in_state('critical').each do |c|
+    res << c
+  end
+  YAML.dump({'warning_or_critical_checks' => res})
+%>

data/samples/consul-ui/css/style.css

@@ -15,6 +15,10 @@
   content: '⚠ ';
 }
 
+.btn-outline-success a:hover, .btn-outline-warning a:hover, .btn-outline-danger a:hover{
+  color: #fff;
+}
+
 #service-wrapper, #instances-wrapper, #keys-wrapper {
   overflow-y: scroll;
   border-top-left-radius: 0px;

data/samples/consul-ui/decorators.js.erb

@@ -1,18 +1,5 @@
-// Utilities
+// Use this file to configure custom decorators for your Consul-UI
 var httpRegexp = new RegExp('^http[s]?://[^ ]+$');
-var dc = "<%= ENV['CRITEO_DC'] || 'par'%>";
-var env = "<%= ENV['CRITEO_ENV'] || 'preprod'%>";
-
-var availability_url = 'https://grafana.crto.in/d/xFX5gCnWz/service-availability?var-datacenter=' + dc + '&var-service=';
-if (env == 'preprod') {
-  availability_url = 'https://grafana.preprod.crto.in/d/E0ANGjnZz/service-availability?var-datacenter=' + dc + '&var-service=';
-}
-var swagger_url = 'https://swaggercatalogapp.' + dc + '.' + env + '.crto.in/explore/swagger?key=';
-var slack_url = 'https://criteo.slack.com/app_redirect?channel=';
-
-var rackguru_url = 'https://rackguru.' + env + '.crto.in/serial/';
-
-var asapi_url = 'https://idm.' + env + '.crto.in/tool/multiGroupInfo/'
 
 function url_decorator(key, value) {
   var e = document.createElement('a');
@@ -26,27 +13,7 @@ function usefullLinksGenerator(instance, serviceName, node_meta_info) {
     top.className = 'instance-links';
 
     var usefullLinks = [
-        {
-            title: "Trigger security scan",
-            iconClassName: "fas fa-shield-alt",
-            href: "https://security.crto.in/#/scan/?ip=" + instance.addr
-        },
-        {
-            title: "Availability Graph",
-            iconClassName: "fas fa-chart-area",
-            href: availability_url + serviceName
-        },
     ];
-    if (node_meta_info!= null) {
-      var serial = node_meta_info['serial_number'];
-      if (serial != null) {
-        usefullLinks.push({
-            title: "RackGuru",
-            iconClassName: "fas fa-server",
-            href: rackguru_url + serial
-        });
-      }
-    }
     var first = true;
     for (let usefullLink of usefullLinks) {
         link = document.createElement('a');
@@ -134,19 +101,6 @@ function groups_decorator(instance, key, value, serviceName) {
   return span;
 }
 
-/**
- * Decorates with a slack channel link
- */
-function slack_channel(instance, key, value, serviceName) {
-  var sName = value;
-  if (sName.startsWith('#')) {
-    sName = sName.substring(1);
-  }
-  return build_link(slack_url + encodeURIComponent(sName), '#'+ sName);
-}
-
-const start_regexp = /(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z/
-
 function decorateIsoDate(value, formated) {
   const parsed  = Date.parse(formated);
   if (isNaN(parsed)) {
@@ -163,55 +117,6 @@ function decorateIsoDate(value, formated) {
 // This is the list of function called
 // When a service meta is found
 var registered_decorators = {
-    alert_availability_slack_channel: slack_channel,
-    gerrit_repository: function(instance, key, value, serviceName) {
-      return build_link('https://review.crto.in/#/q/' + value, value);
-    },
-    version: function(instance, key, value, serviceName) {
-      var asInt = parseInt(value);
-      if (asInt < 10000) {
-        return document.createTextNode(value);
-      }
-      if (instance.sMeta['CRITEO_APP_POOL'] != null) {
-          return build_link("https://devtools.crto.in/log.html?moab=cs&im=nb-to&range=100%2C" + asInt, value);
-      } else {
-          var tUrl = "https://devtools.crto.in/log.html?moab=j&im=nb-to&range=100%2C" + asInt;
-          if (instance.sMeta['jvm_artifact'] != null) {
-            tUrl += '&with-dependencies=true&artifacts=' + encodeURIComponent(instance.sMeta['jvm_artifact']);
-          }
-          return build_link(tUrl, value);
-      }
-    },
-    MESOS_TERM_DEBUG_GRANTED_TO: groups_decorator,
-    OWNERS: groups_decorator,
-    marathon_app_id: function(instance, key, value, serviceName) {
-      var app_name = value;
-      if (app_name[0] != '/') {
-        app_name = '/' + app_name;
-      }
-      if (instance.sMeta && instance.sMeta['marathon_ui']) {
-        return build_link(instance.sMeta['marathon_ui'] + '/#/apps/' + encodeURIComponent(app_name), value);
-      } else {
-        // non decorated value
-        return document.createTextNode(value);
-      }
-    },
-    marathon_app_version: function(instance, key, value, serviceName) {
-      return decorateIsoDate(value, value);
-    },
-    slack_channel: slack_channel,
-    start: function(instance, key, value, serviceName) {
-        const reg = start_regexp.exec(value);
-        if (reg != null) {
-          var formated = reg[1] + '-' + reg[2] + '-' + reg[3] + 'T' + reg[4] + ':' + reg[5] + ':' + reg[6] + 'Z';
-          return decorateIsoDate(value, formated);
-        } else {
-          return document.createTextNode(value);
-        }
-    },
-    swagger_key: function(instance, key, value, serviceName) {
-        return build_link(swagger_url + encodeURIComponent(value), value);
-    },
 }
 
 function service_meta_semantics_decorator(instance, key, value, serviceName) {
@@ -286,85 +191,11 @@ function nodeMetaGenerator(nodeMetaTags, serviceMetaIfAny) {
  * it does not have to return anything.
  */
 function navBarDecorator(navbar) {
-  if (typeof consulManager === 'undefined') {
-    // Timepicker is not supported on this page
-    return;
-  }
-  var timepicker_container = document.createElement('div');
-  timepicker_container.innerHTML = `
-              <div class="row">
-                <button type="button" disabled class="btn btn-secondary" data-toggle="tooltip" data-html="true" title="Data returned is the closest available to request" id="currently-displayed-data-date">
-                  Pick a date to see data from the past
-                </button>
-                <div class="col-sm-14">
-                  <div class="form-group">
-                    <div class="input-group date" id="datetimepicker1" data-target-input="nearest">
-                      <input type="text" class="form-control datetimepicker-input" data-target="#datetimepicker1"/>
-                      <div class="input-group-append" data-target="#datetimepicker1" data-toggle="datetimepicker">
-                        <div class="input-group-text"><i class="fa fa-calendar"></i></div>
-                      </div>
-                    </div>
-                  </div>
-                </div>
-              </div>
-              `;
-
-  navbar.appendChild(timepicker_container);
-  var script= document.createElement('script');
-  script.type='text/javascript';
-  // TODO(g.seux): extract code to another file and set "source" on script element
-  script.innerHTML = `
-                    $('#datetimepicker1').datetimepicker({
-                      format: 'DD/MM/YYYY HH:mm:ss Z',  // default format does not allow to select seconds
-                      sideBySide: true, // display date+time on the same widget
-                      useCurrent: true // by default, select current date
-                    });
-                  $("#datetimepicker1").on("change.datetimepicker", function (e) {
-                    console.log("Will fetch data from date " + e.date);
-                    console.log("will clean existing data");
-                    consulManager.clean().then(function(result) {
-                      switch(consulManager.constructor.name) {
-                        case "ConsulServiceManager":
-                          backup_type = 'consul_services';
-                          break;
-                        case "ConsulKeysManager":
-                          backup_type  = 'consul_keys'
-                          break;
-                        case "ConsulNodesManager":
-                          backup_type = 'consul_nodes'
-                          break;
-                        default:
-                          console.log("Unknown " + consulManager.constructor.name + " type");
-                      }
-                      if (e.date) {
-                        console.log("Will replace data by closest snapshot to " + e.date);
-                        var target_url = "https://consul-info-timeline-history.<%= ENV['CRITEO_DC'] %>.<%= ENV['CRITEO_ENV']%>.crto.in/backup/" + backup_type + "/" + e.date / 1000;
-                      } else {
-                        console.log("Will restore to local version");
-                        var target_url = defaultConsulManager.resourceURL;
-                      }
-                      if (typeof(defaultConsulManager) == 'undefined') {
-                        // store first manager to be able to restore it
-                        defaultConsulManager = consulManager
-                      }
-                      consulManager = new consulManager.constructor(target_url);
-                    });
-                  });
-  `;
-  navbar.appendChild(script);
 }
 
-
 /**
  * fetchedResponseDecorator is called with http response when a resource is fetched by any instance of ConsulUIManager
  * it does not have to return anything.
  */
 async function fetchedResponseDecorator(httpResponse) {
-  const data_date = await httpResponse.headers.get('X-Consul-Snapshot-Timestamp');
-  current_date_display = $('#currently-displayed-data-date');
-  if (data_date > 0) {
-    current_date_display.html("Data is a snapshot from: " + moment.unix(data_date).format('DD/MM/YYYY HH:mm:ss Z'));
-  } else {
-    current_date_display.html("Pick a date to see data from the past");
-  }
-}
+}

data/samples/consul-ui/js/nodes.js

@@ -111,7 +111,7 @@ class NodeMainSelector extends MainSelector {
         nodesChecks.appendChild(checksStatusGenerator(node, node['Node']['Name']));
         content.appendChild(nodesChecks);
 
-        content.appendChild(servicesGenerator(node['Service']));
+        content.appendChild(servicesGenerator(node['Service'], node));
         content.appendChild(tagsGenerator(getTagsNode(node)));
 
         sidebar.setAttribute('status', state);

data/samples/consul-ui/js/service.js

@@ -304,7 +304,7 @@ class ServiceMainSelector extends MainSelector {
         element.setAttribute("class", "list-group-item service-instance");
         var state = nodeState(instance.checks);
         element.appendChild(weightsGenerator(instance.weights, state));
-        element.appendChild(serviceTitleGenerator(instance));
+        element.appendChild(serviceTitleGenerator(instance, serviceName));
         var node_info = this.nodes[instance.name];
         if (node_info != null) {
             node_info = node_info.meta;

data/samples/consul-ui/js/utils.js

@@ -63,7 +63,7 @@ function nodeState(checks) {
 
 supported_protocols = ['https', 'http', 'sftp', 'ftp', 'ssh', 'telnet']
 
-function serviceTitleGenerator(instance) {
+function serviceTitleGenerator(instance, serviceName) {
     var protocol = null;
     for (i in supported_protocols) {
         var protoc = supported_protocols[i]
@@ -74,10 +74,6 @@ function serviceTitleGenerator(instance) {
     }
 
     var htmlTitle = document.createElement('h5');
-    htmlTitle.setAttribute('title', 'Node Name: ' + instance.name +
-        '\nAddress: ' + instance.addr +
-        '\nService ID: ' + instance.id +
-        '\nService Port: ' + instance.port);
 
     htmlTitle.setAttribute('class', 'instance-name');
     var instanceLink = document.createElement('a');
@@ -91,6 +87,15 @@ function serviceTitleGenerator(instance) {
     }
 
     instanceLink.appendChild(document.createTextNode(instance.name + appendPort));
+    const nodeInfo = document.createElement('a');
+    nodeInfo.appendChild(document.createTextNode('\u24D8'));
+    nodeInfo.setAttribute('title', 'Click to see details of Node: ' + instance.name +
+        '\nAddress: ' + instance.addr +
+        '\nService ID: ' + instance.id +
+        '\nService Port: ' + instance.port);
+    nodeInfo.setAttribute('href', 'consul-nodes-ui.html?node_filter=^' + encodeURIComponent(instance.name) + '$');
+    htmlTitle.appendChild(nodeInfo);
+    htmlTitle.appendChild(document.createTextNode(' '));
     htmlTitle.appendChild(instanceLink);
     htmlTitle.appendChild(document.createTextNode(' '));
     htmlTitle.appendChild(document.createTextNode(instance.addr));
@@ -216,24 +221,44 @@ function toCSSClass(state) {
     return state;
 }
 
-function servicesGenerator(instanceServices) {
-    var services = document.createElement('div');
-    services.className = 'instance-services';
-    services.appendChild(document.createTextNode("Services: "));
-    services.appendChild(document.createElement('br'));
+function servicesGenerator(instanceServices, node) {
+    var servicesTop = document.createElement('div');
+    servicesTop.className = 'instance-services';
+    const card = document.createElement('div');
+    card.setAttribute('class', 'card');
+    const servicesCard = document.createElement('div');
+    servicesCard.setAttribute('class', 'card-body');
+    const title = document.createElement('h5');
+    title.appendChild(document.createTextNode('Services'));
+    title.setAttribute('class', 'card-title')
+    servicesCard.appendChild(title);
+    const services = document.createElement('div');
+    servicesCard.appendChild(services);
     for (var serviceKey in instanceServices) {
-        var service = document.createElement('a');
-        var serviceName = instanceServices[serviceKey]['Service']['Service'];
+        const serviceGrp = document.createElement('span');
+        serviceGrp.setAttribute('class', 'btn btn-sm');
+        serviceGrp.classList.add('btn-outline-' + toCSSClass(nodeState(instanceServices[serviceKey]['Checks'])))
+        const service = document.createElement('a');
+        serviceGrp.appendChild(service);
+        const serviceName = instanceServices[serviceKey]['Service']['Service'];
+        service.setAttribute('class', 'serviceLink');
+        service.setAttribute('href', 'consul-services-ui.html?service=' + encodeURIComponent(serviceName) + '&node_filter=^' + encodeURIComponent(node['Node']['Name'])+'$');
+        service.appendChild(document.createTextNode(serviceName));
         var servicePort = instanceServices[serviceKey]['Service']['Port'];
-        service.setAttribute('class', 'btn btn-sm m-1 lookup');
-        service.setAttribute('target', '_blank');
-        nodeAddr = instanceServices[serviceKey]['Service']['Address'];
-        service.setAttribute('href', 'http://' + nodeAddr + ':' + servicePort);
-        service.classList.add('btn-outline-' + toCSSClass(nodeState(instanceServices[serviceKey]['Checks'])))
-        service.appendChild(document.createTextNode(serviceName + ':' + servicePort));
-        services.appendChild(service);
+        if (servicePort) {
+            // Add unbreakable space
+            serviceGrp.appendChild(document.createTextNode('\u00A0'));
+            const servicePortElem = document.createElement('a');
+            servicePortElem.setAttribute('class', 'serviceTargetPort');
+            const nodeAddr = instanceServices[serviceKey]['Service']['Address'];
+            servicePortElem.setAttribute('href', 'http://' + nodeAddr + ':' + servicePort);
+            servicePortElem.appendChild(document.createTextNode(':' + servicePort));
+            serviceGrp.appendChild(servicePortElem);
+        }
+        services.appendChild(serviceGrp);
     }
-    return services;
+    servicesTop.appendChild(servicesCard);
+    return servicesTop;
 }
 
 function checksStatusGenerator(instance, prefix) {

data/samples/display_timestamped_changes.txt.erb

@@ -0,0 +1,17 @@
+<%
+  # This example show how to display local time informaition about changes
+  # Example of usage to display logs of changes on nodes() endpoint:
+  #
+  #   consul-templaterb --template "display_timestamped_changes.txt.erb:display_timestamped_changes.txt:cat display_timestamped_changes.txt" -l error
+  #
+  # Would output:
+  #   Last update: 1588800554 (2020-05-06 21:29:14 UTC), X-Consul-Index: 4345827328
+  #   Last update: 1588800569 (2020-05-06 21:29:29 UTC), X-Consul-Index: 4345829548
+  #   Last update: 1588800676 (2020-05-06 21:31:16 UTC), X-Consul-Index: 4345836342
+  #
+  @my_last_time = Time.now.utc unless @my_last_time
+  val = nodes()
+  new_idx = val.endpoint.x_consul_index
+  @my_last_time = Time.now.utc if @my_last_idx != new_idx
+  @my_last_idx = new_idx
+%>Last update: <%= @my_last_time.to_i %> (<%= @my_last_time %>), X-Consul-Index: <%= new_idx %>

data/samples/prometheus_datacenter_coordinates.erb

@@ -0,0 +1,56 @@
+# HELP consul_wan_servers_rtt_seconds Min Round trip with other servers of DC
+# TYPE consul_wan_servers_rtt_seconds gauge
+<%
+# This computes the RTT over WAN for all DCs
+results = {'min' => {}, 'max' => {}, 'p50' => {}, 'p90' => {}}
+my_dcname = (agent_self['Config'] || {})['Datacenter']
+my_dc = coordinate.datacenters.select { |d| d['Datacenter'] == my_dcname }.first
+
+def percentile(values_sorted, percentile)
+  k = (percentile*(values_sorted.length-1)+1).floor - 1
+  f = (percentile*(values_sorted.length-1)+1).modulo(1)
+  return values_sorted[k] + (f * (values_sorted[k+1] - values_sorted[k]))
+end
+
+if my_dc
+  from = my_dc['Datacenter']
+  coordinate.datacenters.each do |dc_coords|  
+    min = Float::MAX
+    max = 0
+    rtts = []
+    dc_coords['Coordinates'].each do |s2|
+      my_dc['Coordinates'].each do |s1|
+        rtt = coordinate.rtt(s1['Coord'], s2['Coord'])
+        min = rtt if rtt < min
+        max = rtt if rtt > max
+        idx = rtts.bsearch_index { |x| x > rtt }
+        if idx.nil?
+          rtts << rtt
+        else
+          rtts = rtts.insert(idx, rtt)
+        end
+%>
+consul_wan_servers_rtt_seconds{from="<%= from %>",to="<%= dc_coords['Datacenter'] %>",from_node="<%= s1['Node'] %>",to_node="<%= s2['Node'] %>"} <%= rtt.round(3) %><%
+      end
+    end
+    results['min'][dc_coords['Datacenter']] = min
+    results['max'][dc_coords['Datacenter']] = max
+    results['p50'][dc_coords['Datacenter']] = percentile(rtts, 0.5)
+    results['p90'][dc_coords['Datacenter']] = percentile(rtts, 0.9)
+  end
+
+  # Now, we iterate over aggregated results, type is min, max...
+  results.each do |type, res_type|
+  %>
+
+# HELP consul_wan_dc_<%= type %>_seconds <%= type %> round trip with other DCs
+# TYPE consul_wan_dc_<%= type %>_seconds gauge
+<%
+    # Iterate over destinations
+    res_type.each do |dst, val|
+%>consul_wan_dc_<%= type %>_seconds{from="<%= my_dcname %>",to="<%= dst %>"} <%= val.round(3) %>
+<%
+    end
+  end
+end
+%>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.26.2
+  version: 1.28.0
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-15 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -198,6 +198,7 @@ files:
 - samples/all_services.txt.erb
 - samples/all_services_multi_agents.txt.erb
 - samples/all_templates.erb
+- samples/checks_in_warning_or_critical_state.yaml.erb
 - samples/consul-ui/README.md
 - samples/consul-ui/common/footer.html.erb
 - samples/consul-ui/common/header.html.erb
@@ -229,6 +230,7 @@ files:
 - samples/criteo/haproxy.cfg.erb
 - samples/debug/compare_connect_services.txt.erb
 - samples/demos/compute_pricing.txt.erb
+- samples/display_timestamped_changes.txt.erb
 - samples/find_all_invalid_dns_labels.json.erb
 - samples/find_nodes_in_catalog_but_not_in_members.json.erb
 - samples/ha_proxy.cfg.erb
@@ -240,6 +242,7 @@ files:
 - samples/members.json.erb
 - samples/metrics.erb
 - samples/prometheus_consul_coordinates.erb
+- samples/prometheus_datacenter_coordinates.erb
 - samples/render_template_from_kv.erb
 - samples/sample_keys.html.erb
 - samples/service_checks_metrics.erb
@@ -270,8 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Implementation of Consul template using Ruby and .erb templating language