consul-templaterb 1.26.0 → 1.27.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79f77f92ae3811c48a823724fb4fb242b78fb78713de126cc035de4089a5e607
-  data.tar.gz: 3aa6c6f9c1aca03006963dd03ab30c3ba9ceabc4ff88c3a0d7fa8470a024b2d2
+  metadata.gz: 6c4d19fa22c559f4a7960529dd3960c1c741f631b1cf0b51f5d10155b2b8a131
+  data.tar.gz: 926930e64938a40c2a2653abf9a84f0c86057201646a2f8e02fed06ed51927b8
 SHA512:
-  metadata.gz: ed5763f1c109d678a867dddbe77a2b4e8d0b9adde1663f80e9bb6db2812c104f84edcd9f5c929d8e1f4763e9b4fa3af138da87c58af1285d6f1b0c6f5924760d
-  data.tar.gz: af7a53d991a486b4050632576b36c2cb1fe5e899bee75ec73a2e6bd76bb18e350913e5703b69ee8a05aa51b2d64a64a60b8c17dcc1e071b8cac3544331de0600
+  metadata.gz: 5126d79bcbd0d469bcb7ec5d214c979990056c3c11e1547d4819849f3fcd7fddd1534354f565722a52dbb09ba26c63dd4553f40282fcafaccd7be6a2a1d9442f
+  data.tar.gz: 7d6345911c69a8c0622f0b88343968ff951d8243db78d41b8cfad74e26ad5c87a39ae02e3bb24d4cf1615fe329fc53642f81aaf76591d1f5be4098c576baea96

data/.rubocop.yml

@@ -7,28 +7,28 @@ Layout/LineLength:
   Max: 175
 
 Metrics/AbcSize:
-  Max: 82
+  Max: 87
 
 Metrics/BlockLength:
-  Max: 160
+  Max: 182
 
 Metrics/BlockNesting:
   Max: 4
 
 Metrics/ClassLength:
-  Max: 275
+  Max: 285
 
 Metrics/CyclomaticComplexity:
-  Max: 20
+  Max: 21
 
 Metrics/MethodLength:
-  Max: 65
+  Max: 68
 
 Metrics/ParameterLists:
-  Max: 14
+  Max: 18
 
 Metrics/PerceivedComplexity:
-  Max: 23
+  Max: 24
 
 # We use `dc` as a parameter in many methods
 Naming/MethodParameterName:

data/.travis.yml

@@ -1,13 +1,13 @@
 language: ruby
 rvm:
-- 2.4.9
-- 2.5.7
-- 2.6.5
-- 2.7.0
+- 2.4.10
+- 2.5.8
+- 2.6.6
+- 2.7.1
 jobs:
   include:
     - stage: Gem release
-      rvm: 2.5.7
+      rvm: 2.5.8
       script: echo "Publishing consul-templaterb on rubygems.org ..."
       deploy:
         provider: rubygems

data/CHANGELOG.md

@@ -2,16 +2,54 @@
 
 ## (UNRELEASED)
 
+## 1.27.1 (July 28 2020)
+
+BUGIX:
+
+ * Fix collision in JSON queries when using payload in requests #68
+
+## 1.27.0 (June 5, 2020)
+
+NEW FEATURES:
+
+ * For Consul 1.7+, now support `checks_in_state(check_state, dc: nil, [agent: consul_agent_address])`,
+   fixes feature [#65](https://github.com/criteo/consul-templaterb/issues/65)
+ * New options to support/disable TLS validation thanks to [@jeromegn](https://github.com/jeromegn)
+   [#66](https://github.com/criteo/consul-templaterb/pull/66)
+
+## 1.26.3 (April 15, 2020)
+
+BUGFIX:
+
+ * Removed all Criteo specific decorators from Consul-UI
+
+## 1.26.2 (April 15, 2020)
+
+BUGFIX:
+
+ * Fixed broken Dockerfile (was missing the new `decorator.js.erb` file). Fixes #61 (Thanks to @ simongareste)
+
 NEW FEATURES:
 
+ * Added `consul_members_count` metric in [samples/metrics.erb](samples/metrics.erb)
+
+## 1.26.1 (March 27, 2020)
+
+BUGFIX:
+
+ * Using `agent: http://vault_or_consul_agent:port>` was not properly taken into account in some endpoints
+
 ## 1.26.0 (March 5, 2020)
 
+NEW FEATURES:
+
  * Using `agent: http://vault_or_consul_agent:port>` on most methods will now override the agent
    used to perform Consul queries. It might be useful for very large clusters or large WAN federations,
    because you can perform some requests on some agents, and some others on other agenrs. Might
    also be useful to federate data from preprod/prod for instance.
-   See [samples/all_services_multi_agents.txt.erb](samples/all_services_multi_agents.txt.erb) for an
-   example.
+ * Added agent attribute for all methods in documentation
+ * Added [samples/all_services_multi_agents.txt.erb](samples/all_services_multi_agents.txt.erb) as an
+   example of new feature.
 
 ## 1.25.2 (February 29, 2020)
 

data/Dockerfile

@@ -12,4 +12,4 @@ ENV LANG C.UTF-8
 ENV CONSUL_HTTP_ADDR http://consul-relay.service.consul.preprod.crto.in:8500
 
 ENTRYPOINT ["/usr/local/bin/bundle", "exec", "consul-templaterb"]
-CMD ["--template", "samples/consul-ui/consul-keys-ui.html.erb", "--template", "samples/consul-ui/consul-nodes-ui.html.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb", "--template", "samples/consul-ui/consul-timeline-ui.html.erb", "--template", "samples/consul-ui/consul_keys.json.erb", "--template", "samples/consul-ui/consul_nodes.json.erb", "--template", "samples/consul-ui/consul_services.json.erb", "--template", "samples/consul-ui/timeline.json.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb:samples/consul-ui/index.html:touch samples/consul-ui/ready", "--sig-reload=NONE", "--exec=nginx -c /usr/src/app/docker-nginx-conf/nginx.conf"]
+CMD ["--template", "samples/consul-ui/consul-keys-ui.html.erb", "--template", "samples/consul-ui/decorators.js.erb", "--template", "samples/consul-ui/consul-nodes-ui.html.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb", "--template", "samples/consul-ui/consul-timeline-ui.html.erb", "--template", "samples/consul-ui/consul_keys.json.erb", "--template", "samples/consul-ui/consul_nodes.json.erb", "--template", "samples/consul-ui/consul_services.json.erb", "--template", "samples/consul-ui/timeline.json.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb:samples/consul-ui/index.html:touch samples/consul-ui/ready", "--sig-reload=NONE", "--exec=nginx -c /usr/src/app/docker-nginx-conf/nginx.conf"]

data/README.md

@@ -332,7 +332,7 @@ Please consult [CHANGELOG.md](CHANGELOG.md) for fixed bugs.
 
 ## TODO
 
-* [x] Hashi's Vault support (EXPERIMENTAL)
+* [x] Hashi's Vault support
 * [ ] Implement automatic dynamic rate limit
 * [x] More samples: apache, nginx, a full website displaying consul information...
 * [x] Optimize rendering speed at start-up: an iteration is done every second by default, but it would be possible to speed

data/TemplateAPI.md

@@ -388,6 +388,15 @@ name or its ID. If DC is specified, will lookup for given node in another datace
 
 [Find all the checks](https://www.consul.io/api/health.html#list-checks-for-service) of a given service.
 
+## def checks_in_state(check_state, dc: nil, [agent: consul_agent_address])
+
+[Find all the checks in a given state](https://www.consul.io/api-docs/health#list-checks-in-state) in the whole cluster.
+
+The filter check_state must be one of any|critical|warning|passing.
+
+Warning: this endpoint might be very frequently updated in a
+large cluster if you are using `any` value. This endpoint is supported with Consul 1.7+.
+
 ## kv(name, [dc: nil], [keys: false], [recurse: false], [agent: consul_agent_address])
 
 [Read keys from KV Store](https://www.consul.io/api/kv.html#read-key). It can be used for both listing the keys and

data/bin/consul-templaterb

@@ -28,6 +28,9 @@ options = {
     },
     base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
     token: ENV['VAULT_TOKEN'] || nil,
+    tls_cert_chain: ENV['VAULT_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['VAULT_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     token_renew: true,
@@ -48,6 +51,9 @@ options = {
     },
     base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500',
     token: ENV['CONSUL_HTTP_TOKEN'] || nil,
+    tls_cert_chain: ENV['CONSUL_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['CONSUL_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     retry_duration: 10,      # On error, retry after n seconds
@@ -122,6 +128,18 @@ optparse = OptionParser.new do |opts|
     options[:consul][:base_url] = consul_url
   end
 
+  opts.on('--consul-cert-chain=<path/to/cert_chain>', String, 'Path to Consul TLS client certificate chain to use') do |consul_client_cert|
+    options[:consul][:tls_cert_chain] = consul_client_cert
+  end
+
+  opts.on('--consul-private-key=<path/to/private_key>', String, 'Path to Consul TLS client private key to use') do |consul_client_key|
+    options[:consul][:tls_private_key] = consul_client_key
+  end
+
+  opts.on('--skip-consul-verify-tls', 'Skip verifying Consul TLS via certificate authority (DANGEROUS)') do
+    options[:consul][:tls_verify_peer] = false
+  end
+
   opts.on('-l', '--log-level=<log_level>', String, "Log level, default=info, any of #{::Consul::Async::Debug.levels.join('|')}") do |log_level|
     ::Consul::Async::Debug.level = log_level
   end
@@ -134,6 +152,18 @@ optparse = OptionParser.new do |opts|
     options[:vault][:base_url] = vault_url
   end
 
+  opts.on('--vault-cert-chain=<path/to/cert_chain>', String, 'Path to Vault TLS client certificate chain to use') do |vault_client_cert|
+    options[:vault][:tls_cert_chain] = vault_client_cert
+  end
+
+  opts.on('--vault-private-key=<path/to/private_key>', String, 'Path to Vault TLS client private key to use') do |vault_client_key|
+    options[:vault][:tls_private_key] = vault_client_key
+  end
+
+  opts.on('--skip-vault-verify-tls', 'Skip verifying Vault TLS via certificate authority (DANGEROUS)') do
+    options[:vault][:tls_verify_peer] = false
+  end
+
   opts.on('-T', '--vault-token=<token>', String, 'Token used to authenticate against vault.') do |vault_token|
     options[:vault][:token] = vault_token
   end

data/docs/article-06_Template-based_discovery_with_consul-templaterb.md

@@ -120,4 +120,5 @@ If you don’t want to bother configuring and tuning it, you can try it very qui
 * [Mixing Observability with Service Discovery](https://medium.com/criteo-labs/mixing-observability-with-service-discovery-2bb8909e8530)
 
 Pierre Souchay, 2020-03-03
-Also published on https://medium.com/criteo-labs/template-based-discovery-with-consul-templaterb-8ff88434c457
+
+Also published on https://medium.com/criteo-labs/template-based-discovery-with-consul-templaterb-8ff88434c457

data/lib/consul/async/consul_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class ConsulConfiguration
       attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
                   :missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression,
-                  :fail_fast_errors, :max_consecutive_errors_on_endpoint
+                  :fail_fast_errors, :max_consecutive_errors_on_endpoint, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(base_url: 'http://localhost:8500',
                      debug: { network: false },
                      token: nil,
@@ -23,7 +23,10 @@ module Consul
                      enable_gzip_compression: true,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: 1)
+                     fail_fast_errors: 1,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token = token
         @debug = debug
@@ -38,6 +41,9 @@ module Consul
         @paths = paths
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -71,7 +77,10 @@ module Consul
                                 enable_gzip_compression: enable_gzip_compression,
                                 paths: @paths,
                                 max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
-                                fail_fast_errors: @fail_fast_errors)
+                                fail_fast_errors: @fail_fast_errors,
+                                tls_cert_chain: ch(path, :tls_cert_chain),
+                                tls_private_key: ch(path, :tls_private_key),
+                                tls_verify_peer: ch(path, :tls_verify_peer))
       end
     end
 
@@ -233,6 +242,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: conf.wait_duration + 1 + (conf.wait_duration / 16) # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.base_url, options)
         }

data/lib/consul/async/consul_template.rb

@@ -31,7 +31,8 @@ module Consul
 
       def as_json(url, default_value, refresh_delay_secs: 10, **opts)
         conf = JSONConfiguration.new(url: url, min_duration: refresh_delay_secs, retry_on_non_diff: refresh_delay_secs, **opts)
-        @endp_manager.create_if_missing(url, {}) do
+        endpoint_id = url + opts.to_json
+        @endp_manager.create_if_missing(url, {}, endpoint_id: endpoint_id) do
           if default_value.is_a?(Array)
             ConsulTemplateJSONArray.new(JSONEndpoint.new(conf, url, default_value))
           else
@@ -53,7 +54,7 @@ module Consul
         query_params = {}
         query_params[:dc] = dc if dc
         @endp_manager.create_if_missing(path, query_params, agent: agent) do
-          ConsulTemplateNodes.new(ConsulEndpoint.new(@endp_manager.consul_conf, path, true, query_params, '[]', agent: agent))
+          ConsulTemplateNodes.new(ConsulEndpoint.new(@endp_manager.consul_conf, path, true, query_params, '[]', agent))
         end
       end
 
@@ -63,7 +64,7 @@ module Consul
         query_params = {}
         query_params[:dc] = dc if dc
         @endp_manager.create_if_missing(path, query_params, agent: agent) do
-          ConsulTemplateNodes.new(ConsulEndpoint.new(@endp_manager.consul_conf, path, true, query_params, '[]', agent: agent))
+          ConsulTemplateNodes.new(ConsulEndpoint.new(@endp_manager.consul_conf, path, true, query_params, '[]', agent))
         end
       end
 
@@ -136,7 +137,7 @@ module Consul
         query_params[:dc] = dc if dc
         query_params[:passing] = passing if passing
         query_params[:tag] = tag if tag
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateService.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateService.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
       # https://www.consul.io/api/health.html#list-checks-for-service
@@ -147,7 +148,7 @@ module Consul
         query_params = {}
         query_params[:dc] = dc if dc
         query_params[:passing] = passing if passing
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
       # https://www.consul.io/api/health.html#list-checks-for-node
@@ -158,7 +159,19 @@ module Consul
         query_params = {}
         query_params[:dc] = dc if dc
         query_params[:passing] = passing if passing
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
+      end
+
+      # https://www.consul.io/api-docs/health#list-checks-in-state
+      # Supported in Consul 1.7+
+      def checks_in_state(check_state, dc: nil, agent: nil)
+        valid_checks_states = %w[any critical passing warning]
+        raise "checks_in_state('#{check_state}'...) must be one of #{valid_checks_states}" unless valid_checks_states.include?(check_state)
+
+        path = "/v1/health/state/#{check_state}"
+        query_params = {}
+        query_params[:dc] = dc if dc
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
       # https://www.consul.io/api/catalog.html#list-nodes
@@ -166,7 +179,7 @@ module Consul
         path = '/v1/catalog/nodes'
         query_params = {}
         query_params[:dc] = dc if dc
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateNodes.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateNodes.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
       # https://www.consul.io/api/catalog.html#list-services-for-node
@@ -174,7 +187,7 @@ module Consul
         path = "/v1/catalog/node/#{name_or_id}"
         query_params = {}
         query_params[:dc] = dc if dc
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateNodes.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '{}')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateNodes.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '{}', agent)) }
       end
 
       # https://www.consul.io/api/agent.html#read-configuration
@@ -182,7 +195,7 @@ module Consul
         path = '/v1/agent/self'
         query_params = {}
         default_value = '{"Config":{}, "Coord":{}, "Member":{}, "Meta":{}, "Stats":{}}'
-        create_if_missing(path, query_params, agent: agent) { ConsulAgentSelf.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value)) }
+        create_if_missing(path, query_params, agent: agent) { ConsulAgentSelf.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value, agent)) }
       end
 
       # https://www.consul.io/api/agent.html#view-metrics
@@ -190,7 +203,7 @@ module Consul
         path = '/v1/agent/metrics'
         query_params = {}
         default_value = '{"Gauges":[], "Points":[], "Member":{}, "Counters":[], "Samples":{}}'
-        create_if_missing(path, query_params, agent: agent) { ConsulAgentMetrics.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value)) }
+        create_if_missing(path, query_params, agent: agent) { ConsulAgentMetrics.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value, agent)) }
       end
 
       # https://www.consul.io/api/agent.html#list-members
@@ -199,7 +212,7 @@ module Consul
         query_params = {}
         query_params['wan'] = true if wan
         default_value = '[]'
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateMembers.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value)) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateMembers.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value, agent)) }
       end
 
       # Return a param of template
@@ -222,14 +235,14 @@ module Consul
         query_params[:dc] = dc if dc
         # Tag filtering is performed on client side
         query_params[:tag] = tag if tag
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateServices.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '{}')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateServices.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '{}', agent)) }
       end
 
       # https://www.consul.io/api/catalog.html#list-datacenters
       def datacenters(agent: nil)
         path = '/v1/catalog/datacenters'
         query_params = {}
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateDatacenters.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateDatacenters.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
       # https://www.consul.io/api/kv.html#read-key
@@ -240,7 +253,7 @@ module Consul
         query_params[:recurse] = recurse if recurse
         query_params[:keys] = keys if keys
         default_value = '[]'
-        create_if_missing(path, query_params, agent: agent) { ConsulTemplateKV.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value), name) }
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateKV.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value, agent), name) }
       end
 
       def secrets(path = '', agent: nil)
@@ -382,16 +395,21 @@ module Consul
         VaultEndpoint.new(vault_conf, path, :POST, {}, {})
       end
 
-      def create_if_missing(path, query_params, fail_fast_errors: @fail_fast_errors, max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint, agent: nil)
-        fqdn = path.dup
-        query_params.each_pair do |k, v|
-          fqdn = "#{agent}#{fqdn}&#{k}=#{v}"
-        end
-        tpl = @endpoints[fqdn]
+      def create_if_missing(path, query_params, fail_fast_errors: @fail_fast_errors,
+                            max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
+                            agent: nil, endpoint_id: nil)
+        endpoint_id ||= begin
+                          fqdn = path.dup
+                          query_params.each_pair do |k, v|
+                            fqdn = "#{agent}#{fqdn}&#{k}=#{v}"
+                          end
+                          fqdn
+                        end
+        tpl = @endpoints[endpoint_id]
         unless tpl
           tpl = yield
           ::Consul::Async::Debug.print_debug "path #{path.ljust(64)} #{query_params.inspect}\r"
-          @endpoints[fqdn] = tpl
+          @endpoints[endpoint_id] = tpl
           tpl.endpoint.on_response do |result|
             @net_info[:success] += 1
             @net_info[:bytes_read] += result.data.bytesize

data/lib/consul/async/json_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class JSONConfiguration
       attr_reader :url, :retry_duration, :min_duration, :retry_on_non_diff,
                   :debug, :enable_gzip_compression, :request_method, :json_body,
-                  :headers
+                  :headers, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(url:,
                      debug: { network: false },
                      retry_duration: 10,
@@ -18,7 +18,10 @@ module Consul
                      request_method: :get,
                      json_body: nil,
                      headers: {},
-                     enable_gzip_compression: true)
+                     enable_gzip_compression: true,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @url = url
         @debug = debug
         @enable_gzip_compression = enable_gzip_compression
@@ -28,6 +31,9 @@ module Consul
         @request_method = request_method
         @json_body = json_body
         @headers = headers
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def create(_url)
@@ -181,6 +187,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 60 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.url, options)
         }

data/lib/consul/async/vault_endpoint.rb

@@ -10,7 +10,8 @@ module Consul
     # Configuration for Vault Endpoints
     class VaultConfiguration
       attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
-                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors
+                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors, :tls_cert_chain, :tls_private_key,
+                  :tls_verify_peer
 
       def initialize(base_url: 'http://localhost:8200',
                      debug: { network: false },
@@ -22,7 +23,10 @@ module Consul
                      max_retry_duration: 600,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: false)
+                     fail_fast_errors: false,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token_renew = token_renew
         @debug = debug
@@ -34,6 +38,9 @@ module Consul
         @token = token
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -226,6 +233,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 1 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = EventMachine::HttpRequest.new(conf.base_url, options)
         cb = proc do |_|
           http = connection.send(http_method.downcase, build_request) # Under the hood: c.send('get', {stuff}) === c.get({stuff})

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.26.0'.freeze
+    VERSION = '1.27.1'.freeze
   end
 end

data/samples/checks_in_warning_or_critical_state.yaml.erb

@@ -0,0 +1,13 @@
+<%=
+  # This sample displays checks for the whole cluster
+  # in warning or critical state
+  # API available with Consul 1.7+
+  res = []
+  checks_in_state('warning').each do |c|
+    res << c
+  end
+  checks_in_state('critical').each do |c|
+    res << c
+  end
+  YAML.dump({'warning_or_critical_checks' => res})
+%>

data/samples/consul-ui/README.md

@@ -50,6 +50,25 @@ The content is statically created, so you can serve it using any HTTP server ver
 For development, you might use `python -m SimpleHTTPServer` in order to server the result
 on your browser.
 
+### Running it in production
+
+Whatever your solution, be sure to have a index.html, so read next below on
+how generating an index.html.
+
+#### Running with web server
+
+You can run it with your favoite web server, at Criteo we run it with nginx
+which handles nicely cache and offer good performance.
+
+In that case, consul-templaterb can start nginx with `--exec` or you can run it
+as a daemon, in which case, consul-templaterb only generate the files.
+
+#### Run it in Consul
+
+You can run consul with `-ui-dir=/path/to/directory/of/consul-ui`, in such case
+reaching consul on poort 8500 will redirect to the /ui/ path, displaying the UI
+of your choice on http://consul-agent.example.org:8500/ui/.
+
 ### Generating index.html
 
 By default, the command `consul-templaterb -c http://localhost:8500 samples/consul-ui/*.erb``

data/samples/consul-ui/decorators.js.erb

@@ -1,18 +1,5 @@
-// Utilities
+// Use this file to configure custom decorators for your Consul-UI
 var httpRegexp = new RegExp('^http[s]?://[^ ]+$');
-var dc = "<%= ENV['CRITEO_DC'] || 'par'%>";
-var env = "<%= ENV['CRITEO_ENV'] || 'preprod'%>";
-
-var availability_url = 'https://grafana.crto.in/d/xFX5gCnWz/service-availability?var-datacenter=' + dc + '&var-service=';
-if (env == 'preprod') {
-  availability_url = 'https://grafana.preprod.crto.in/d/E0ANGjnZz/service-availability?var-datacenter=' + dc + '&var-service=';
-}
-var swagger_url = 'https://swaggercatalogapp.' + dc + '.' + env + '.crto.in/explore/swagger?key=';
-var slack_url = 'https://criteo.slack.com/app_redirect?channel=';
-
-var rackguru_url = 'https://rackguru.' + env + '.crto.in/serial/';
-
-var asapi_url = 'https://idm.' + env + '.crto.in/tool/multiGroupInfo/'
 
 function url_decorator(key, value) {
   var e = document.createElement('a');
@@ -26,27 +13,7 @@ function usefullLinksGenerator(instance, serviceName, node_meta_info) {
     top.className = 'instance-links';
 
     var usefullLinks = [
-        {
-            title: "Trigger security scan",
-            iconClassName: "fas fa-shield-alt",
-            href: "https://security.crto.in/#/scan/?ip=" + instance.addr
-        },
-        {
-            title: "Availability Graph",
-            iconClassName: "fas fa-chart-area",
-            href: availability_url + serviceName
-        },
     ];
-    if (node_meta_info!= null) {
-      var serial = node_meta_info['serial_number'];
-      if (serial != null) {
-        usefullLinks.push({
-            title: "RackGuru",
-            iconClassName: "fas fa-server",
-            href: rackguru_url + serial
-        });
-      }
-    }
     var first = true;
     for (let usefullLink of usefullLinks) {
         link = document.createElement('a');
@@ -134,19 +101,6 @@ function groups_decorator(instance, key, value, serviceName) {
   return span;
 }
 
-/**
- * Decorates with a slack channel link
- */
-function slack_channel(instance, key, value, serviceName) {
-  var sName = value;
-  if (sName.startsWith('#')) {
-    sName = sName.substring(1);
-  }
-  return build_link(slack_url + encodeURIComponent(sName), '#'+ sName);
-}
-
-const start_regexp = /(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z/
-
 function decorateIsoDate(value, formated) {
   const parsed  = Date.parse(formated);
   if (isNaN(parsed)) {
@@ -163,55 +117,6 @@ function decorateIsoDate(value, formated) {
 // This is the list of function called
 // When a service meta is found
 var registered_decorators = {
-    alert_availability_slack_channel: slack_channel,
-    gerrit_repository: function(instance, key, value, serviceName) {
-      return build_link('https://review.crto.in/#/q/' + value, value);
-    },
-    version: function(instance, key, value, serviceName) {
-      var asInt = parseInt(value);
-      if (asInt < 10000) {
-        return document.createTextNode(value);
-      }
-      if (instance.sMeta['CRITEO_APP_POOL'] != null) {
-          return build_link("https://devtools.crto.in/log.html?moab=cs&im=nb-to&range=100%2C" + asInt, value);
-      } else {
-          var tUrl = "https://devtools.crto.in/log.html?moab=j&im=nb-to&range=100%2C" + asInt;
-          if (instance.sMeta['jvm_artifact'] != null) {
-            tUrl += '&with-dependencies=true&artifacts=' + encodeURIComponent(instance.sMeta['jvm_artifact']);
-          }
-          return build_link(tUrl, value);
-      }
-    },
-    MESOS_TERM_DEBUG_GRANTED_TO: groups_decorator,
-    OWNERS: groups_decorator,
-    marathon_app_id: function(instance, key, value, serviceName) {
-      var app_name = value;
-      if (app_name[0] != '/') {
-        app_name = '/' + app_name;
-      }
-      if (instance.sMeta && instance.sMeta['marathon_ui']) {
-        return build_link(instance.sMeta['marathon_ui'] + '/#/apps/' + encodeURIComponent(app_name), value);
-      } else {
-        // non decorated value
-        return document.createTextNode(value);
-      }
-    },
-    marathon_app_version: function(instance, key, value, serviceName) {
-      return decorateIsoDate(value, value);
-    },
-    slack_channel: slack_channel,
-    start: function(instance, key, value, serviceName) {
-        const reg = start_regexp.exec(value);
-        if (reg != null) {
-          var formated = reg[1] + '-' + reg[2] + '-' + reg[3] + 'T' + reg[4] + ':' + reg[5] + ':' + reg[6] + 'Z';
-          return decorateIsoDate(value, formated);
-        } else {
-          return document.createTextNode(value);
-        }
-    },
-    swagger_key: function(instance, key, value, serviceName) {
-        return build_link(swagger_url + encodeURIComponent(value), value);
-    },
 }
 
 function service_meta_semantics_decorator(instance, key, value, serviceName) {
@@ -286,85 +191,11 @@ function nodeMetaGenerator(nodeMetaTags, serviceMetaIfAny) {
  * it does not have to return anything.
  */
 function navBarDecorator(navbar) {
-  if (typeof consulManager === 'undefined') {
-    // Timepicker is not supported on this page
-    return;
-  }
-  var timepicker_container = document.createElement('div');
-  timepicker_container.innerHTML = `
-              <div class="row">
-                <button type="button" disabled class="btn btn-secondary" data-toggle="tooltip" data-html="true" title="Data returned is the closest available to request" id="currently-displayed-data-date">
-                  Pick a date to see data from the past
-                </button>
-                <div class="col-sm-14">
-                  <div class="form-group">
-                    <div class="input-group date" id="datetimepicker1" data-target-input="nearest">
-                      <input type="text" class="form-control datetimepicker-input" data-target="#datetimepicker1"/>
-                      <div class="input-group-append" data-target="#datetimepicker1" data-toggle="datetimepicker">
-                        <div class="input-group-text"><i class="fa fa-calendar"></i></div>
-                      </div>
-                    </div>
-                  </div>
-                </div>
-              </div>
-              `;
-
-  navbar.appendChild(timepicker_container);
-  var script= document.createElement('script');
-  script.type='text/javascript';
-  // TODO(g.seux): extract code to another file and set "source" on script element
-  script.innerHTML = `
-                    $('#datetimepicker1').datetimepicker({
-                      format: 'DD/MM/YYYY HH:mm:ss Z',  // default format does not allow to select seconds
-                      sideBySide: true, // display date+time on the same widget
-                      useCurrent: true // by default, select current date
-                    });
-                  $("#datetimepicker1").on("change.datetimepicker", function (e) {
-                    console.log("Will fetch data from date " + e.date);
-                    console.log("will clean existing data");
-                    consulManager.clean().then(function(result) {
-                      switch(consulManager.constructor.name) {
-                        case "ConsulServiceManager":
-                          backup_type = 'consul_services';
-                          break;
-                        case "ConsulKeysManager":
-                          backup_type  = 'consul_keys'
-                          break;
-                        case "ConsulNodesManager":
-                          backup_type = 'consul_nodes'
-                          break;
-                        default:
-                          console.log("Unknown " + consulManager.constructor.name + " type");
-                      }
-                      if (e.date) {
-                        console.log("Will replace data by closest snapshot to " + e.date);
-                        var target_url = "https://consul-info-timeline-history.<%= ENV['CRITEO_DC'] %>.<%= ENV['CRITEO_ENV']%>.crto.in/backup/" + backup_type + "/" + e.date / 1000;
-                      } else {
-                        console.log("Will restore to local version");
-                        var target_url = defaultConsulManager.resourceURL;
-                      }
-                      if (typeof(defaultConsulManager) == 'undefined') {
-                        // store first manager to be able to restore it
-                        defaultConsulManager = consulManager
-                      }
-                      consulManager = new consulManager.constructor(target_url);
-                    });
-                  });
-  `;
-  navbar.appendChild(script);
 }
 
-
 /**
  * fetchedResponseDecorator is called with http response when a resource is fetched by any instance of ConsulUIManager
  * it does not have to return anything.
  */
 async function fetchedResponseDecorator(httpResponse) {
-  const data_date = await httpResponse.headers.get('X-Consul-Snapshot-Timestamp');
-  current_date_display = $('#currently-displayed-data-date');
-  if (data_date > 0) {
-    current_date_display.html("Data is a snapshot from: " + moment.unix(data_date).format('DD/MM/YYYY HH:mm:ss Z'));
-  } else {
-    current_date_display.html("Pick a date to see data from the past");
-  }
-}
+}

data/samples/display_timestamped_changes.txt.erb

@@ -0,0 +1,17 @@
+<%
+  # This example show how to display local time informaition about changes
+  # Example of usage to display logs of changes on nodes() endpoint:
+  #
+  #   consul-templaterb --template "display_timestamped_changes.txt.erb:display_timestamped_changes.txt:cat display_timestamped_changes.txt" -l error
+  #
+  # Would output:
+  #   Last update: 1588800554 (2020-05-06 21:29:14 UTC), X-Consul-Index: 4345827328
+  #   Last update: 1588800569 (2020-05-06 21:29:29 UTC), X-Consul-Index: 4345829548
+  #   Last update: 1588800676 (2020-05-06 21:31:16 UTC), X-Consul-Index: 4345836342
+  #
+  @my_last_time = Time.now.utc unless @my_last_time
+  val = nodes()
+  new_idx = val.endpoint.x_consul_index
+  @my_last_time = Time.now.utc if @my_last_idx != new_idx
+  @my_last_idx = new_idx
+%>Last update: <%= @my_last_time.to_i %> (<%= @my_last_time %>), X-Consul-Index: <%= new_idx %>

data/samples/find_all_invalid_dns_labels.json.erb

@@ -0,0 +1,12 @@
+<%
+  r = /^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,62}[a-zA-Z0-9])?$/
+  invalid_services = {}
+  services.each do |servive_name, tags|
+    errors = []
+    errors << "Invalid service name #{servive_name}" unless r.match(servive_name)
+    tags.each do |tag|
+      errors << "Invalid tag #{tag}" unless r.match(tag)
+    end
+    invalid_services[servive_name] = errors unless errors.empty?
+  end
+%><%= JSON.pretty_generate({"errors": invalid_services.size, "details": invalid_services}) %>

data/samples/find_nodes_in_catalog_but_not_in_members.json.erb

@@ -0,0 +1,28 @@
+<%= 
+   all_catalog = nodes.map{ |n| [n['Node'], n] }.to_h
+   all_members = agent_members.sort{ |a,b| a['Name'] <=> b['Name'] }
+   members_by_state = {}
+   all_members.each do |m|
+     state = m.status
+     members = members_by_state[state] || 0
+     members_by_state[state] = members + 1
+   end
+   # Find all virtual nodes (eg: k8s-sync)
+   catalog_external_sources = nodes.select{|n| (n['Meta']||{})['external-source'] }.map{|n| n['Node']}
+   in_catalog_but_not_in_members = all_catalog.keys - all_members.map{ |m| m['Name'] }
+   not_virtual_not_in_members = in_catalog_but_not_in_members - catalog_external_sources
+   JSON.pretty_generate(
+    {
+      'stats' => {
+        'catalog' => all_catalog.count,
+        'catalog_virtual' => in_catalog_but_not_in_members.count,
+        'members' => all_members.count,
+        'members_by_state': members_by_state,
+        'catalog_external_sources' => catalog_external_sources,
+      },
+      'errors': {
+        'in_catalog_but_not_in_members' => in_catalog_but_not_in_members,
+        'not_virtual_not_in_members'    =>  not_virtual_not_in_members,
+      }
+   })
+%>

data/samples/metrics.erb

@@ -21,6 +21,20 @@
   all_stats = {}
   service_count = 0
 
+  statuses = agent_members().map {|m| m.status}.reduce({}) do |sum, s|
+    v = sum[s] || 0
+    sum[s] = v + 1
+    sum
+  end
+%>
+# HELP consul_members_count A gauge of number of serf members with their count and their status('alive', 'leaving', 'left', 'failed')
+# TYPE consul_members_count gauge
+<%
+  statuses.each do |k, v|
+%>
+consul_members_count{serf="lan",status="<%= k %>"} <%= v %><%
+  end
+
   services(tag: service_tag_filter).each do |service_name, tags|
     if !services_blacklist.any? {|r| r.match(service_name)} && (instance_must_tag.nil? || tags.include?(instance_must_tag))
       service_count += 1

data/samples/prometheus_datacenter_coordinates.erb

@@ -0,0 +1,56 @@
+# HELP consul_wan_servers_rtt_seconds Min Round trip with other servers of DC
+# TYPE consul_wan_servers_rtt_seconds gauge
+<%
+# This computes the RTT over WAN for all DCs
+results = {'min' => {}, 'max' => {}, 'p50' => {}, 'p90' => {}}
+my_dcname = (agent_self['Config'] || {})['Datacenter']
+my_dc = coordinate.datacenters.select { |d| d['Datacenter'] == my_dcname }.first
+
+def percentile(values_sorted, percentile)
+  k = (percentile*(values_sorted.length-1)+1).floor - 1
+  f = (percentile*(values_sorted.length-1)+1).modulo(1)
+  return values_sorted[k] + (f * (values_sorted[k+1] - values_sorted[k]))
+end
+
+if my_dc
+  from = my_dc['Datacenter']
+  coordinate.datacenters.each do |dc_coords|  
+    min = Float::MAX
+    max = 0
+    rtts = []
+    dc_coords['Coordinates'].each do |s2|
+      my_dc['Coordinates'].each do |s1|
+        rtt = coordinate.rtt(s1['Coord'], s2['Coord'])
+        min = rtt if rtt < min
+        max = rtt if rtt > max
+        idx = rtts.bsearch_index { |x| x > rtt }
+        if idx.nil?
+          rtts << rtt
+        else
+          rtts = rtts.insert(idx, rtt)
+        end
+%>
+consul_wan_servers_rtt_seconds{from="<%= from %>",to="<%= dc_coords['Datacenter'] %>",from_node="<%= s1['Node'] %>",to_node="<%= s2['Node'] %>"} <%= rtt.round(3) %><%
+      end
+    end
+    results['min'][dc_coords['Datacenter']] = min
+    results['max'][dc_coords['Datacenter']] = max
+    results['p50'][dc_coords['Datacenter']] = percentile(rtts, 0.5)
+    results['p90'][dc_coords['Datacenter']] = percentile(rtts, 0.9)
+  end
+
+  # Now, we iterate over aggregated results, type is min, max...
+  results.each do |type, res_type|
+  %>
+
+# HELP consul_wan_dc_<%= type %>_seconds <%= type %> round trip with other DCs
+# TYPE consul_wan_dc_<%= type %>_seconds gauge
+<%
+    # Iterate over destinations
+    res_type.each do |dst, val|
+%>consul_wan_dc_<%= type %>_seconds{from="<%= my_dcname %>",to="<%= dst %>"} <%= val.round(3) %>
+<%
+    end
+  end
+end
+%>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.26.0
+  version: 1.27.1
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-05 00:00:00.000000000 Z
+date: 2020-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -198,6 +198,7 @@ files:
 - samples/all_services.txt.erb
 - samples/all_services_multi_agents.txt.erb
 - samples/all_templates.erb
+- samples/checks_in_warning_or_critical_state.yaml.erb
 - samples/consul-ui/README.md
 - samples/consul-ui/common/footer.html.erb
 - samples/consul-ui/common/header.html.erb
@@ -229,6 +230,9 @@ files:
 - samples/criteo/haproxy.cfg.erb
 - samples/debug/compare_connect_services.txt.erb
 - samples/demos/compute_pricing.txt.erb
+- samples/display_timestamped_changes.txt.erb
+- samples/find_all_invalid_dns_labels.json.erb
+- samples/find_nodes_in_catalog_but_not_in_members.json.erb
 - samples/ha_proxy.cfg.erb
 - samples/haproxy_dns.cfg.erb
 - samples/hosts.erb
@@ -238,6 +242,7 @@ files:
 - samples/members.json.erb
 - samples/metrics.erb
 - samples/prometheus_consul_coordinates.erb
+- samples/prometheus_datacenter_coordinates.erb
 - samples/render_template_from_kv.erb
 - samples/sample_keys.html.erb
 - samples/service_checks_metrics.erb
@@ -268,8 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Implementation of Consul template using Ruby and .erb templating language