consul-templaterb 1.26.1 → 1.27.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91552da1b0e2061c69095c958665896efc6de84ae6c5374a1b66d1c9550a3297
-  data.tar.gz: 8bdf67de550a4e2cb7d621ccad117e1e0841b3a0a49bcefb5c914e63daeddb41
+  metadata.gz: a67e2df58bcd5c322789984815ff60bd95e50fc047cb20c929a67ca29c07f553
+  data.tar.gz: c4df4357ae2852ad487eb90336a0d2d7d380767d51d05e02e7b452d29acda9c7
 SHA512:
-  metadata.gz: 24a9e291106b8e32ddef2bdc3d5e87ded36122c84dd1f0547fb034410d606609dd0041d011cd4eeb58da7b38a579888fb5901e877814d49926ef7a0722020d15
-  data.tar.gz: d3b1b5578cfdc3ee7c594916fc53e22692506d1bc0faaafd91ca21891665732528e95a384766548c90c75bcec0b515f856d1236ffbd18572911363d6ab8f836b
+  metadata.gz: 723beb9fca614d1675e50cc1516f37c4aad93815a1725abcdecb33c8bf0aa5e399449f8c3826a14eac8705aa8cb354aa1034796cf68b46304624f958c1b2ba90
+  data.tar.gz: 379cb1ca86d8edc44c68694f70b6bd7c1373b8b707f26bef3459379554f94fa12f27653354dc52647eadbd2cf5b0df7cec7f01bcb457e7ed159b7396709ce00d

data/.rubocop.yml

@@ -7,28 +7,28 @@ Layout/LineLength:
   Max: 175
 
 Metrics/AbcSize:
-  Max: 82
+  Max: 87
 
 Metrics/BlockLength:
-  Max: 160
+  Max: 182
 
 Metrics/BlockNesting:
   Max: 4
 
 Metrics/ClassLength:
-  Max: 275
+  Max: 285
 
 Metrics/CyclomaticComplexity:
-  Max: 20
+  Max: 21
 
 Metrics/MethodLength:
-  Max: 65
+  Max: 68
 
 Metrics/ParameterLists:
-  Max: 14
+  Max: 18
 
 Metrics/PerceivedComplexity:
-  Max: 23
+  Max: 24
 
 # We use `dc` as a parameter in many methods
 Naming/MethodParameterName:

data/.travis.yml

@@ -1,13 +1,13 @@
 language: ruby
 rvm:
-- 2.4.9
-- 2.5.7
-- 2.6.5
-- 2.7.0
+- 2.4.10
+- 2.5.8
+- 2.6.6
+- 2.7.1
 jobs:
   include:
     - stage: Gem release
-      rvm: 2.5.7
+      rvm: 2.5.8
       script: echo "Publishing consul-templaterb on rubygems.org ..."
       deploy:
         provider: rubygems

data/CHANGELOG.md

@@ -2,6 +2,43 @@
 
 ## (UNRELEASED)
 
+## 1.27.2 (Sept 4, 2020)
+
+IMPROVEMENTS:
+
+ * Consul-UI now supports navigation between nodes and services in both ways
+
+## 1.27.1 (July 28, 2020)
+
+BUGIX:
+
+ * Fix collision in JSON queries when using payload in requests #68
+
+## 1.27.0 (June 5, 2020)
+
+NEW FEATURES:
+
+ * For Consul 1.7+, now support `checks_in_state(check_state, dc: nil, [agent: consul_agent_address])`,
+   fixes feature [#65](https://github.com/criteo/consul-templaterb/issues/65)
+ * New options to support/disable TLS validation thanks to [@jeromegn](https://github.com/jeromegn)
+   [#66](https://github.com/criteo/consul-templaterb/pull/66)
+
+## 1.26.3 (April 15, 2020)
+
+BUGFIX:
+
+ * Removed all Criteo specific decorators from Consul-UI
+
+## 1.26.2 (April 15, 2020)
+
+BUGFIX:
+
+ * Fixed broken Dockerfile (was missing the new `decorator.js.erb` file). Fixes #61 (Thanks to @ simongareste)
+
+NEW FEATURES:
+
+ * Added `consul_members_count` metric in [samples/metrics.erb](samples/metrics.erb)
+
 ## 1.26.1 (March 27, 2020)
 
 BUGFIX:

data/Dockerfile

@@ -12,4 +12,4 @@ ENV LANG C.UTF-8
 ENV CONSUL_HTTP_ADDR http://consul-relay.service.consul.preprod.crto.in:8500
 
 ENTRYPOINT ["/usr/local/bin/bundle", "exec", "consul-templaterb"]
-CMD ["--template", "samples/consul-ui/consul-keys-ui.html.erb", "--template", "samples/consul-ui/consul-nodes-ui.html.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb", "--template", "samples/consul-ui/consul-timeline-ui.html.erb", "--template", "samples/consul-ui/consul_keys.json.erb", "--template", "samples/consul-ui/consul_nodes.json.erb", "--template", "samples/consul-ui/consul_services.json.erb", "--template", "samples/consul-ui/timeline.json.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb:samples/consul-ui/index.html:touch samples/consul-ui/ready", "--sig-reload=NONE", "--exec=nginx -c /usr/src/app/docker-nginx-conf/nginx.conf"]
+CMD ["--template", "samples/consul-ui/consul-keys-ui.html.erb", "--template", "samples/consul-ui/decorators.js.erb", "--template", "samples/consul-ui/consul-nodes-ui.html.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb", "--template", "samples/consul-ui/consul-timeline-ui.html.erb", "--template", "samples/consul-ui/consul_keys.json.erb", "--template", "samples/consul-ui/consul_nodes.json.erb", "--template", "samples/consul-ui/consul_services.json.erb", "--template", "samples/consul-ui/timeline.json.erb", "--template", "samples/consul-ui/consul-services-ui.html.erb:samples/consul-ui/index.html:touch samples/consul-ui/ready", "--sig-reload=NONE", "--exec=nginx -c /usr/src/app/docker-nginx-conf/nginx.conf"]

data/README.md

@@ -332,7 +332,7 @@ Please consult [CHANGELOG.md](CHANGELOG.md) for fixed bugs.
 
 ## TODO
 
-* [x] Hashi's Vault support (EXPERIMENTAL)
+* [x] Hashi's Vault support
 * [ ] Implement automatic dynamic rate limit
 * [x] More samples: apache, nginx, a full website displaying consul information...
 * [x] Optimize rendering speed at start-up: an iteration is done every second by default, but it would be possible to speed

data/TemplateAPI.md

@@ -388,6 +388,15 @@ name or its ID. If DC is specified, will lookup for given node in another datace
 
 [Find all the checks](https://www.consul.io/api/health.html#list-checks-for-service) of a given service.
 
+## def checks_in_state(check_state, dc: nil, [agent: consul_agent_address])
+
+[Find all the checks in a given state](https://www.consul.io/api-docs/health#list-checks-in-state) in the whole cluster.
+
+The filter check_state must be one of any|critical|warning|passing.
+
+Warning: this endpoint might be very frequently updated in a
+large cluster if you are using `any` value. This endpoint is supported with Consul 1.7+.
+
 ## kv(name, [dc: nil], [keys: false], [recurse: false], [agent: consul_agent_address])
 
 [Read keys from KV Store](https://www.consul.io/api/kv.html#read-key). It can be used for both listing the keys and

data/bin/consul-templaterb

@@ -28,6 +28,9 @@ options = {
     },
     base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
     token: ENV['VAULT_TOKEN'] || nil,
+    tls_cert_chain: ENV['VAULT_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['VAULT_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     token_renew: true,
@@ -48,6 +51,9 @@ options = {
     },
     base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500',
     token: ENV['CONSUL_HTTP_TOKEN'] || nil,
+    tls_cert_chain: ENV['CONSUL_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['CONSUL_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     retry_duration: 10,      # On error, retry after n seconds
@@ -122,6 +128,18 @@ optparse = OptionParser.new do |opts|
     options[:consul][:base_url] = consul_url
   end
 
+  opts.on('--consul-cert-chain=<path/to/cert_chain>', String, 'Path to Consul TLS client certificate chain to use') do |consul_client_cert|
+    options[:consul][:tls_cert_chain] = consul_client_cert
+  end
+
+  opts.on('--consul-private-key=<path/to/private_key>', String, 'Path to Consul TLS client private key to use') do |consul_client_key|
+    options[:consul][:tls_private_key] = consul_client_key
+  end
+
+  opts.on('--skip-consul-verify-tls', 'Skip verifying Consul TLS via certificate authority (DANGEROUS)') do
+    options[:consul][:tls_verify_peer] = false
+  end
+
   opts.on('-l', '--log-level=<log_level>', String, "Log level, default=info, any of #{::Consul::Async::Debug.levels.join('|')}") do |log_level|
     ::Consul::Async::Debug.level = log_level
   end
@@ -134,6 +152,18 @@ optparse = OptionParser.new do |opts|
     options[:vault][:base_url] = vault_url
   end
 
+  opts.on('--vault-cert-chain=<path/to/cert_chain>', String, 'Path to Vault TLS client certificate chain to use') do |vault_client_cert|
+    options[:vault][:tls_cert_chain] = vault_client_cert
+  end
+
+  opts.on('--vault-private-key=<path/to/private_key>', String, 'Path to Vault TLS client private key to use') do |vault_client_key|
+    options[:vault][:tls_private_key] = vault_client_key
+  end
+
+  opts.on('--skip-vault-verify-tls', 'Skip verifying Vault TLS via certificate authority (DANGEROUS)') do
+    options[:vault][:tls_verify_peer] = false
+  end
+
   opts.on('-T', '--vault-token=<token>', String, 'Token used to authenticate against vault.') do |vault_token|
     options[:vault][:token] = vault_token
   end

data/lib/consul/async/consul_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class ConsulConfiguration
       attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
                   :missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression,
-                  :fail_fast_errors, :max_consecutive_errors_on_endpoint
+                  :fail_fast_errors, :max_consecutive_errors_on_endpoint, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(base_url: 'http://localhost:8500',
                      debug: { network: false },
                      token: nil,
@@ -23,7 +23,10 @@ module Consul
                      enable_gzip_compression: true,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: 1)
+                     fail_fast_errors: 1,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token = token
         @debug = debug
@@ -38,6 +41,9 @@ module Consul
         @paths = paths
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -71,7 +77,10 @@ module Consul
                                 enable_gzip_compression: enable_gzip_compression,
                                 paths: @paths,
                                 max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
-                                fail_fast_errors: @fail_fast_errors)
+                                fail_fast_errors: @fail_fast_errors,
+                                tls_cert_chain: ch(path, :tls_cert_chain),
+                                tls_private_key: ch(path, :tls_private_key),
+                                tls_verify_peer: ch(path, :tls_verify_peer))
       end
     end
 
@@ -233,6 +242,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: conf.wait_duration + 1 + (conf.wait_duration / 16) # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.base_url, options)
         }

data/lib/consul/async/consul_template.rb

@@ -31,7 +31,8 @@ module Consul
 
       def as_json(url, default_value, refresh_delay_secs: 10, **opts)
         conf = JSONConfiguration.new(url: url, min_duration: refresh_delay_secs, retry_on_non_diff: refresh_delay_secs, **opts)
-        @endp_manager.create_if_missing(url, {}) do
+        endpoint_id = url + opts.to_json
+        @endp_manager.create_if_missing(url, {}, endpoint_id: endpoint_id) do
           if default_value.is_a?(Array)
             ConsulTemplateJSONArray.new(JSONEndpoint.new(conf, url, default_value))
           else
@@ -161,6 +162,18 @@ module Consul
         create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
       end
 
+      # https://www.consul.io/api-docs/health#list-checks-in-state
+      # Supported in Consul 1.7+
+      def checks_in_state(check_state, dc: nil, agent: nil)
+        valid_checks_states = %w[any critical passing warning]
+        raise "checks_in_state('#{check_state}'...) must be one of #{valid_checks_states}" unless valid_checks_states.include?(check_state)
+
+        path = "/v1/health/state/#{check_state}"
+        query_params = {}
+        query_params[:dc] = dc if dc
+        create_if_missing(path, query_params, agent: agent) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]', agent)) }
+      end
+
       # https://www.consul.io/api/catalog.html#list-nodes
       def nodes(dc: nil, agent: nil)
         path = '/v1/catalog/nodes'
@@ -382,16 +395,21 @@ module Consul
         VaultEndpoint.new(vault_conf, path, :POST, {}, {})
       end
 
-      def create_if_missing(path, query_params, fail_fast_errors: @fail_fast_errors, max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint, agent: nil)
-        fqdn = path.dup
-        query_params.each_pair do |k, v|
-          fqdn = "#{agent}#{fqdn}&#{k}=#{v}"
-        end
-        tpl = @endpoints[fqdn]
+      def create_if_missing(path, query_params, fail_fast_errors: @fail_fast_errors,
+                            max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
+                            agent: nil, endpoint_id: nil)
+        endpoint_id ||= begin
+                          fqdn = path.dup
+                          query_params.each_pair do |k, v|
+                            fqdn = "#{agent}#{fqdn}&#{k}=#{v}"
+                          end
+                          fqdn
+                        end
+        tpl = @endpoints[endpoint_id]
         unless tpl
           tpl = yield
           ::Consul::Async::Debug.print_debug "path #{path.ljust(64)} #{query_params.inspect}\r"
-          @endpoints[fqdn] = tpl
+          @endpoints[endpoint_id] = tpl
           tpl.endpoint.on_response do |result|
             @net_info[:success] += 1
             @net_info[:bytes_read] += result.data.bytesize

data/lib/consul/async/json_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class JSONConfiguration
       attr_reader :url, :retry_duration, :min_duration, :retry_on_non_diff,
                   :debug, :enable_gzip_compression, :request_method, :json_body,
-                  :headers
+                  :headers, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(url:,
                      debug: { network: false },
                      retry_duration: 10,
@@ -18,7 +18,10 @@ module Consul
                      request_method: :get,
                      json_body: nil,
                      headers: {},
-                     enable_gzip_compression: true)
+                     enable_gzip_compression: true,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @url = url
         @debug = debug
         @enable_gzip_compression = enable_gzip_compression
@@ -28,6 +31,9 @@ module Consul
         @request_method = request_method
         @json_body = json_body
         @headers = headers
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def create(_url)
@@ -181,6 +187,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 60 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.url, options)
         }

data/lib/consul/async/vault_endpoint.rb

@@ -10,7 +10,8 @@ module Consul
     # Configuration for Vault Endpoints
     class VaultConfiguration
       attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
-                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors
+                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors, :tls_cert_chain, :tls_private_key,
+                  :tls_verify_peer
 
       def initialize(base_url: 'http://localhost:8200',
                      debug: { network: false },
@@ -22,7 +23,10 @@ module Consul
                      max_retry_duration: 600,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: false)
+                     fail_fast_errors: false,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token_renew = token_renew
         @debug = debug
@@ -34,6 +38,9 @@ module Consul
         @token = token
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -226,6 +233,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 1 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = EventMachine::HttpRequest.new(conf.base_url, options)
         cb = proc do |_|
           http = connection.send(http_method.downcase, build_request) # Under the hood: c.send('get', {stuff}) === c.get({stuff})

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.26.1'.freeze
+    VERSION = '1.27.2'.freeze
   end
 end

data/samples/checks_in_warning_or_critical_state.yaml.erb

@@ -0,0 +1,13 @@
+<%=
+  # This sample displays checks for the whole cluster
+  # in warning or critical state
+  # API available with Consul 1.7+
+  res = []
+  checks_in_state('warning').each do |c|
+    res << c
+  end
+  checks_in_state('critical').each do |c|
+    res << c
+  end
+  YAML.dump({'warning_or_critical_checks' => res})
+%>

data/samples/consul-ui/README.md

@@ -50,6 +50,25 @@ The content is statically created, so you can serve it using any HTTP server ver
 For development, you might use `python -m SimpleHTTPServer` in order to server the result
 on your browser.
 
+### Running it in production
+
+Whatever your solution, be sure to have a index.html, so read next below on
+how generating an index.html.
+
+#### Running with web server
+
+You can run it with your favoite web server, at Criteo we run it with nginx
+which handles nicely cache and offer good performance.
+
+In that case, consul-templaterb can start nginx with `--exec` or you can run it
+as a daemon, in which case, consul-templaterb only generate the files.
+
+#### Run it in Consul
+
+You can run consul with `-ui-dir=/path/to/directory/of/consul-ui`, in such case
+reaching consul on poort 8500 will redirect to the /ui/ path, displaying the UI
+of your choice on http://consul-agent.example.org:8500/ui/.
+
 ### Generating index.html
 
 By default, the command `consul-templaterb -c http://localhost:8500 samples/consul-ui/*.erb``

data/samples/consul-ui/css/style.css

@@ -15,6 +15,10 @@
   content: '⚠ ';
 }
 
+.btn-outline-success a:hover, .btn-outline-warning a:hover, .btn-outline-danger a:hover{
+  color: #fff;
+}
+
 #service-wrapper, #instances-wrapper, #keys-wrapper {
   overflow-y: scroll;
   border-top-left-radius: 0px;

data/samples/consul-ui/decorators.js.erb

@@ -1,18 +1,5 @@
-// Utilities
+// Use this file to configure custom decorators for your Consul-UI
 var httpRegexp = new RegExp('^http[s]?://[^ ]+$');
-var dc = "<%= ENV['CRITEO_DC'] || 'par'%>";
-var env = "<%= ENV['CRITEO_ENV'] || 'preprod'%>";
-
-var availability_url = 'https://grafana.crto.in/d/xFX5gCnWz/service-availability?var-datacenter=' + dc + '&var-service=';
-if (env == 'preprod') {
-  availability_url = 'https://grafana.preprod.crto.in/d/E0ANGjnZz/service-availability?var-datacenter=' + dc + '&var-service=';
-}
-var swagger_url = 'https://swaggercatalogapp.' + dc + '.' + env + '.crto.in/explore/swagger?key=';
-var slack_url = 'https://criteo.slack.com/app_redirect?channel=';
-
-var rackguru_url = 'https://rackguru.' + env + '.crto.in/serial/';
-
-var asapi_url = 'https://idm.' + env + '.crto.in/tool/multiGroupInfo/'
 
 function url_decorator(key, value) {
   var e = document.createElement('a');
@@ -26,27 +13,7 @@ function usefullLinksGenerator(instance, serviceName, node_meta_info) {
     top.className = 'instance-links';
 
     var usefullLinks = [
-        {
-            title: "Trigger security scan",
-            iconClassName: "fas fa-shield-alt",
-            href: "https://security.crto.in/#/scan/?ip=" + instance.addr
-        },
-        {
-            title: "Availability Graph",
-            iconClassName: "fas fa-chart-area",
-            href: availability_url + serviceName
-        },
     ];
-    if (node_meta_info!= null) {
-      var serial = node_meta_info['serial_number'];
-      if (serial != null) {
-        usefullLinks.push({
-            title: "RackGuru",
-            iconClassName: "fas fa-server",
-            href: rackguru_url + serial
-        });
-      }
-    }
     var first = true;
     for (let usefullLink of usefullLinks) {
         link = document.createElement('a');
@@ -134,19 +101,6 @@ function groups_decorator(instance, key, value, serviceName) {
   return span;
 }
 
-/**
- * Decorates with a slack channel link
- */
-function slack_channel(instance, key, value, serviceName) {
-  var sName = value;
-  if (sName.startsWith('#')) {
-    sName = sName.substring(1);
-  }
-  return build_link(slack_url + encodeURIComponent(sName), '#'+ sName);
-}
-
-const start_regexp = /(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z/
-
 function decorateIsoDate(value, formated) {
   const parsed  = Date.parse(formated);
   if (isNaN(parsed)) {
@@ -163,55 +117,6 @@ function decorateIsoDate(value, formated) {
 // This is the list of function called
 // When a service meta is found
 var registered_decorators = {
-    alert_availability_slack_channel: slack_channel,
-    gerrit_repository: function(instance, key, value, serviceName) {
-      return build_link('https://review.crto.in/#/q/' + value, value);
-    },
-    version: function(instance, key, value, serviceName) {
-      var asInt = parseInt(value);
-      if (asInt < 10000) {
-        return document.createTextNode(value);
-      }
-      if (instance.sMeta['CRITEO_APP_POOL'] != null) {
-          return build_link("https://devtools.crto.in/log.html?moab=cs&im=nb-to&range=100%2C" + asInt, value);
-      } else {
-          var tUrl = "https://devtools.crto.in/log.html?moab=j&im=nb-to&range=100%2C" + asInt;
-          if (instance.sMeta['jvm_artifact'] != null) {
-            tUrl += '&with-dependencies=true&artifacts=' + encodeURIComponent(instance.sMeta['jvm_artifact']);
-          }
-          return build_link(tUrl, value);
-      }
-    },
-    MESOS_TERM_DEBUG_GRANTED_TO: groups_decorator,
-    OWNERS: groups_decorator,
-    marathon_app_id: function(instance, key, value, serviceName) {
-      var app_name = value;
-      if (app_name[0] != '/') {
-        app_name = '/' + app_name;
-      }
-      if (instance.sMeta && instance.sMeta['marathon_ui']) {
-        return build_link(instance.sMeta['marathon_ui'] + '/#/apps/' + encodeURIComponent(app_name), value);
-      } else {
-        // non decorated value
-        return document.createTextNode(value);
-      }
-    },
-    marathon_app_version: function(instance, key, value, serviceName) {
-      return decorateIsoDate(value, value);
-    },
-    slack_channel: slack_channel,
-    start: function(instance, key, value, serviceName) {
-        const reg = start_regexp.exec(value);
-        if (reg != null) {
-          var formated = reg[1] + '-' + reg[2] + '-' + reg[3] + 'T' + reg[4] + ':' + reg[5] + ':' + reg[6] + 'Z';
-          return decorateIsoDate(value, formated);
-        } else {
-          return document.createTextNode(value);
-        }
-    },
-    swagger_key: function(instance, key, value, serviceName) {
-        return build_link(swagger_url + encodeURIComponent(value), value);
-    },
 }
 
 function service_meta_semantics_decorator(instance, key, value, serviceName) {
@@ -286,85 +191,11 @@ function nodeMetaGenerator(nodeMetaTags, serviceMetaIfAny) {
  * it does not have to return anything.
  */
 function navBarDecorator(navbar) {
-  if (typeof consulManager === 'undefined') {
-    // Timepicker is not supported on this page
-    return;
-  }
-  var timepicker_container = document.createElement('div');
-  timepicker_container.innerHTML = `
-              <div class="row">
-                <button type="button" disabled class="btn btn-secondary" data-toggle="tooltip" data-html="true" title="Data returned is the closest available to request" id="currently-displayed-data-date">
-                  Pick a date to see data from the past
-                </button>
-                <div class="col-sm-14">
-                  <div class="form-group">
-                    <div class="input-group date" id="datetimepicker1" data-target-input="nearest">
-                      <input type="text" class="form-control datetimepicker-input" data-target="#datetimepicker1"/>
-                      <div class="input-group-append" data-target="#datetimepicker1" data-toggle="datetimepicker">
-                        <div class="input-group-text"><i class="fa fa-calendar"></i></div>
-                      </div>
-                    </div>
-                  </div>
-                </div>
-              </div>
-              `;
-
-  navbar.appendChild(timepicker_container);
-  var script= document.createElement('script');
-  script.type='text/javascript';
-  // TODO(g.seux): extract code to another file and set "source" on script element
-  script.innerHTML = `
-                    $('#datetimepicker1').datetimepicker({
-                      format: 'DD/MM/YYYY HH:mm:ss Z',  // default format does not allow to select seconds
-                      sideBySide: true, // display date+time on the same widget
-                      useCurrent: true // by default, select current date
-                    });
-                  $("#datetimepicker1").on("change.datetimepicker", function (e) {
-                    console.log("Will fetch data from date " + e.date);
-                    console.log("will clean existing data");
-                    consulManager.clean().then(function(result) {
-                      switch(consulManager.constructor.name) {
-                        case "ConsulServiceManager":
-                          backup_type = 'consul_services';
-                          break;
-                        case "ConsulKeysManager":
-                          backup_type  = 'consul_keys'
-                          break;
-                        case "ConsulNodesManager":
-                          backup_type = 'consul_nodes'
-                          break;
-                        default:
-                          console.log("Unknown " + consulManager.constructor.name + " type");
-                      }
-                      if (e.date) {
-                        console.log("Will replace data by closest snapshot to " + e.date);
-                        var target_url = "https://consul-info-timeline-history.<%= ENV['CRITEO_DC'] %>.<%= ENV['CRITEO_ENV']%>.crto.in/backup/" + backup_type + "/" + e.date / 1000;
-                      } else {
-                        console.log("Will restore to local version");
-                        var target_url = defaultConsulManager.resourceURL;
-                      }
-                      if (typeof(defaultConsulManager) == 'undefined') {
-                        // store first manager to be able to restore it
-                        defaultConsulManager = consulManager
-                      }
-                      consulManager = new consulManager.constructor(target_url);
-                    });
-                  });
-  `;
-  navbar.appendChild(script);
 }
 
-
 /**
  * fetchedResponseDecorator is called with http response when a resource is fetched by any instance of ConsulUIManager
  * it does not have to return anything.
  */
 async function fetchedResponseDecorator(httpResponse) {
-  const data_date = await httpResponse.headers.get('X-Consul-Snapshot-Timestamp');
-  current_date_display = $('#currently-displayed-data-date');
-  if (data_date > 0) {
-    current_date_display.html("Data is a snapshot from: " + moment.unix(data_date).format('DD/MM/YYYY HH:mm:ss Z'));
-  } else {
-    current_date_display.html("Pick a date to see data from the past");
-  }
-}
+}

data/samples/consul-ui/js/nodes.js

@@ -111,7 +111,7 @@ class NodeMainSelector extends MainSelector {
         nodesChecks.appendChild(checksStatusGenerator(node, node['Node']['Name']));
         content.appendChild(nodesChecks);
 
-        content.appendChild(servicesGenerator(node['Service']));
+        content.appendChild(servicesGenerator(node['Service'], node));
         content.appendChild(tagsGenerator(getTagsNode(node)));
 
         sidebar.setAttribute('status', state);

data/samples/consul-ui/js/utils.js

@@ -63,7 +63,7 @@ function nodeState(checks) {
 
 supported_protocols = ['https', 'http', 'sftp', 'ftp', 'ssh', 'telnet']
 
-function serviceTitleGenerator(instance) {
+function serviceTitleGenerator(instance, serviceName) {
     var protocol = null;
     for (i in supported_protocols) {
         var protoc = supported_protocols[i]
@@ -74,10 +74,6 @@ function serviceTitleGenerator(instance) {
     }
 
     var htmlTitle = document.createElement('h5');
-    htmlTitle.setAttribute('title', 'Node Name: ' + instance.name +
-        '\nAddress: ' + instance.addr +
-        '\nService ID: ' + instance.id +
-        '\nService Port: ' + instance.port);
 
     htmlTitle.setAttribute('class', 'instance-name');
     var instanceLink = document.createElement('a');
@@ -91,6 +87,15 @@ function serviceTitleGenerator(instance) {
     }
 
     instanceLink.appendChild(document.createTextNode(instance.name + appendPort));
+    const nodeInfo = document.createElement('a');
+    nodeInfo.appendChild(document.createTextNode('\u24D8'));
+    nodeInfo.setAttribute('title', 'Click to see details of Node: ' + instance.name +
+        '\nAddress: ' + instance.addr +
+        '\nService ID: ' + instance.id +
+        '\nService Port: ' + instance.port);
+    nodeInfo.setAttribute('href', 'consul-nodes-ui.html?node_filter=^' + encodeURIComponent(instance.name) + '$');
+    htmlTitle.appendChild(nodeInfo);
+    htmlTitle.appendChild(document.createTextNode(' '));
     htmlTitle.appendChild(instanceLink);
     htmlTitle.appendChild(document.createTextNode(' '));
     htmlTitle.appendChild(document.createTextNode(instance.addr));
@@ -216,24 +221,44 @@ function toCSSClass(state) {
     return state;
 }
 
-function servicesGenerator(instanceServices) {
-    var services = document.createElement('div');
-    services.className = 'instance-services';
-    services.appendChild(document.createTextNode("Services: "));
-    services.appendChild(document.createElement('br'));
+function servicesGenerator(instanceServices, node) {
+    var servicesTop = document.createElement('div');
+    servicesTop.className = 'instance-services';
+    const card = document.createElement('div');
+    card.setAttribute('class', 'card');
+    const servicesCard = document.createElement('div');
+    servicesCard.setAttribute('class', 'card-body');
+    const title = document.createElement('h5');
+    title.appendChild(document.createTextNode('Services'));
+    title.setAttribute('class', 'card-title')
+    servicesCard.appendChild(title);
+    const services = document.createElement('div');
+    servicesCard.appendChild(services);
     for (var serviceKey in instanceServices) {
-        var service = document.createElement('a');
-        var serviceName = instanceServices[serviceKey]['Service']['Service'];
+        const serviceGrp = document.createElement('span');
+        serviceGrp.setAttribute('class', 'btn btn-sm');
+        serviceGrp.classList.add('btn-outline-' + toCSSClass(nodeState(instanceServices[serviceKey]['Checks'])))
+        const service = document.createElement('a');
+        serviceGrp.appendChild(service);
+        const serviceName = instanceServices[serviceKey]['Service']['Service'];
+        service.setAttribute('class', 'serviceLink');
+        service.setAttribute('href', 'consul-services-ui.html?service=' + encodeURIComponent(serviceName) + '&node_filter=^' + encodeURIComponent(node['Node']['Name'])+'$');
+        service.appendChild(document.createTextNode(serviceName));
         var servicePort = instanceServices[serviceKey]['Service']['Port'];
-        service.setAttribute('class', 'btn btn-sm m-1 lookup');
-        service.setAttribute('target', '_blank');
-        nodeAddr = instanceServices[serviceKey]['Service']['Address'];
-        service.setAttribute('href', 'http://' + nodeAddr + ':' + servicePort);
-        service.classList.add('btn-outline-' + toCSSClass(nodeState(instanceServices[serviceKey]['Checks'])))
-        service.appendChild(document.createTextNode(serviceName + ':' + servicePort));
-        services.appendChild(service);
+        if (servicePort) {
+            // Add unbreakable space
+            serviceGrp.appendChild(document.createTextNode('\u00A0'));
+            const servicePortElem = document.createElement('a');
+            servicePortElem.setAttribute('class', 'serviceTargetPort');
+            const nodeAddr = instanceServices[serviceKey]['Service']['Address'];
+            servicePortElem.setAttribute('href', 'http://' + nodeAddr + ':' + servicePort);
+            servicePortElem.appendChild(document.createTextNode(':' + servicePort));
+            serviceGrp.appendChild(servicePortElem);
+        }
+        services.appendChild(serviceGrp);
     }
-    return services;
+    servicesTop.appendChild(servicesCard);
+    return servicesTop;
 }
 
 function checksStatusGenerator(instance, prefix) {

data/samples/display_timestamped_changes.txt.erb

@@ -0,0 +1,17 @@
+<%
+  # This example show how to display local time informaition about changes
+  # Example of usage to display logs of changes on nodes() endpoint:
+  #
+  #   consul-templaterb --template "display_timestamped_changes.txt.erb:display_timestamped_changes.txt:cat display_timestamped_changes.txt" -l error
+  #
+  # Would output:
+  #   Last update: 1588800554 (2020-05-06 21:29:14 UTC), X-Consul-Index: 4345827328
+  #   Last update: 1588800569 (2020-05-06 21:29:29 UTC), X-Consul-Index: 4345829548
+  #   Last update: 1588800676 (2020-05-06 21:31:16 UTC), X-Consul-Index: 4345836342
+  #
+  @my_last_time = Time.now.utc unless @my_last_time
+  val = nodes()
+  new_idx = val.endpoint.x_consul_index
+  @my_last_time = Time.now.utc if @my_last_idx != new_idx
+  @my_last_idx = new_idx
+%>Last update: <%= @my_last_time.to_i %> (<%= @my_last_time %>), X-Consul-Index: <%= new_idx %>

data/samples/find_nodes_in_catalog_but_not_in_members.json.erb

@@ -0,0 +1,28 @@
+<%= 
+   all_catalog = nodes.map{ |n| [n['Node'], n] }.to_h
+   all_members = agent_members.sort{ |a,b| a['Name'] <=> b['Name'] }
+   members_by_state = {}
+   all_members.each do |m|
+     state = m.status
+     members = members_by_state[state] || 0
+     members_by_state[state] = members + 1
+   end
+   # Find all virtual nodes (eg: k8s-sync)
+   catalog_external_sources = nodes.select{|n| (n['Meta']||{})['external-source'] }.map{|n| n['Node']}
+   in_catalog_but_not_in_members = all_catalog.keys - all_members.map{ |m| m['Name'] }
+   not_virtual_not_in_members = in_catalog_but_not_in_members - catalog_external_sources
+   JSON.pretty_generate(
+    {
+      'stats' => {
+        'catalog' => all_catalog.count,
+        'catalog_virtual' => in_catalog_but_not_in_members.count,
+        'members' => all_members.count,
+        'members_by_state': members_by_state,
+        'catalog_external_sources' => catalog_external_sources,
+      },
+      'errors': {
+        'in_catalog_but_not_in_members' => in_catalog_but_not_in_members,
+        'not_virtual_not_in_members'    =>  not_virtual_not_in_members,
+      }
+   })
+%>

data/samples/metrics.erb

@@ -21,6 +21,20 @@
   all_stats = {}
   service_count = 0
 
+  statuses = agent_members().map {|m| m.status}.reduce({}) do |sum, s|
+    v = sum[s] || 0
+    sum[s] = v + 1
+    sum
+  end
+%>
+# HELP consul_members_count A gauge of number of serf members with their count and their status('alive', 'leaving', 'left', 'failed')
+# TYPE consul_members_count gauge
+<%
+  statuses.each do |k, v|
+%>
+consul_members_count{serf="lan",status="<%= k %>"} <%= v %><%
+  end
+
   services(tag: service_tag_filter).each do |service_name, tags|
     if !services_blacklist.any? {|r| r.match(service_name)} && (instance_must_tag.nil? || tags.include?(instance_must_tag))
       service_count += 1

data/samples/prometheus_datacenter_coordinates.erb

@@ -0,0 +1,56 @@
+# HELP consul_wan_servers_rtt_seconds Min Round trip with other servers of DC
+# TYPE consul_wan_servers_rtt_seconds gauge
+<%
+# This computes the RTT over WAN for all DCs
+results = {'min' => {}, 'max' => {}, 'p50' => {}, 'p90' => {}}
+my_dcname = (agent_self['Config'] || {})['Datacenter']
+my_dc = coordinate.datacenters.select { |d| d['Datacenter'] == my_dcname }.first
+
+def percentile(values_sorted, percentile)
+  k = (percentile*(values_sorted.length-1)+1).floor - 1
+  f = (percentile*(values_sorted.length-1)+1).modulo(1)
+  return values_sorted[k] + (f * (values_sorted[k+1] - values_sorted[k]))
+end
+
+if my_dc
+  from = my_dc['Datacenter']
+  coordinate.datacenters.each do |dc_coords|  
+    min = Float::MAX
+    max = 0
+    rtts = []
+    dc_coords['Coordinates'].each do |s2|
+      my_dc['Coordinates'].each do |s1|
+        rtt = coordinate.rtt(s1['Coord'], s2['Coord'])
+        min = rtt if rtt < min
+        max = rtt if rtt > max
+        idx = rtts.bsearch_index { |x| x > rtt }
+        if idx.nil?
+          rtts << rtt
+        else
+          rtts = rtts.insert(idx, rtt)
+        end
+%>
+consul_wan_servers_rtt_seconds{from="<%= from %>",to="<%= dc_coords['Datacenter'] %>",from_node="<%= s1['Node'] %>",to_node="<%= s2['Node'] %>"} <%= rtt.round(3) %><%
+      end
+    end
+    results['min'][dc_coords['Datacenter']] = min
+    results['max'][dc_coords['Datacenter']] = max
+    results['p50'][dc_coords['Datacenter']] = percentile(rtts, 0.5)
+    results['p90'][dc_coords['Datacenter']] = percentile(rtts, 0.9)
+  end
+
+  # Now, we iterate over aggregated results, type is min, max...
+  results.each do |type, res_type|
+  %>
+
+# HELP consul_wan_dc_<%= type %>_seconds <%= type %> round trip with other DCs
+# TYPE consul_wan_dc_<%= type %>_seconds gauge
+<%
+    # Iterate over destinations
+    res_type.each do |dst, val|
+%>consul_wan_dc_<%= type %>_seconds{from="<%= my_dcname %>",to="<%= dst %>"} <%= val.round(3) %>
+<%
+    end
+  end
+end
+%>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.26.1
+  version: 1.27.2
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
+date: 2020-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -198,6 +198,7 @@ files:
 - samples/all_services.txt.erb
 - samples/all_services_multi_agents.txt.erb
 - samples/all_templates.erb
+- samples/checks_in_warning_or_critical_state.yaml.erb
 - samples/consul-ui/README.md
 - samples/consul-ui/common/footer.html.erb
 - samples/consul-ui/common/header.html.erb
@@ -229,7 +230,9 @@ files:
 - samples/criteo/haproxy.cfg.erb
 - samples/debug/compare_connect_services.txt.erb
 - samples/demos/compute_pricing.txt.erb
+- samples/display_timestamped_changes.txt.erb
 - samples/find_all_invalid_dns_labels.json.erb
+- samples/find_nodes_in_catalog_but_not_in_members.json.erb
 - samples/ha_proxy.cfg.erb
 - samples/haproxy_dns.cfg.erb
 - samples/hosts.erb
@@ -239,6 +242,7 @@ files:
 - samples/members.json.erb
 - samples/metrics.erb
 - samples/prometheus_consul_coordinates.erb
+- samples/prometheus_datacenter_coordinates.erb
 - samples/render_template_from_kv.erb
 - samples/sample_keys.html.erb
 - samples/service_checks_metrics.erb
@@ -269,8 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Implementation of Consul template using Ruby and .erb templating language