consul-templaterb 1.26.3 → 1.28.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eca26ee1958aac8a862d3c1c96b35446ac7a7775f3209095f64d8cd01200fee9
-  data.tar.gz: 15292c1651add5460162cf629814c37aeb9c965daaae0a5bfcb9a083e0c40758
+  metadata.gz: ca62994383929bd3874db5527f85efeaaf184fdb83b8f3b17e05fbadbfbbc90a
+  data.tar.gz: e878ff880557ffdfd8db6e5cb900a19e8191d4db11ba28fa0b4e7bbe9b1b9240
 SHA512:
-  metadata.gz: a7e04120d50851589cb0e90e45327768b81c2c9954634ca7b1d80d51b88de5e6186402c4f3be0bbc7a67ef220c8950c4117730bf50abaf2c080c5cd3e007d18b
-  data.tar.gz: 9b434745e5e8fb44c955350d38c68a469dc08b249b331800c38e112952757f85713e7ee2e924a3bd62883115443467329774177b98a0193badce3f2dfbb9a4d1
+  metadata.gz: 62afc852ef61329a30bba3a67f43ed372130d9a87cd1afce559b5f88f9aa0a6db336c84c63b173eaa96896e8c9088686cfa37f10d540cd91c80ca2fb976b00b3
+  data.tar.gz: 4a02aa15f2d4f95e5d144e916bf6a51190a90756e134456aece5b4afbf7ba106e150faeb8cf7b0e09ff27f1f61b237c47dd11f236299d922a519e91b019e3705

data/CHANGELOG.md

@@ -2,19 +2,57 @@
 
 ## (UNRELEASED)
 
+## 1.28.1 (Sept 30,, 2020)
+
+IMPROVEMENTS:
+
+ * Reduced size of GEM from 400k o 100k by removing not needed files
+
+## 1.28.0 (Sept 25, 2020)
+
+NEW FEATURES:
+
+ * Added `-W` or `--wait-between-reload-signal` to avoid sending too many signals
+   to a process executed. This avoids for instance reloading too much a HAProxy configuration
+   without having to play with `-w` as described in [#69](https://github.com/criteo/consul-templaterb/issues/69)
+
+BUG FIXES:
+
+ * Removed warnings at runtime with Ruby 2.7+
+ * Minor JS fix in Consul-UI (Added missing unused parameter to function `serviceTitleGenerator`)
+
+## 1.27.2 (Sept 4, 2020)
+
+IMPROVEMENTS:
+
+ * Consul-UI now supports navigation between nodes and services in both ways
+
+## 1.27.1 (July 28, 2020)
+
+BUGIX:
+
+ * Fix collision in JSON queries when using payload in requests [#68](https://github.com/criteo/consul-templaterb/pull/68)
+
+## 1.27.0 (June 5, 2020)
+
 NEW FEATURES:
 
+ * For Consul 1.7+, now support `checks_in_state(check_state, dc: nil, [agent: consul_agent_address])`,
+   fixes feature [#65](https://github.com/criteo/consul-templaterb/issues/65)
+ * New options to support/disable TLS validation thanks to [@jeromegn](https://github.com/jeromegn)
+   [#66](https://github.com/criteo/consul-templaterb/pull/66)
+
 ## 1.26.3 (April 15, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Removed all Criteo specific decorators from Consul-UI
 
 ## 1.26.2 (April 15, 2020)
 
-BUGFIX:
+BUG FIXES:
 
- * Fixed broken Dockerfile (was missing the new `decorator.js.erb` file). Fixes #61 (Thanks to @ simongareste)
+ * Fixed broken Dockerfile (was missing the new `decorator.js.erb` file). Fixes #61 (Thanks to [@simongareste](https://github.com/simongareste))
 
 NEW FEATURES:
 
@@ -22,7 +60,7 @@ NEW FEATURES:
 
 ## 1.26.1 (March 27, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Using `agent: http://vault_or_consul_agent:port>` was not properly taken into account in some endpoints
 
@@ -40,7 +78,7 @@ NEW FEATURES:
 
 ## 1.25.2 (February 29, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Update rake to 12.3.3 to fix [CVE-2020-8130](https://github.com/advisories/GHSA-jppv-gw3r-w3q8)
 
@@ -59,7 +97,7 @@ NEW FEATURES:
 
 ## 1.24.1 (February 19, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Properly set service meta in node meta decorator
 
@@ -76,7 +114,7 @@ NEW FEATURES:
  * Implementation of #59 - implementation of `--retry` and `--vault-retry` new flags
    Those flags work in a similar way as in consul-template: stop program after X failures
    of consul or vault endpoints
- * Added --fail-fast that stop the programs immediately if vault or consul are not available
+ * Added `--fail-fast` that stop the programs immediately if vault or consul are not available
    at startup (also works with `--once`)
 
 ## 1.22.0 (January 17, 2020)
@@ -87,7 +125,7 @@ NEW FEATURES:
 
 ## 1.21.8 (January 2, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Escape properly metadata containing double quotes in prometheus exporter
 
@@ -101,13 +139,13 @@ IMPROVEMENTS:
 
 IMPROVEMENTS:
 
- * Added node_meta_info for serviceInstanceDecorator and serviceMetaDecorator
+ * Added `node_meta_info` for `serviceInstanceDecorator` and `serviceMetaDecorator` in Consul-UI
 
 ## 1.21.5 (December 6, 2019)
 
 NEW FEATURES:
 
- * Added clean() method in nodes.js to allow nexw behaviors
+ * Added `clean()` method in nodes.js to allow new behaviors
 
 ## 1.21.4 (November 28, 2019)
 
@@ -138,7 +176,7 @@ IMPROVEMENTS:
 ## 1.21.0 (November 21, 2019)
 
 * added function `templates` to list all templates being rendered
-* added support for JS decorators in consul-ui (thanks to @Thib17)
+* added support for JS decorators in consul-ui (thanks to [@Thib17](https://github.com/Thib17))
 
 ## 1.20.0 (October 16, 2019)
 
@@ -157,7 +195,7 @@ NEW FEATURES:
 
 * Added new function `checks_for_node`
 
-BUGFIXs:
+BUG FIXES:
 
 * Avoid try publishing several times Gem on rubygems.org
 
@@ -179,13 +217,13 @@ IMPROVEMENTS:
 
 ## 1.18.3 (September 2, 2019)
 
-BUGFIX:
+BUG FIXES:
 
 * When vault receives at timeout, correctly reschedule it
 
 ## 1.18.2 (August 28,  2019)
 
-BUGFIX:
+BUG FIXES:
 
 * In Consul UI, showing data from KV with markup was not properly handled
 
@@ -201,7 +239,7 @@ Support any request method for remote_resource.as_json (#41)
 
 ## 1.18.1 (July 27, 2019)
 
-BUGFIX:
+BUG FIXES:
 
 Fixed wrong lazy initialization in `remote_resource.as_json` that
 cause too many connections to be opened.
@@ -210,13 +248,13 @@ cause too many connections to be opened.
 
 NEW FEATURES:
 
- * Support for `remote_resource` provided by @kamaradclimber
+ * Support for `remote_resource` provided by [@kamaradclimber](https://github.com/kamaradclimber)
  * Added support for `remote_resource.as_json` to fetch JSON remote resource from a web server
  * Added `samples/list_ruby_versions_from_rubygems.txt.erb` to demonstrate usage
 
 ## 1.17.3 (July 18, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Added gem parallel as a dependency to allow `samples/prometheus_consul_coordinates.erb`
    to work properly
@@ -341,7 +379,7 @@ NEW FEATURES:
 
 ## 1.10.1 (February 28, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Ensure that timeline sort properly events when healthchecks are removed (eg: maintenance)
 
@@ -364,7 +402,7 @@ IMPROVEMENTS:
 
 ## 1.9.8 (January 16, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * When default value was the same as real value, endpoints were always marked as
    dirty, thus rendering of templates did never succeed.
@@ -380,7 +418,7 @@ IMPROVEMENTS:
 
 ## 1.9.6 (January 15, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Keep connections open properly as it increase timeouts on Consul servers on
    very large templates
@@ -391,7 +429,7 @@ IMPROVEMENTS:
 
 ## 1.9.5 (January 14, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Ensure to always re-open Connection to Consul agent in case of network error
 
@@ -441,11 +479,11 @@ IMPROVEMENTS:
 
 OPTIMIZATIONS:
 
- * Better network issue because of X-Consul-Index parsing bug
+ * Better network usage because of X-Consul-Index parsing bug
 
-NEW FEATURES:
+BUG FIXES:
 
- * value.endpoint.x_consul_index now works as expected
+ * `value.endpoint.x_consul_index` now works as expected
 
 IMPROVEMENTS:
 
@@ -485,7 +523,7 @@ NEW FEATURES:
 
 ## 1.8.1 (December 12, 2018)
 
-BUGFIX:
+BUG FIXES:
 
  * Properly fill `template_info` strtucture when hot reload is performed so templates using
    `template_info()` new function can behave nicely.
@@ -594,8 +632,8 @@ IMPROVEMENTS:
  * [Prometheus template](samples/metrics.erb) to export easily Consul
    informations about nodes, datacenters and all services states
  * Code style cleanup + travis now enforces Rubocop
- * Remove criteo references in spec files thanks to @pierrecdn
- * Bitrate display more consistent thanks to @pierrecdn
+ * Remove criteo references in spec files thanks to [@pierrecdn](https://github.com/pierrecdn)
+ * Bitrate display more consistent thanks to [@pierrecdn](https://github.com/pierrecdn)
 
 ## 1.5.3 (September 24, 2018)
 

data/README.md

@@ -165,9 +165,19 @@ USAGE: consul-templaterb [[options]]
     -f, --[no-]fail-fast             If consul/vault endpoints fail at startup, fail immediately
     -g, --no-gzip-compression        Disable GZIP compression in HTTP requests
     -c, --consul-addr=<address>      Address of Consul, eg: http://localhost:8500
+        --consul-cert-chain=<path/to/cert_chain>
+                                     Path to Consul TLS client certificate chain to use
+        --consul-private-key=<path/to/private_key>
+                                     Path to Consul TLS client private key to use
+        --skip-consul-verify-tls     Skip verifying Consul TLS via certificate authority (DANGEROUS)
     -l, --log-level=<log_level>      Log level, default=info, any of none|error|info|debug
         --consul-token=<token>       Use a token to connect to Consul
     -V, --vault-addr=<address>       Address of Vault, eg: http://localhost:8200
+        --vault-cert-chain=<path/to/cert_chain>
+                                     Path to Vault TLS client certificate chain to use
+        --vault-private-key=<path/to/private_key>
+                                     Path to Vault TLS client private key to use
+        --skip-vault-verify-tls      Skip verifying Vault TLS via certificate authority (DANGEROUS)
         --vault-token=<token>        Token used to authenticate against vault.
         --[no-]vault-renew           Control auto-renewal of the Vault token. Default: activated
         --vault-retry, --vault-retry-attempts [RETRIES]
@@ -178,10 +188,11 @@ USAGE: consul-templaterb [[options]]
     -r, --retry-delay=<min_duration> Min Retry delay on Error/Missing Consul Index
     -k, --hot-reload=<behavior>      Control hot reload behaviour, one of :[die (kill daemon on hot reload failure), keep (on error, keep running), disable (hot reload disabled)]
     -K, --sig-term=kill_signal       Signal to send to next --exec command on kill, default=TERM
+    -M, --debug-memory-usage         Display messages when RAM grows
     -T, --trim-mode=trim_mode        ERB Trim mode to use (- by default)
     -R, --sig-reload=reload_signal   Signal to send to next --exec command on reload (NONE supported), default=HUP
-    -M, --debug-memory-usage         Display messages when RAM grows
-    -e, --exec=<command>             Execute the following command
+    -W, --wait-signal=min_duration   Wait at least n seconds before each reload signal being sent to next --exec process
+    -e, --exec=<command>             Execute the following command in as a subprocess when all templates are ready
     -d, --debug-network-usage        Debug the network usage
     -t erb_file:[output]:[command]:[params_file],
         --template                   Add a erb template, its output and optional reload command
@@ -235,7 +246,11 @@ nor write the file.
 Signals can be customized per process. Two signals are supported with options `--sig-reload` and
 `--sig-term`. When the option is added, the next `--exec` options to start a process will use the
 given signal. By default, HUP will be sent to reload events (you can use NONE to avoid sending any
-reload signal), TERM will be used when leaving consul-templaterb.
+reload signal), TERM will be used when leaving consul-templaterb. A minimum duration between reload
+signals can be specified for each sub process by prepending `--wait-signal=min_duration` to `--exec`
+command.
+In such case, the signal will be sent every `min_duration` as a maximum (very useful for templates
+changing a lot, but you don't want to trigger too many reloads, for instance for a load-balancer).
 
 ### Bandwidth limitation
 
@@ -332,7 +347,7 @@ Please consult [CHANGELOG.md](CHANGELOG.md) for fixed bugs.
 
 ## TODO
 
-* [x] Hashi's Vault support (EXPERIMENTAL)
+* [x] Hashi's Vault support
 * [ ] Implement automatic dynamic rate limit
 * [x] More samples: apache, nginx, a full website displaying consul information...
 * [x] Optimize rendering speed at start-up: an iteration is done every second by default, but it would be possible to speed

data/TemplateAPI.md

@@ -388,6 +388,15 @@ name or its ID. If DC is specified, will lookup for given node in another datace
 
 [Find all the checks](https://www.consul.io/api/health.html#list-checks-for-service) of a given service.
 
+## checks_in_state(check_state, dc: nil, [agent: consul_agent_address])
+
+[Find all the checks in a given state](https://www.consul.io/api-docs/health#list-checks-in-state) in the whole cluster.
+
+The filter check_state must be one of any|critical|warning|passing.
+
+Warning: this endpoint might be very frequently updated in a
+large cluster if you are using `any` value. This endpoint is supported with Consul 1.7+.
+
 ## kv(name, [dc: nil], [keys: false], [recurse: false], [agent: consul_agent_address])
 
 [Read keys from KV Store](https://www.consul.io/api/kv.html#read-key). It can be used for both listing the keys and

data/bin/consul-templaterb

@@ -28,6 +28,9 @@ options = {
     },
     base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
     token: ENV['VAULT_TOKEN'] || nil,
+    tls_cert_chain: ENV['VAULT_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['VAULT_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     token_renew: true,
@@ -48,6 +51,9 @@ options = {
     },
     base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://localhost:8500',
     token: ENV['CONSUL_HTTP_TOKEN'] || nil,
+    tls_cert_chain: ENV['CONSUL_CLIENT_CERT'] || nil,
+    tls_private_key: ENV['CONSUL_CLIENT_KEY'] || nil,
+    tls_verify_peer: true,
     max_consecutive_errors_on_endpoint: 10, # Stop program after n consecutive failures on same endpoint
     fail_fast_errors: nil,                  # fail fast the program if endpoint was never success
     retry_duration: 10,      # On error, retry after n seconds
@@ -88,6 +94,7 @@ consul_engine = Consul::Async::ConsulTemplateEngine.new
 @programs = {}
 cur_sig_reload = 'HUP'.freeze
 cur_sig_term = 'TERM'.freeze
+cur_min_duration_between_signals = 1
 
 optparse = OptionParser.new do |opts|
   opts.banner = usage_text
@@ -122,6 +129,18 @@ optparse = OptionParser.new do |opts|
     options[:consul][:base_url] = consul_url
   end
 
+  opts.on('--consul-cert-chain=<path/to/cert_chain>', String, 'Path to Consul TLS client certificate chain to use') do |consul_client_cert|
+    options[:consul][:tls_cert_chain] = consul_client_cert
+  end
+
+  opts.on('--consul-private-key=<path/to/private_key>', String, 'Path to Consul TLS client private key to use') do |consul_client_key|
+    options[:consul][:tls_private_key] = consul_client_key
+  end
+
+  opts.on('--skip-consul-verify-tls', 'Skip verifying Consul TLS via certificate authority (DANGEROUS)') do
+    options[:consul][:tls_verify_peer] = false
+  end
+
   opts.on('-l', '--log-level=<log_level>', String, "Log level, default=info, any of #{::Consul::Async::Debug.levels.join('|')}") do |log_level|
     ::Consul::Async::Debug.level = log_level
   end
@@ -134,6 +153,18 @@ optparse = OptionParser.new do |opts|
     options[:vault][:base_url] = vault_url
   end
 
+  opts.on('--vault-cert-chain=<path/to/cert_chain>', String, 'Path to Vault TLS client certificate chain to use') do |vault_client_cert|
+    options[:vault][:tls_cert_chain] = vault_client_cert
+  end
+
+  opts.on('--vault-private-key=<path/to/private_key>', String, 'Path to Vault TLS client private key to use') do |vault_client_key|
+    options[:vault][:tls_private_key] = vault_client_key
+  end
+
+  opts.on('--skip-vault-verify-tls', 'Skip verifying Vault TLS via certificate authority (DANGEROUS)') do
+    options[:vault][:tls_verify_peer] = false
+  end
+
   opts.on('-T', '--vault-token=<token>', String, 'Token used to authenticate against vault.') do |vault_token|
     options[:vault][:token] = vault_token
   end
@@ -185,6 +216,10 @@ optparse = OptionParser.new do |opts|
     cur_sig_term = compute_signal(sig, nil)
   end
 
+  opts.on('-M', '--debug-memory-usage', 'Display messages when RAM grows') do
+    consul_engine.debug_memory = true
+  end
+
   opts.on('-T', '--trim-mode=trim_mode', String,
           "ERB Trim mode to use (#{options[:erb][:trim_mode]} by default)") do |trim_mode|
     options[:erb][:trim_mode] = trim_mode
@@ -195,25 +230,36 @@ optparse = OptionParser.new do |opts|
     cur_sig_reload = compute_signal(sig, 'NONE')
   end
 
-  opts.on('-M', '--debug-memory-usage', 'Display messages when RAM grows') do
-    consul_engine.debug_memory = true
+  opts.on('-W', '--wait-signal=min_duration', Float, 'Wait at least n seconds before each reload signal being sent to next --exec process') do |min_duration|
+    raise "-wait-between-reload-signal=#{min_duration} must be greater than 0" unless min_duration.positive?
+
+    cur_min_duration_between_signals = min_duration
   end
 
-  opts.on('-e', '--exec=<command>', String, 'Execute the following command') do |cmd|
+  opts.on('-e', '--exec=<command>', String, 'Execute the following command in as a subprocess when all templates are ready') do |cmd|
     sig_reload = cur_sig_reload
     sig_term = cur_sig_term
+    sig_min_interval = cur_min_duration_between_signals
     consul_engine.add_template_callback do |all_ready, template_manager, results|
       if all_ready
         modified = results.any?(&:modified)
         if @programs[cmd].nil?
-          warn "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload || 'NONE'} on_term=#{sig_term}"
+          warn "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload || 'NONE'} on_term=#{sig_term}, delay between reloads=#{sig_min_interval}s"
           @programs[cmd] = Consul::Async::ProcessHandler.new(cmd, sig_reload: sig_reload, sig_term: sig_term)
           @programs[cmd].start
         else
+
           # At least one template has been modified
-          @programs[cmd].reload if modified
+          process_to_reload = @programs[cmd]
+          if modified && !process_to_reload.reload_scheduled
+            process_to_reload.reload_scheduled = true
+            now = Time.now
+            delay = sig_min_interval - (now - @programs[cmd].last_signal_sent)
+            delay = 0 if delay.negative?
+            EventMachine.add_timer(delay) { process_to_reload.reload }
+          end
           begin
-            @programs[cmd].process_status
+            process_to_reload.process_status
           rescue Consul::Async::ProcessDoesNotExist => e
             warn "[FATAL] The process is dead, aborting run: #{e.inspect}"
             template_manager.terminate

data/lib/consul/async/consul_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class ConsulConfiguration
       attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
                   :missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug, :enable_gzip_compression,
-                  :fail_fast_errors, :max_consecutive_errors_on_endpoint
+                  :fail_fast_errors, :max_consecutive_errors_on_endpoint, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(base_url: 'http://localhost:8500',
                      debug: { network: false },
                      token: nil,
@@ -23,7 +23,10 @@ module Consul
                      enable_gzip_compression: true,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: 1)
+                     fail_fast_errors: 1,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token = token
         @debug = debug
@@ -38,6 +41,9 @@ module Consul
         @paths = paths
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -71,7 +77,10 @@ module Consul
                                 enable_gzip_compression: enable_gzip_compression,
                                 paths: @paths,
                                 max_consecutive_errors_on_endpoint: @max_consecutive_errors_on_endpoint,
-                                fail_fast_errors: @fail_fast_errors)
+                                fail_fast_errors: @fail_fast_errors,
+                                tls_cert_chain: ch(path, :tls_cert_chain),
+                                tls_private_key: ch(path, :tls_private_key),
+                                tls_verify_peer: ch(path, :tls_verify_peer))
       end
     end
 
@@ -233,6 +242,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: conf.wait_duration + 1 + (conf.wait_duration / 16) # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.base_url, options)
         }