consul-templaterb 1.28.0 → 1.31.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c15ee78520048d7d091882b9d3b1d43b4cc5f5658cf39d782f06e1dc617d8cd9
-  data.tar.gz: 6f77a9daab557420f2640a314a97f78af68e26fec91c1a66f3ee575d9bf150f8
+  metadata.gz: a0929b48d32c3a556bff69053efcb0f239ee7cc18b39907c71ae1309bebeabbb
+  data.tar.gz: f6a8130e754621e9603a11cbbbc0963c67ef043b5578c35944ed6a26f9e59d3e
 SHA512:
-  metadata.gz: 8c2b374a9400061f3ffd673e418771bb5685ec3d13a631c3839660121d86575be1fbbd1cd99f7ee1a4ddedf850ec72fa984ca084575da587b7a239d2e4654b2e
-  data.tar.gz: '0933c0a2f1f58c617c184679ab68c1e90d55f1d9ab796e45336ddbbf49ae2fd73cf1aafbfea6e921254a520c8be3e5f0f380cfbdacd694ba2de6c5f8e4f5a074'
+  metadata.gz: 643d665507cd9a633dffdc741f4d643a55a0c4acaefc7feb08396f95383f491dd8d1917f3264b49a04cc67d258efa8dee58a7bc50d7fe74b440fbad74877c36b
+  data.tar.gz: dfe3a05334f0d65216b5973ea9f5716c983ce6d428751243991d8708781d9b95ff19404fba886f83df711563b2c46a3850a06f391b7a26c2ec82a7987df046fa

data/CHANGELOG.md

@@ -1,6 +1,39 @@
 # CHANGELOG
 
-## (UNRELEASED)
+## 1.31.0 (Jan 18, 2021)
+
+IMPROVEMENTS:
+
+* Do not apply penalty on unmodified services not changing much (Will improve streaming performance)
+
+## 1.30.0 (Dec 18, 2020)
+
+IMPROVEMENTS:
+
+ * Allow customization to sort nodes in service view in Consul-UI ([#71](https://github.com/criteo/consul-templaterb/pull/71))
+
+NEW FEATURES:
+
+ * Auto-detection of rate-limit on Consul Side (introduced in Consul 1.6.2+), this feature
+   will work only Consul 1.9+ (see [Consul #7527](https://github.com/hashicorp/consul/issues/7527)). 
+
+## 1.29.0 (Oct 6, 2020)
+
+IMPROVEMENTS:
+
+ * When no template has been specified, stop early with an error message
+
+## 1.28.2 (Oct 5, 2020)
+
+NEW FEATURES:
+
+ * Added new decorator to compute node names in consul-ui ([#70](https://github.com/criteo/consul-templaterb/pull/70))
+
+## 1.28.1 (Sept 30, 2020)
+
+IMPROVEMENTS:
+
+ * Reduced size of GEM from 400k o 100k by removing not needed files
 
 ## 1.28.0 (Sept 25, 2020)
 
@@ -10,7 +43,7 @@ NEW FEATURES:
    to a process executed. This avoids for instance reloading too much a HAProxy configuration
    without having to play with `-w` as described in [#69](https://github.com/criteo/consul-templaterb/issues/69)
 
-BUGFIX:
+BUG FIXES:
 
  * Removed warnings at runtime with Ruby 2.7+
  * Minor JS fix in Consul-UI (Added missing unused parameter to function `serviceTitleGenerator`)
@@ -25,7 +58,7 @@ IMPROVEMENTS:
 
 BUGIX:
 
- * Fix collision in JSON queries when using payload in requests #68
+ * Fix collision in JSON queries when using payload in requests [#68](https://github.com/criteo/consul-templaterb/pull/68)
 
 ## 1.27.0 (June 5, 2020)
 
@@ -38,15 +71,15 @@ NEW FEATURES:
 
 ## 1.26.3 (April 15, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Removed all Criteo specific decorators from Consul-UI
 
 ## 1.26.2 (April 15, 2020)
 
-BUGFIX:
+BUG FIXES:
 
- * Fixed broken Dockerfile (was missing the new `decorator.js.erb` file). Fixes #61 (Thanks to @ simongareste)
+ * Fixed broken Dockerfile (was missing the new `decorator.js.erb` file). Fixes #61 (Thanks to [@simongareste](https://github.com/simongareste))
 
 NEW FEATURES:
 
@@ -54,7 +87,7 @@ NEW FEATURES:
 
 ## 1.26.1 (March 27, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Using `agent: http://vault_or_consul_agent:port>` was not properly taken into account in some endpoints
 
@@ -72,7 +105,7 @@ NEW FEATURES:
 
 ## 1.25.2 (February 29, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Update rake to 12.3.3 to fix [CVE-2020-8130](https://github.com/advisories/GHSA-jppv-gw3r-w3q8)
 
@@ -91,7 +124,7 @@ NEW FEATURES:
 
 ## 1.24.1 (February 19, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Properly set service meta in node meta decorator
 
@@ -108,7 +141,7 @@ NEW FEATURES:
  * Implementation of #59 - implementation of `--retry` and `--vault-retry` new flags
    Those flags work in a similar way as in consul-template: stop program after X failures
    of consul or vault endpoints
- * Added --fail-fast that stop the programs immediately if vault or consul are not available
+ * Added `--fail-fast` that stop the programs immediately if vault or consul are not available
    at startup (also works with `--once`)
 
 ## 1.22.0 (January 17, 2020)
@@ -119,7 +152,7 @@ NEW FEATURES:
 
 ## 1.21.8 (January 2, 2020)
 
-BUGFIX:
+BUG FIXES:
 
  * Escape properly metadata containing double quotes in prometheus exporter
 
@@ -133,13 +166,13 @@ IMPROVEMENTS:
 
 IMPROVEMENTS:
 
- * Added node_meta_info for serviceInstanceDecorator and serviceMetaDecorator
+ * Added `node_meta_info` for `serviceInstanceDecorator` and `serviceMetaDecorator` in Consul-UI
 
 ## 1.21.5 (December 6, 2019)
 
 NEW FEATURES:
 
- * Added clean() method in nodes.js to allow nexw behaviors
+ * Added `clean()` method in nodes.js to allow new behaviors
 
 ## 1.21.4 (November 28, 2019)
 
@@ -170,7 +203,7 @@ IMPROVEMENTS:
 ## 1.21.0 (November 21, 2019)
 
 * added function `templates` to list all templates being rendered
-* added support for JS decorators in consul-ui (thanks to @Thib17)
+* added support for JS decorators in consul-ui (thanks to [@Thib17](https://github.com/Thib17))
 
 ## 1.20.0 (October 16, 2019)
 
@@ -189,7 +222,7 @@ NEW FEATURES:
 
 * Added new function `checks_for_node`
 
-BUGFIXs:
+BUG FIXES:
 
 * Avoid try publishing several times Gem on rubygems.org
 
@@ -211,13 +244,13 @@ IMPROVEMENTS:
 
 ## 1.18.3 (September 2, 2019)
 
-BUGFIX:
+BUG FIXES:
 
 * When vault receives at timeout, correctly reschedule it
 
 ## 1.18.2 (August 28,  2019)
 
-BUGFIX:
+BUG FIXES:
 
 * In Consul UI, showing data from KV with markup was not properly handled
 
@@ -233,7 +266,7 @@ Support any request method for remote_resource.as_json (#41)
 
 ## 1.18.1 (July 27, 2019)
 
-BUGFIX:
+BUG FIXES:
 
 Fixed wrong lazy initialization in `remote_resource.as_json` that
 cause too many connections to be opened.
@@ -242,13 +275,13 @@ cause too many connections to be opened.
 
 NEW FEATURES:
 
- * Support for `remote_resource` provided by @kamaradclimber
+ * Support for `remote_resource` provided by [@kamaradclimber](https://github.com/kamaradclimber)
  * Added support for `remote_resource.as_json` to fetch JSON remote resource from a web server
  * Added `samples/list_ruby_versions_from_rubygems.txt.erb` to demonstrate usage
 
 ## 1.17.3 (July 18, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Added gem parallel as a dependency to allow `samples/prometheus_consul_coordinates.erb`
    to work properly
@@ -373,7 +406,7 @@ NEW FEATURES:
 
 ## 1.10.1 (February 28, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Ensure that timeline sort properly events when healthchecks are removed (eg: maintenance)
 
@@ -396,7 +429,7 @@ IMPROVEMENTS:
 
 ## 1.9.8 (January 16, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * When default value was the same as real value, endpoints were always marked as
    dirty, thus rendering of templates did never succeed.
@@ -412,7 +445,7 @@ IMPROVEMENTS:
 
 ## 1.9.6 (January 15, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Keep connections open properly as it increase timeouts on Consul servers on
    very large templates
@@ -423,7 +456,7 @@ IMPROVEMENTS:
 
 ## 1.9.5 (January 14, 2019)
 
-BUGFIX:
+BUG FIXES:
 
  * Ensure to always re-open Connection to Consul agent in case of network error
 
@@ -473,11 +506,11 @@ IMPROVEMENTS:
 
 OPTIMIZATIONS:
 
- * Better network issue because of X-Consul-Index parsing bug
+ * Better network usage because of X-Consul-Index parsing bug
 
-NEW FEATURES:
+BUG FIXES:
 
- * value.endpoint.x_consul_index now works as expected
+ * `value.endpoint.x_consul_index` now works as expected
 
 IMPROVEMENTS:
 
@@ -517,7 +550,7 @@ NEW FEATURES:
 
 ## 1.8.1 (December 12, 2018)
 
-BUGFIX:
+BUG FIXES:
 
  * Properly fill `template_info` strtucture when hot reload is performed so templates using
    `template_info()` new function can behave nicely.
@@ -626,8 +659,8 @@ IMPROVEMENTS:
  * [Prometheus template](samples/metrics.erb) to export easily Consul
    informations about nodes, datacenters and all services states
  * Code style cleanup + travis now enforces Rubocop
- * Remove criteo references in spec files thanks to @pierrecdn
- * Bitrate display more consistent thanks to @pierrecdn
+ * Remove criteo references in spec files thanks to [@pierrecdn](https://github.com/pierrecdn)
+ * Bitrate display more consistent thanks to [@pierrecdn](https://github.com/pierrecdn)
 
 ## 1.5.3 (September 24, 2018)
 

data/TemplateAPI.md

@@ -388,7 +388,7 @@ name or its ID. If DC is specified, will lookup for given node in another datace
 
 [Find all the checks](https://www.consul.io/api/health.html#list-checks-for-service) of a given service.
 
-## def checks_in_state(check_state, dc: nil, [agent: consul_agent_address])
+## checks_in_state(check_state, dc: nil, [agent: consul_agent_address])
 
 [Find all the checks in a given state](https://www.consul.io/api-docs/health#list-checks-in-state) in the whole cluster.
 

data/bin/consul-templaterb

@@ -375,6 +375,11 @@ ARGV.each do |tpl|
   consul_engine.add_template(tpl, dest)
 end
 
+if consul_engine.templates.count.zero?
+  puts 'You must specify at least one template as arg or using -t'
+  exit 255
+end
+
 template_manager = Consul::Async::EndPointsManager.new(consul_conf, vault_conf, consul_engine.templates, options[:erb][:trim_mode])
 
 Signal.trap('USR1', 'IGNORE') unless Gem.win_platform?

data/lib/consul/async/consul_endpoint.rb

@@ -225,9 +225,34 @@ module Consul
         retry_in / 2 + Consul::Async::Utilities.random.rand(retry_in)
       end
 
+      # rubocop:disable Style/ClassVars
+      def _last_429
+        @@_last_429 ||= { count: 0 }
+      end
+      # rubocop:enable Style/ClassVars
+
       def _handle_error(http, consul_index)
         retry_in = _compute_retry_in([600, conf.retry_duration + 2**@consecutive_errors].min)
-        ::Consul::Async::Debug.puts_error "[#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        if http.response_header.status == 429
+          _last_429
+          retry_in = 60 + Consul::Async::Utilities.random.rand(180) if retry_in < 60
+          _last_429[:time] = Time.now.utc
+          _last_429[:count] += 1
+          if (_last_429[:count] % 10) == 1
+            if _last_429[:count] == 1
+              ::Consul::Async::Debug.puts_error "Rate limiting detected on Consul side (HTTP 429)!\n\n" \
+                                                "******************************* CONFIGURATION ISSUE DETECTED *******************************\n" \
+                                                "* Too many simultaneous connections for Consul agent #{conf.base_url}\n" \
+                                                "* You should tune 'limits.http_max_conns_per_client' to a higher value.\n" \
+                                                "* This program will behave badly until you change this.\n" \
+                                                "* See https://www.consul.io/docs/agent/options.html#http_max_conns_per_client for more info\n" \
+                                                "********************************************************************************************\n\n"
+            end
+            ::Consul::Async::Debug.puts_error "[#{path}] Too many conns to #{conf.base_url}, errors=#{_last_429[:count]} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+          end
+        else
+          ::Consul::Async::Debug.puts_error "[#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        end
         @consecutive_errors += 1
         http_result = HttpResponse.new(http)
         EventMachine.add_timer(retry_in) do
@@ -272,11 +297,12 @@ module Consul
                             end
               new_content = http_result.response.freeze
               modified = @last_result.fake? || @last_result.data != new_content
-              if n_consul_index == consul_index || n_consul_index.nil?
+              if n_consul_index.nil?
                 retry_in = modified ? conf.missing_index_retry_time_on_diff : conf.missing_index_retry_time_on_unchanged
                 n_consul_index = consul_index
               else
                 retry_in = modified ? conf.min_duration : conf.retry_on_non_diff
+                retry_in = 0.1 if retry_in < (Time.now - @last_result.last_update)
               end
               retry_in = _compute_retry_in(retry_in)
               retry_in = 0.1 if retry_in < 0.1

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.28.0'.freeze
+    VERSION = '1.31.0'.freeze
   end
 end

data/samples/consul-ui/.preferences.rb.samples

@@ -0,0 +1 @@
+SORT_CONSUL_SERVICE_NODES = -> (nodes) { nodes.sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] } }

data/samples/consul-ui/README.md

@@ -94,3 +94,9 @@ This app supports the following environment variables:
 * `EXCLUDE_SERVICES`: comma-separated services to exclude (default: lbl7.*,netsvc-probe.*,consul-probed.*)
 * `CONSUL_TIMELINE_BUFFER`: number of entries to keep in the timeline. 1000 by default.
 * `CONSUL_TIMELINE_BLACKLIST`: regexp of services to hide from timeline
+
+### Preferences
+
+Some templates allows you to override some behavior through `.preferences.rb` file.
+In order to do that, rename `.preferences.rb.samples` file to `.preferences.rb`
+and update lambda defined inside to match targeted behavior.

data/samples/consul-ui/consul_services.json.erb

@@ -6,6 +6,19 @@
 #   INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag (default: canary)
 #   EXCLUDE_SERVICES: comma-separated services regexps to exclude (default: lbl7.*,netsvc-probe.*,consul-probed.*)
 
+  unless @sort_consul_service_nodes
+    begin
+      target_dir = File.split(File.expand_path(template_info['source']))[0]
+      target = File.join(target_dir, '.preferences.rb')
+      load "#{target}"
+      STDERR.puts "Using #{target} file."
+      @sort_consul_service_nodes = SORT_CONSUL_SERVICE_NODES
+    rescue LoadError
+      STDERR.puts 'Couldn\'t find .preferences.rb file ; default configuration will be used.'
+      @sort_consul_service_nodes = -> (nodes) { nodes.sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] } }
+    end
+  end
+
   service_tag_filter = ENV['SERVICES_TAG_FILTER'] || nil
   instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
   instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG']
@@ -20,7 +33,7 @@
     if !services_blacklist.any? {|r| r.match(service_name)} && (instance_must_tag.nil? || tags.include?(instance_must_tag))
       tags_per_service[service_name] = tags.sort
       the_backends = []
-      service(service_name).sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |snode|
+      @sort_consul_service_nodes.call(service(service_name)).each do |snode|
         tags_of_instance = snode['Service']['Tags'].sort
         if (instance_must_tag.nil? || tags_of_instance.include?(instance_must_tag)) && !tags_of_instance.include?(instance_exclude_tag)
           the_backends << snode if snode['Service']['Port']

data/samples/consul-ui/decorators.js.erb

@@ -36,6 +36,13 @@ function usefullLinksGenerator(instance, serviceName, node_meta_info) {
     return top;
 }
 
+/**
+ * createNodeDisplayElement resolves and displays the node name.
+ */
+function createNodeDisplayElement(nodeName, nodemeta) {
+  return document.createTextNode(nodeName);
+}
+
 /**
  * serviceInstanceDecorator is called to decorate an instance.
  */

data/samples/consul-ui/js/nodes.js

@@ -71,7 +71,7 @@ class NodeMainSelector extends MainSelector {
         content.setAttribute('class', 'instance-content');
         var contentHead = document.createElement('div');
         contentHead.setAttribute('class', 'instance-content-header');
-        contentHead.appendChild(nodeNameGenator(node['Node']['Name'], node['Node']['Address']));
+        contentHead.appendChild(nodeNameGenator(node['Node']));
         contentHead.appendChild(nodeAddressGenator(node['Node']['Address']));
         contentHead.appendChild(nodeMetaGenerator(node['Node']['Meta']));
         content.appendChild(contentHead);

data/samples/consul-ui/js/service.js

@@ -304,8 +304,8 @@ class ServiceMainSelector extends MainSelector {
         element.setAttribute("class", "list-group-item service-instance");
         var state = nodeState(instance.checks);
         element.appendChild(weightsGenerator(instance.weights, state));
-        element.appendChild(serviceTitleGenerator(instance, serviceName));
         var node_info = this.nodes[instance.name];
+        element.appendChild(serviceTitleGenerator(instance, serviceName, node_info));
         if (node_info != null) {
             node_info = node_info.meta;
             element.appendChild(nodeMetaGenerator(node_info, instance.sMeta));

data/samples/consul-ui/js/utils.js

@@ -63,7 +63,7 @@ function nodeState(checks) {
 
 supported_protocols = ['https', 'http', 'sftp', 'ftp', 'ssh', 'telnet']
 
-function serviceTitleGenerator(instance, serviceName) {
+function serviceTitleGenerator(instance, serviceName, node_info) {
     var protocol = null;
     for (i in supported_protocols) {
         var protoc = supported_protocols[i]
@@ -86,7 +86,10 @@ function serviceTitleGenerator(instance, serviceName) {
         instanceLink.setAttribute('target', '_blank');
     }
 
-    instanceLink.appendChild(document.createTextNode(instance.name + appendPort));
+    var nodemeta = (node_info != null) ? node_info.meta : null; 
+    instanceLink.appendChild(createNodeDisplayElement(instance.name, nodemeta));
+    instanceLink.appendChild(document.createTextNode(appendPort));
+
     const nodeInfo = document.createElement('a');
     nodeInfo.appendChild(document.createTextNode('\u24D8'));
     nodeInfo.setAttribute('title', 'Click to see details of Node: ' + instance.name +
@@ -103,7 +106,7 @@ function serviceTitleGenerator(instance, serviceName) {
     return htmlTitle;
 }
 
-function nodeNameGenator(nodename, nodeaddr) {
+function nodeNameGenator(node) {
     var protocol = 'ssh://'
 
     var htmlTitle = document.createElement('h5');
@@ -111,10 +114,10 @@ function nodeNameGenator(nodename, nodeaddr) {
     var instanceLink = document.createElement('a');
     instanceLink.setAttribute('class', 'instance-name');
     if (protocol != null) {
-        instanceLink.setAttribute('href', protocol + nodeaddr);
+        instanceLink.setAttribute('href', protocol + node['Address']);
         instanceLink.setAttribute('target', '_blank');
     }
-    instanceLink.appendChild(document.createTextNode(nodename));
+    instanceLink.appendChild(createNodeDisplayElement(node['Name'], node['Meta']));
     htmlTitle.appendChild(instanceLink);
 
     return htmlTitle;