consul-templaterb 1.28.1 → 1.31.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca62994383929bd3874db5527f85efeaaf184fdb83b8f3b17e05fbadbfbbc90a
-  data.tar.gz: e878ff880557ffdfd8db6e5cb900a19e8191d4db11ba28fa0b4e7bbe9b1b9240
+  metadata.gz: f5d188151cdf2f004d4993cc5620ad86ce8632e2de820ca6498d37ce8acbd4c9
+  data.tar.gz: 6469c2b6d1a9032bb16fdfbe53b062a2271e3a787dcc0f52bb0e749c081081e4
 SHA512:
-  metadata.gz: 62afc852ef61329a30bba3a67f43ed372130d9a87cd1afce559b5f88f9aa0a6db336c84c63b173eaa96896e8c9088686cfa37f10d540cd91c80ca2fb976b00b3
-  data.tar.gz: 4a02aa15f2d4f95e5d144e916bf6a51190a90756e134456aece5b4afbf7ba106e150faeb8cf7b0e09ff27f1f61b237c47dd11f236299d922a519e91b019e3705
+  metadata.gz: 498a5282930d98afe30840f8f167559783aafa7d17b9a902ff8950dac5727d8496c666d0dea01d10ac59cfe4e845ca5664ded482b16641d4723f3edd5fa9053a
+  data.tar.gz: a2526b83b9083275b6d1057948d80457f8ea4aede46201b15408fe331a329430d6785f8372339b8c2e139da2f1fe96c13a699d4b6c936f98d411c26b797413c2

data/CHANGELOG.md

@@ -1,8 +1,41 @@
 # CHANGELOG
 
-## (UNRELEASED)
+## 1.31.1 (Jan 25, 2021)
 
-## 1.28.1 (Sept 30,, 2020)
+IMPROVEMENTS:
+
+* Use Node.Meta.fqdn to display node names in timeline if set
+
+## 1.31.0 (Jan 18, 2021)
+
+IMPROVEMENTS:
+
+* Do not apply penalty on unmodified services not changing much (Will improve streaming performance)
+
+## 1.30.0 (Dec 18, 2020)
+
+IMPROVEMENTS:
+
+ * Allow customization to sort nodes in service view in Consul-UI ([#71](https://github.com/criteo/consul-templaterb/pull/71))
+
+NEW FEATURES:
+
+ * Auto-detection of rate-limit on Consul Side (introduced in Consul 1.6.2+), this feature
+   will work only Consul 1.9+ (see [Consul #7527](https://github.com/hashicorp/consul/issues/7527)). 
+
+## 1.29.0 (Oct 6, 2020)
+
+IMPROVEMENTS:
+
+ * When no template has been specified, stop early with an error message
+
+## 1.28.2 (Oct 5, 2020)
+
+NEW FEATURES:
+
+ * Added new decorator to compute node names in consul-ui ([#70](https://github.com/criteo/consul-templaterb/pull/70))
+
+## 1.28.1 (Sept 30, 2020)
 
 IMPROVEMENTS:
 

data/bin/consul-templaterb

@@ -375,6 +375,11 @@ ARGV.each do |tpl|
   consul_engine.add_template(tpl, dest)
 end
 
+if consul_engine.templates.count.zero?
+  puts 'You must specify at least one template as arg or using -t'
+  exit 255
+end
+
 template_manager = Consul::Async::EndPointsManager.new(consul_conf, vault_conf, consul_engine.templates, options[:erb][:trim_mode])
 
 Signal.trap('USR1', 'IGNORE') unless Gem.win_platform?

data/lib/consul/async/consul_endpoint.rb

@@ -225,9 +225,34 @@ module Consul
         retry_in / 2 + Consul::Async::Utilities.random.rand(retry_in)
       end
 
+      # rubocop:disable Style/ClassVars
+      def _last_429
+        @@_last_429 ||= { count: 0 }
+      end
+      # rubocop:enable Style/ClassVars
+
       def _handle_error(http, consul_index)
         retry_in = _compute_retry_in([600, conf.retry_duration + 2**@consecutive_errors].min)
-        ::Consul::Async::Debug.puts_error "[#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        if http.response_header.status == 429
+          _last_429
+          retry_in = 60 + Consul::Async::Utilities.random.rand(180) if retry_in < 60
+          _last_429[:time] = Time.now.utc
+          _last_429[:count] += 1
+          if (_last_429[:count] % 10) == 1
+            if _last_429[:count] == 1
+              ::Consul::Async::Debug.puts_error "Rate limiting detected on Consul side (HTTP 429)!\n\n" \
+                                                "******************************* CONFIGURATION ISSUE DETECTED *******************************\n" \
+                                                "* Too many simultaneous connections for Consul agent #{conf.base_url}\n" \
+                                                "* You should tune 'limits.http_max_conns_per_client' to a higher value.\n" \
+                                                "* This program will behave badly until you change this.\n" \
+                                                "* See https://www.consul.io/docs/agent/options.html#http_max_conns_per_client for more info\n" \
+                                                "********************************************************************************************\n\n"
+            end
+            ::Consul::Async::Debug.puts_error "[#{path}] Too many conns to #{conf.base_url}, errors=#{_last_429[:count]} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+          end
+        else
+          ::Consul::Async::Debug.puts_error "[#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        end
         @consecutive_errors += 1
         http_result = HttpResponse.new(http)
         EventMachine.add_timer(retry_in) do
@@ -272,11 +297,12 @@ module Consul
                             end
               new_content = http_result.response.freeze
               modified = @last_result.fake? || @last_result.data != new_content
-              if n_consul_index == consul_index || n_consul_index.nil?
+              if n_consul_index.nil?
                 retry_in = modified ? conf.missing_index_retry_time_on_diff : conf.missing_index_retry_time_on_unchanged
                 n_consul_index = consul_index
               else
                 retry_in = modified ? conf.min_duration : conf.retry_on_non_diff
+                retry_in = 0.1 if retry_in < (Time.now - @last_result.last_update)
               end
               retry_in = _compute_retry_in(retry_in)
               retry_in = 0.1 if retry_in < 0.1

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.28.1'.freeze
+    VERSION = '1.31.1'.freeze
   end
 end

data/samples/consul-ui/.preferences.rb.samples

@@ -0,0 +1 @@
+SORT_CONSUL_SERVICE_NODES = -> (nodes) { nodes.sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] } }

data/samples/consul-ui/README.md

@@ -94,3 +94,9 @@ This app supports the following environment variables:
 * `EXCLUDE_SERVICES`: comma-separated services to exclude (default: lbl7.*,netsvc-probe.*,consul-probed.*)
 * `CONSUL_TIMELINE_BUFFER`: number of entries to keep in the timeline. 1000 by default.
 * `CONSUL_TIMELINE_BLACKLIST`: regexp of services to hide from timeline
+
+### Preferences
+
+Some templates allows you to override some behavior through `.preferences.rb` file.
+In order to do that, rename `.preferences.rb.samples` file to `.preferences.rb`
+and update lambda defined inside to match targeted behavior.

data/samples/consul-ui/consul_services.json.erb

@@ -6,6 +6,19 @@
 #   INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag (default: canary)
 #   EXCLUDE_SERVICES: comma-separated services regexps to exclude (default: lbl7.*,netsvc-probe.*,consul-probed.*)
 
+  unless @sort_consul_service_nodes
+    begin
+      target_dir = File.split(File.expand_path(template_info['source']))[0]
+      target = File.join(target_dir, '.preferences.rb')
+      load "#{target}"
+      STDERR.puts "Using #{target} file."
+      @sort_consul_service_nodes = SORT_CONSUL_SERVICE_NODES
+    rescue LoadError
+      STDERR.puts 'Couldn\'t find .preferences.rb file ; default configuration will be used.'
+      @sort_consul_service_nodes = -> (nodes) { nodes.sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] } }
+    end
+  end
+
   service_tag_filter = ENV['SERVICES_TAG_FILTER'] || nil
   instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
   instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG']
@@ -20,7 +33,7 @@
     if !services_blacklist.any? {|r| r.match(service_name)} && (instance_must_tag.nil? || tags.include?(instance_must_tag))
       tags_per_service[service_name] = tags.sort
       the_backends = []
-      service(service_name).sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |snode|
+      @sort_consul_service_nodes.call(service(service_name)).each do |snode|
         tags_of_instance = snode['Service']['Tags'].sort
         if (instance_must_tag.nil? || tags_of_instance.include?(instance_must_tag)) && !tags_of_instance.include?(instance_exclude_tag)
           the_backends << snode if snode['Service']['Port']

data/samples/consul-ui/decorators.js.erb

@@ -36,6 +36,13 @@ function usefullLinksGenerator(instance, serviceName, node_meta_info) {
     return top;
 }
 
+/**
+ * createNodeDisplayElement resolves and displays the node name.
+ */
+function createNodeDisplayElement(nodeName, nodemeta) {
+  return document.createTextNode(nodeName);
+}
+
 /**
  * serviceInstanceDecorator is called to decorate an instance.
  */

data/samples/consul-ui/js/nodes.js

@@ -71,7 +71,7 @@ class NodeMainSelector extends MainSelector {
         content.setAttribute('class', 'instance-content');
         var contentHead = document.createElement('div');
         contentHead.setAttribute('class', 'instance-content-header');
-        contentHead.appendChild(nodeNameGenator(node['Node']['Name'], node['Node']['Address']));
+        contentHead.appendChild(nodeNameGenator(node['Node']));
         contentHead.appendChild(nodeAddressGenator(node['Node']['Address']));
         contentHead.appendChild(nodeMetaGenerator(node['Node']['Meta']));
         content.appendChild(contentHead);

data/samples/consul-ui/js/service.js

@@ -304,8 +304,8 @@ class ServiceMainSelector extends MainSelector {
         element.setAttribute("class", "list-group-item service-instance");
         var state = nodeState(instance.checks);
         element.appendChild(weightsGenerator(instance.weights, state));
-        element.appendChild(serviceTitleGenerator(instance, serviceName));
         var node_info = this.nodes[instance.name];
+        element.appendChild(serviceTitleGenerator(instance, serviceName, node_info));
         if (node_info != null) {
             node_info = node_info.meta;
             element.appendChild(nodeMetaGenerator(node_info, instance.sMeta));

data/samples/consul-ui/js/timeline.js

@@ -434,7 +434,7 @@ class ServiceTimeline {
             }
             var text = e.instance;
             if (e.instance_info && e.instance_info.node) {
-                text = e.instance_info.node;
+                text = e.instance_info.fqdn;
                 if (e.instance_info.port > 0) {
                     text += ":" + e.instance_info.port
                 }

data/samples/consul-ui/js/utils.js

@@ -63,7 +63,7 @@ function nodeState(checks) {
 
 supported_protocols = ['https', 'http', 'sftp', 'ftp', 'ssh', 'telnet']
 
-function serviceTitleGenerator(instance, serviceName) {
+function serviceTitleGenerator(instance, serviceName, node_info) {
     var protocol = null;
     for (i in supported_protocols) {
         var protoc = supported_protocols[i]
@@ -86,7 +86,10 @@ function serviceTitleGenerator(instance, serviceName) {
         instanceLink.setAttribute('target', '_blank');
     }
 
-    instanceLink.appendChild(document.createTextNode(instance.name + appendPort));
+    var nodemeta = (node_info != null) ? node_info.meta : null; 
+    instanceLink.appendChild(createNodeDisplayElement(instance.name, nodemeta));
+    instanceLink.appendChild(document.createTextNode(appendPort));
+
     const nodeInfo = document.createElement('a');
     nodeInfo.appendChild(document.createTextNode('\u24D8'));
     nodeInfo.setAttribute('title', 'Click to see details of Node: ' + instance.name +
@@ -103,7 +106,7 @@ function serviceTitleGenerator(instance, serviceName) {
     return htmlTitle;
 }
 
-function nodeNameGenator(nodename, nodeaddr) {
+function nodeNameGenator(node) {
     var protocol = 'ssh://'
 
     var htmlTitle = document.createElement('h5');
@@ -111,10 +114,10 @@ function nodeNameGenator(nodename, nodeaddr) {
     var instanceLink = document.createElement('a');
     instanceLink.setAttribute('class', 'instance-name');
     if (protocol != null) {
-        instanceLink.setAttribute('href', protocol + nodeaddr);
+        instanceLink.setAttribute('href', protocol + node['Address']);
         instanceLink.setAttribute('target', '_blank');
     }
-    instanceLink.appendChild(document.createTextNode(nodename));
+    instanceLink.appendChild(createNodeDisplayElement(node['Name'], node['Meta']));
     htmlTitle.appendChild(instanceLink);
 
     return htmlTitle;

data/samples/consul-ui/timeline.json.erb

@@ -41,10 +41,12 @@ cur_state = services.map do |service_name, _tags|
       mindex = chk['ModifyIndex'] || 0
       mod_index = mindex if mod_index < mindex
     end
-    ["#{instance['Node']['Node']}:#{instance['Service']['ID']}",
+    node_name = instance['Node']['Node']
+    ["#{node_name}:#{instance['Service']['ID']}",
      {
        'address' => instance.service_address,
-       'node' => instance['Node']['Node'],
+       'node' => node_name,
+       'fqdn' => instance.node_meta['fqdn'] || node_name,
        'port' => instance['Service']['Port'],
        'idx'  => mod_index || cur_index,
        'status' => instance.status,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.28.1
+  version: 1.31.1
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-29 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -181,6 +181,7 @@ files:
 - samples/all_services_multi_agents.txt.erb
 - samples/all_templates.erb
 - samples/checks_in_warning_or_critical_state.yaml.erb
+- samples/consul-ui/.preferences.rb.samples
 - samples/consul-ui/README.md
 - samples/consul-ui/common/footer.html.erb
 - samples/consul-ui/common/header.html.erb