consul-templaterb 1.25.2 → 1.27.0

data/lib/consul/async/json_endpoint.rb

@@ -9,7 +9,7 @@ module Consul
     class JSONConfiguration
       attr_reader :url, :retry_duration, :min_duration, :retry_on_non_diff,
                   :debug, :enable_gzip_compression, :request_method, :json_body,
-                  :headers
+                  :headers, :tls_cert_chain, :tls_private_key, :tls_verify_peer
       def initialize(url:,
                      debug: { network: false },
                      retry_duration: 10,
@@ -18,7 +18,10 @@ module Consul
                      request_method: :get,
                      json_body: nil,
                      headers: {},
-                     enable_gzip_compression: true)
+                     enable_gzip_compression: true,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @url = url
         @debug = debug
         @enable_gzip_compression = enable_gzip_compression
@@ -28,6 +31,9 @@ module Consul
         @request_method = request_method
         @json_body = json_body
         @headers = headers
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def create(_url)
@@ -181,6 +187,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 60 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = {
           conn: EventMachine::HttpRequest.new(conf.url, options)
         }

data/lib/consul/async/vault_endpoint.rb

@@ -10,7 +10,8 @@ module Consul
     # Configuration for Vault Endpoints
     class VaultConfiguration
       attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
-                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors
+                  :lease_duration_factor, :debug, :max_consecutive_errors_on_endpoint, :fail_fast_errors, :tls_cert_chain, :tls_private_key,
+                  :tls_verify_peer
 
       def initialize(base_url: 'http://localhost:8200',
                      debug: { network: false },
@@ -22,7 +23,10 @@ module Consul
                      max_retry_duration: 600,
                      paths: {},
                      max_consecutive_errors_on_endpoint: 10,
-                     fail_fast_errors: false)
+                     fail_fast_errors: false,
+                     tls_cert_chain: nil,
+                     tls_private_key: nil,
+                     tls_verify_peer: true)
         @base_url = base_url
         @token_renew = token_renew
         @debug = debug
@@ -34,6 +38,9 @@ module Consul
         @token = token
         @max_consecutive_errors_on_endpoint = max_consecutive_errors_on_endpoint
         @fail_fast_errors = fail_fast_errors
+        @tls_cert_chain = tls_cert_chain
+        @tls_private_key = tls_private_key
+        @tls_verify_peer = tls_verify_peer
       end
 
       def ch(path, symbol)
@@ -46,10 +53,15 @@ module Consul
         end
       end
 
-      def create(path)
+      def create(path, agent: nil)
         return self unless @paths[path.to_sym]
 
-        VaultConfiguration.new(base_url: ch(path, :base_url),
+        base_url = ch(path, :base_url)
+        if agent
+          agent = "http://#{agent}" unless agent.start_with? 'http', 'https'
+          base_url = agent
+        end
+        VaultConfiguration.new(base_url: base_url,
                                debug: ch(path, :debug),
                                token: ch(path, :token),
                                retry_duration: ch(path, :retry_duration),
@@ -117,8 +129,8 @@ module Consul
     class VaultEndpoint
       attr_reader :conf, :path, :http_method, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
 
-      def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data = {})
-        @conf = conf.create(path)
+      def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data = {}, agent: nil)
+        @conf = conf.create(path, agent: agent)
         @default_value = default_value
         @path = path
         @http_method = http_method
@@ -221,6 +233,13 @@ module Consul
           connect_timeout: 5, # default connection setup timeout
           inactivity_timeout: 1 # default connection inactivity (post-setup) timeout
         }
+        unless conf.tls_cert_chain.nil?
+          options[:tls] = {
+            cert_chain_file: conf.tls_cert_chain,
+            private_key_file: conf.tls_private_key,
+            verify_peer: conf.tls_verify_peer
+          }
+        end
         connection = EventMachine::HttpRequest.new(conf.base_url, options)
         cb = proc do |_|
           http = connection.send(http_method.downcase, build_request) # Under the hood: c.send('get', {stuff}) === c.get({stuff})

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.25.2'.freeze
+    VERSION = '1.27.0'.freeze
   end
 end

data/samples/all_services.txt.erb

@@ -9,7 +9,7 @@
   require 'set'
 
   # Services to hide
-  services_blacklist_raw = (ENV['EXCLUDE_SERVICES'] || '.*netsvc-probe.*,.*consul-probed.*').split(',')
+  services_blacklist_raw = (ENV['EXCLUDE_SERVICES'] || '.*netsvc-probe.*,.*consul-probed.*,.lbl7.*').split(',')
   services_blacklist = services_blacklist_raw.map { |v| Regexp.new(v) }
 
   num_services={}

data/samples/all_services_multi_agents.txt.erb

@@ -0,0 +1,69 @@
+ __________________________________________________________________
+|  DC   | Services |______________Instances________________| Nodes |
+|       |          | Passing | Warning | Critic  |  Total  |       |
+|-------+----------+---------+---------+---------+---------+-------|
+<%
+  # This template list all instances on all DCs
+  # And aggregates a list of Services, Services Instances/status
+  # And nodes.
+  require 'set'
+
+
+  def find_best_agent_for_dc(dc)
+    agent = service('consul-relay', passing: true, dc:dc).first
+    agent = service('consul-agent-http', passing: true, dc:dc).first unless agent
+    service('consul', passing: true, dc:dc).first unless agent
+    return nil unless agent
+    port = agent['Service']['Port'] || 8500
+    "http://#{agent.service_address}:#{port}"
+  end
+
+  # Services to hide
+  services_blacklist_raw = (ENV['EXCLUDE_SERVICES'] || '.*netsvc-probe.*,.*consul-probed.*').split(',')
+  services_blacklist = services_blacklist_raw.map { |v| Regexp.new(v) }
+
+  num_services={}
+  num_instances={}
+  all_states = ['passing', 'warning', 'critical', 'total']
+  distinct_services=Set.new
+  datacenters().each do |dc|
+    next unless find_best_agent_for_dc(dc)
+    num_services[dc] = 0
+    num_instances[dc] = {
+      'passing'  => 0,
+      'warning'  => 0,
+      'critical' => 0,
+      'total'    => 0
+    }
+    services(dc:dc, agent: find_best_agent_for_dc(dc)).each do |service_name, tags|
+      next if services_blacklist.any? {|r| r.match(service_name)}
+      distinct_services.add(service_name)
+      num_services[dc]+=1
+      service(service_name, dc:dc, agent: find_best_agent_for_dc(dc)).each do |snode|
+        num_instances[dc][snode.status]+=1
+        num_instances[dc]['total']+=1
+      end 
+    end
+  end
+  num_instances_total={
+    'passing'  => 0,
+    'warning'  => 0,
+    'critical' => 0,
+    'total'    => 0
+  }
+  num_nodes_total=0
+  num_services.each do |dc, num_services_for_dc|
+    all_states.each do |s|
+      num_instances_total[s] += num_instances[dc][s]
+    end
+    num_nodes=nodes(dc:dc, agent: find_best_agent_for_dc(dc)).count
+    num_nodes_total+=num_nodes
+%>| <%= dc.rjust(5) %> | <%= num_services_for_dc.to_s.rjust(8) %> |<% all_states.each do |status|
+%> <%= num_instances[dc][status].to_s.rjust(7) %> |<% end %> <%= num_nodes.to_s.rjust(5) %> |
+<%
+  end
+%>|-------+----------+---------+---------+---------+---------+-------|
+| TOTAL | <%= distinct_services.count.to_s.rjust(8) %> |<% all_states.each do |status|
+%> <%= num_instances_total[status].to_s.rjust(7) %> |<%
+end %> <%= num_nodes_total.to_s.rjust(5) %> |
+'_______|__________|_________|_________|_________|_________|_______'

data/samples/checks_in_warning_or_critical_state.yaml.erb

@@ -0,0 +1,13 @@
+<%=
+  # This sample displays checks for the whole cluster
+  # in warning or critical state
+  # API available with Consul 1.7+
+  res = []
+  checks_in_state('warning').each do |c|
+    res << c
+  end
+  checks_in_state('critical').each do |c|
+    res << c
+  end
+  YAML.dump({'warning_or_critical_checks' => res})
+%>

data/samples/consul-ui/README.md

@@ -50,6 +50,25 @@ The content is statically created, so you can serve it using any HTTP server ver
 For development, you might use `python -m SimpleHTTPServer` in order to server the result
 on your browser.
 
+### Running it in production
+
+Whatever your solution, be sure to have a index.html, so read next below on
+how generating an index.html.
+
+#### Running with web server
+
+You can run it with your favoite web server, at Criteo we run it with nginx
+which handles nicely cache and offer good performance.
+
+In that case, consul-templaterb can start nginx with `--exec` or you can run it
+as a daemon, in which case, consul-templaterb only generate the files.
+
+#### Run it in Consul
+
+You can run consul with `-ui-dir=/path/to/directory/of/consul-ui`, in such case
+reaching consul on poort 8500 will redirect to the /ui/ path, displaying the UI
+of your choice on http://consul-agent.example.org:8500/ui/.
+
 ### Generating index.html
 
 By default, the command `consul-templaterb -c http://localhost:8500 samples/consul-ui/*.erb``

data/samples/consul-ui/decorators.js.erb

@@ -1,18 +1,5 @@
-// Utilities
+// Use this file to configure custom decorators for your Consul-UI
 var httpRegexp = new RegExp('^http[s]?://[^ ]+$');
-var dc = "<%= ENV['CRITEO_DC'] || 'par'%>";
-var env = "<%= ENV['CRITEO_ENV'] || 'preprod'%>";
-
-var availability_url = 'https://grafana.crto.in/d/xFX5gCnWz/service-availability?var-datacenter=' + dc + '&var-service=';
-if (env == 'preprod') {
-  availability_url = 'https://grafana.preprod.crto.in/d/E0ANGjnZz/service-availability?var-datacenter=' + dc + '&var-service=';
-}
-var swagger_url = 'https://swaggercatalogapp.' + dc + '.' + env + '.crto.in/explore/swagger?key=';
-var slack_url = 'https://criteo.slack.com/app_redirect?channel=';
-
-var rackguru_url = 'https://rackguru.' + env + '.crto.in/serial/';
-
-var asapi_url = 'https://idm.' + env + '.crto.in/tool/multiGroupInfo/'
 
 function url_decorator(key, value) {
   var e = document.createElement('a');
@@ -26,27 +13,7 @@ function usefullLinksGenerator(instance, serviceName, node_meta_info) {
     top.className = 'instance-links';
 
     var usefullLinks = [
-        {
-            title: "Trigger security scan",
-            iconClassName: "fas fa-shield-alt",
-            href: "https://security.crto.in/#/scan/?ip=" + instance.addr
-        },
-        {
-            title: "Availability Graph",
-            iconClassName: "fas fa-chart-area",
-            href: availability_url + serviceName
-        },
     ];
-    if (node_meta_info!= null) {
-      var serial = node_meta_info['serial_number'];
-      if (serial != null) {
-        usefullLinks.push({
-            title: "RackGuru",
-            iconClassName: "fas fa-server",
-            href: rackguru_url + serial
-        });
-      }
-    }
     var first = true;
     for (let usefullLink of usefullLinks) {
         link = document.createElement('a');
@@ -134,19 +101,6 @@ function groups_decorator(instance, key, value, serviceName) {
   return span;
 }
 
-/**
- * Decorates with a slack channel link
- */
-function slack_channel(instance, key, value, serviceName) {
-  var sName = value;
-  if (sName.startsWith('#')) {
-    sName = sName.substring(1);
-  }
-  return build_link(slack_url + encodeURIComponent(sName), '#'+ sName);
-}
-
-const start_regexp = /(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z/
-
 function decorateIsoDate(value, formated) {
   const parsed  = Date.parse(formated);
   if (isNaN(parsed)) {
@@ -163,55 +117,6 @@ function decorateIsoDate(value, formated) {
 // This is the list of function called
 // When a service meta is found
 var registered_decorators = {
-    alert_availability_slack_channel: slack_channel,
-    gerrit_repository: function(instance, key, value, serviceName) {
-      return build_link('https://review.crto.in/#/q/' + value, value);
-    },
-    version: function(instance, key, value, serviceName) {
-      var asInt = parseInt(value);
-      if (asInt < 10000) {
-        return document.createTextNode(value);
-      }
-      if (instance.sMeta['CRITEO_APP_POOL'] != null) {
-          return build_link("https://devtools.crto.in/log.html?moab=cs&im=nb-to&range=100%2C" + asInt, value);
-      } else {
-          var tUrl = "https://devtools.crto.in/log.html?moab=j&im=nb-to&range=100%2C" + asInt;
-          if (instance.sMeta['jvm_artifact'] != null) {
-            tUrl += '&with-dependencies=true&artifacts=' + encodeURIComponent(instance.sMeta['jvm_artifact']);
-          }
-          return build_link(tUrl, value);
-      }
-    },
-    MESOS_TERM_DEBUG_GRANTED_TO: groups_decorator,
-    OWNERS: groups_decorator,
-    marathon_app_id: function(instance, key, value, serviceName) {
-      var app_name = value;
-      if (app_name[0] != '/') {
-        app_name = '/' + app_name;
-      }
-      if (instance.sMeta && instance.sMeta['marathon_ui']) {
-        return build_link(instance.sMeta['marathon_ui'] + '/#/apps/' + encodeURIComponent(app_name), value);
-      } else {
-        // non decorated value
-        return document.createTextNode(value);
-      }
-    },
-    marathon_app_version: function(instance, key, value, serviceName) {
-      return decorateIsoDate(value, value);
-    },
-    slack_channel: slack_channel,
-    start: function(instance, key, value, serviceName) {
-        const reg = start_regexp.exec(value);
-        if (reg != null) {
-          var formated = reg[1] + '-' + reg[2] + '-' + reg[3] + 'T' + reg[4] + ':' + reg[5] + ':' + reg[6] + 'Z';
-          return decorateIsoDate(value, formated);
-        } else {
-          return document.createTextNode(value);
-        }
-    },
-    swagger_key: function(instance, key, value, serviceName) {
-        return build_link(swagger_url + encodeURIComponent(value), value);
-    },
 }
 
 function service_meta_semantics_decorator(instance, key, value, serviceName) {
@@ -286,85 +191,11 @@ function nodeMetaGenerator(nodeMetaTags, serviceMetaIfAny) {
  * it does not have to return anything.
  */
 function navBarDecorator(navbar) {
-  if (typeof consulManager === 'undefined') {
-    // Timepicker is not supported on this page
-    return;
-  }
-  var timepicker_container = document.createElement('div');
-  timepicker_container.innerHTML = `
-              <div class="row">
-                <button type="button" disabled class="btn btn-secondary" data-toggle="tooltip" data-html="true" title="Data returned is the closest available to request" id="currently-displayed-data-date">
-                  Pick a date to see data from the past
-                </button>
-                <div class="col-sm-14">
-                  <div class="form-group">
-                    <div class="input-group date" id="datetimepicker1" data-target-input="nearest">
-                      <input type="text" class="form-control datetimepicker-input" data-target="#datetimepicker1"/>
-                      <div class="input-group-append" data-target="#datetimepicker1" data-toggle="datetimepicker">
-                        <div class="input-group-text"><i class="fa fa-calendar"></i></div>
-                      </div>
-                    </div>
-                  </div>
-                </div>
-              </div>
-              `;
-
-  navbar.appendChild(timepicker_container);
-  var script= document.createElement('script');
-  script.type='text/javascript';
-  // TODO(g.seux): extract code to another file and set "source" on script element
-  script.innerHTML = `
-                    $('#datetimepicker1').datetimepicker({
-                      format: 'DD/MM/YYYY HH:mm:ss Z',  // default format does not allow to select seconds
-                      sideBySide: true, // display date+time on the same widget
-                      useCurrent: true // by default, select current date
-                    });
-                  $("#datetimepicker1").on("change.datetimepicker", function (e) {
-                    console.log("Will fetch data from date " + e.date);
-                    console.log("will clean existing data");
-                    consulManager.clean().then(function(result) {
-                      switch(consulManager.constructor.name) {
-                        case "ConsulServiceManager":
-                          backup_type = 'consul_services';
-                          break;
-                        case "ConsulKeysManager":
-                          backup_type  = 'consul_keys'
-                          break;
-                        case "ConsulNodesManager":
-                          backup_type = 'consul_nodes'
-                          break;
-                        default:
-                          console.log("Unknown " + consulManager.constructor.name + " type");
-                      }
-                      if (e.date) {
-                        console.log("Will replace data by closest snapshot to " + e.date);
-                        var target_url = "https://consul-info-timeline-history.<%= ENV['CRITEO_DC'] %>.<%= ENV['CRITEO_ENV']%>.crto.in/backup/" + backup_type + "/" + e.date / 1000;
-                      } else {
-                        console.log("Will restore to local version");
-                        var target_url = defaultConsulManager.resourceURL;
-                      }
-                      if (typeof(defaultConsulManager) == 'undefined') {
-                        // store first manager to be able to restore it
-                        defaultConsulManager = consulManager
-                      }
-                      consulManager = new consulManager.constructor(target_url);
-                    });
-                  });
-  `;
-  navbar.appendChild(script);
 }
 
-
 /**
  * fetchedResponseDecorator is called with http response when a resource is fetched by any instance of ConsulUIManager
  * it does not have to return anything.
  */
 async function fetchedResponseDecorator(httpResponse) {
-  const data_date = await httpResponse.headers.get('X-Consul-Snapshot-Timestamp');
-  current_date_display = $('#currently-displayed-data-date');
-  if (data_date > 0) {
-    current_date_display.html("Data is a snapshot from: " + moment.unix(data_date).format('DD/MM/YYYY HH:mm:ss Z'));
-  } else {
-    current_date_display.html("Pick a date to see data from the past");
-  }
-}
+}