consul-templaterb 1.5.3 → 1.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 108f2c7ab2a9ad7a5153a9bb0589a5964da4cc6c87354fd8b09504bfde519431
-  data.tar.gz: 7307427ef26bc8e0fec2d38ea64568d327d5df18a530b092d97e71ce14d736cd
+  metadata.gz: 29433f0be694b9d9358498c00ac3f4c0cf2112fdabd86801d94c2afe899243f9
+  data.tar.gz: df4dcdd2a3862122ffb122e3abaad2da1691366fd672c4a111b703586161ad37
 SHA512:
-  metadata.gz: 0791904a177e227316c3c4e062ec1291620dd1e067ac5a3a80f172df501dbd2f902ba747841d2b94b53903bd5ee3ddaa6a33cdbf553f2d730b71af696ff98fb6
-  data.tar.gz: 8c305969be4be2eef1b726ef7ea04dd0d28a0a3861aa05dd08e54911207461e3c696949b029ea4a46d4a603772946d93b115aa4b8f33f3f75e92338e6a9058b8
+  metadata.gz: b20f9e3549f5a886da874cf48b567a264269573c8fdb0204c0d6e3627adc68297411a55de04fa9ec9e8946e5141b8a7dce548d90448d843792b29b8df533d680
+  data.tar.gz: 06722bbfaf0149c3fb3ee5b071de9950fb950ebaa53167926ad8f7d24f0bb86ea43a07e0d08f4809d19e15d7607e6d4c3a5bb6ea5365c0dd3905c403b3b1d664

data/.gitignore

@@ -18,6 +18,7 @@ samples/consul-ui/*.json
 samples/consul-ui/common/*.html
 /samples/criteo/haproxy.cfg
 /samples/consul_template
+/samples/metrics
 /samples/find_blocked_choregraphies
 /spec/consul/async/resources/templates/*.txt
 /spec/consul/async/resources/templates/*.result

data/.rubocop.yml

@@ -7,19 +7,19 @@ Metrics/AbcSize:
   Max: 80
 
 Metrics/BlockLength:
-  Max: 120
+  Max: 135
 
 Metrics/BlockNesting:
   Max: 4
 
 Metrics/ClassLength:
-  Max: 180
+  Max: 200
 
 Metrics/CyclomaticComplexity:
   Max: 15
 
 Metrics/LineLength:
-  Max: 150
+  Max: 175
 
 Metrics/MethodLength:
   Max: 50

data/CHANGELOG.md

@@ -2,9 +2,21 @@
 
 ## (UNRELEASED)
 
-## 1.5.2 (September 24, 2018)
+## 1.5.4 (September 26, 2018)
 
- * added to_a method to Arrays
+IMPROVEMENTS:
+
+ * [Prometheus template](samples/metrics.erb) to export easily Consul
+   informations about nodes, datacenters and all services states
+ * Code style cleanup + travis now enforces Rubocop
+ * Remove criteo references in spec files thanks to @pierrecdn
+ * Bitrate display more consistent thanks to @pierrecdn
+
+## 1.5.3 (September 24, 2018)
+
+IMPROVEMENTS:
+
+ * added to_a method to Arrays objects (ex: datacenters)
  * Improved Consul-UI templates to have proper list of DCs
 
 ## 1.5.2 (September 12, 2018)

data/README.md

@@ -225,6 +225,8 @@ examples:
 6. [Services in XML](samples/consul_template.xml.erb)
 7. [Services in JSON](samples/consul_template.json.erb)
 8. [Generate HaProxy Configuration](samples/ha_proxy.cfg.erb)
+9. [Export Consul Statistics to Prometheus](samples/metrics.erb) : count all services, their state,
+   datacenters and nodes and export it to prometheus easily to trigger alerts.
 
 If you want to test it quickly, you might try with (assuming your consul agent is listening on
 `http://localhost:8500`):

data/Rakefile

@@ -1,6 +1,8 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
 
+RuboCop::RakeTask.new
 RSpec::Core::RakeTask.new(:spec)
 
-task default: :spec
+task default: %i[rubocop spec]

data/lib/consul/async/consul_template.rb

@@ -38,10 +38,8 @@ module Consul
           params: {}
         }
 
-        unless @vault_conf.token.nil? || !@vault_conf.token_renew
-          #Setup token renewal
-          vault_setup_token_renew
-        end
+        # Setup token renewal
+        vault_setup_token_renew unless @vault_conf.token.nil? || !@vault_conf.token_renew
       end
 
       # https://www.consul.io/api/health.html#list-nodes-for-service
@@ -135,16 +133,18 @@ module Consul
 
       def secrets(path = '')
         raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
-        path = "/v1/#{path}".gsub(/\/{2,}/, '/')
-        query_params = {list: "true"}
-        create_if_missing(path, query_params) { ConsulTemplateVaultSecretList.new(VaultEndpoint.new(vault_conf, path, 'GET',true, query_params,JSON.generate(data: {keys: []}))) }
+        path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
+        query_params = { list: 'true' }
+        create_if_missing(path, query_params) do
+          ConsulTemplateVaultSecretList.new(VaultEndpoint.new(vault_conf, path, 'GET', true, query_params, JSON.generate(data: { keys: [] })))
+        end
       end
 
       def secret(path = '', post_data = nil)
         raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
-        path = "/v1/#{path}".gsub(/\/{2,}/, '/')
+        path = "/v1/#{path}".gsub(%r{/{2,}}, '/')
         query_params = {}
-        method = post_data ? "POST" : "GET"
+        method = post_data ? 'POST' : 'GET'
         create_if_missing(path, query_params) { ConsulTemplateVaultSecret.new(VaultEndpoint.new(vault_conf, path, method, true, query_params, JSON.generate(data: {}))) }
       end
 
@@ -228,7 +228,7 @@ module Consul
 
       def vault_setup_token_renew
         path = 'v1/auth/token/renew-self'
-        STDERR.print "[INFO] Setting up vault token renewal"
+        STDERR.print '[INFO] Setting up vault token renewal'
         VaultEndpoint.new(vault_conf, path, :POST, {}, {})
       end
 

data/lib/consul/async/endpoint.rb

@@ -55,11 +55,11 @@ class Endpoint
 
   def build_request(headers = {}, query_params = {})
     req = {
-        head: headers,
-        path: path,
-        query: query_params,
-        keepalive: true,
-        callback: method(:on_response)
+      head: headers,
+      path: path,
+      query: query_params,
+      keepalive: true,
+      callback: method(:on_response)
     }
     @query_params.each_pair do |k, v|
       req[:query][k] = v

data/lib/consul/async/stats.rb

@@ -37,4 +37,3 @@ module Consul
     end
   end
 end
-

data/lib/consul/async/utilities.rb

@@ -7,12 +7,18 @@ module Consul
       def self.bytes_to_h(bytes)
         if bytes < 1024
           "#{bytes} b"
-        elsif bytes < 1_048_576
-          "#{(bytes / 1024).round(2)} Kb"
-        elsif bytes < 1_073_741_824
-          "#{(bytes / 1_048_576.0).round(2)} Mb"
         else
-          "#{(bytes / 1_073_741_824.0).round(2)} Gb"
+          if bytes < 1_048_576
+            bytes_h = bytes / 1024.0
+            unit_prefix = 'K'
+          elsif bytes < 1_073_741_824
+            bytes_h = bytes / 1_048_576.0
+            unit_prefix = 'M'
+          else
+            bytes_h = bytes / 1_073_741_824.0
+            unit_prefix = 'G'
+          end
+          "#{bytes_h.round(2)} #{unit_prefix}b"
         end
       end
 

data/lib/consul/async/vault_endpoint.rb

@@ -29,7 +29,6 @@ module Consul
         @lease_duration_factor = lease_duration_factor
         @paths = paths
         @token = token
-
       end
 
       def ch(path, symbol)
@@ -85,7 +84,6 @@ module Consul
         @data_json = JSON.parse(data) if @data_json.nil?
         @data_json
       end
-
     end
     class VaultHttpResponse
       attr_reader :response_header, :response, :error, :json
@@ -107,7 +105,7 @@ module Consul
     class VaultEndpoint
       attr_reader :conf, :path, :http_method, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
 
-      def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data ={})
+      def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data = {})
         @conf = conf.create(path)
         @default_value = default_value
         @path = path
@@ -123,7 +121,7 @@ module Consul
         @post_data = post_data
         @stopping = false
         @stats = EndPointStats.new
-        @last_result = VaultResult.new(VaultHttpResponse.new(nil, default_value), false, stats ,1)
+        @last_result = VaultResult.new(VaultHttpResponse.new(nil, default_value), false, stats, 1)
         on_response { |result| @stats.on_response result }
         on_error { |http| @stats.on_error http }
         _enable_network_debug if conf.debug && conf.debug[:network]
@@ -161,7 +159,7 @@ module Consul
 
       private
 
-      def build_request()
+      def build_request
         res = {
           head: {
             'Accept' => 'application/json',
@@ -200,7 +198,7 @@ module Consul
         http_result = VaultHttpResponse.new(http, default_value)
         EventMachine.add_timer(retry_in) do
           yield
-          queue.push()
+          queue.push
         end
         @e_callbacks.each { |c| c.call(http_result) }
       end

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.5.3'.freeze
+    VERSION = '1.5.4'.freeze
   end
 end

data/samples/criteo/haproxy.cfg.erb

@@ -116,26 +116,9 @@ listen stats
 
 # ==== Managed by consul-templaterb ====
 
-frontend fe_main # HTTP(S) Service
+frontend fe_main
     bind *:80 name http
 
-    # Monitoring
-    monitor-uri /delivery/monitor/mesos-haproxy/lb-check
-    errorfile 200 /etc/haproxy/200-criteo-ok.txt
-
-    # Adding a header indicating client side is using HTTPS
-    acl http  ssl_fc,not
-    acl https ssl_fc
-    http-request set-header X-Forwarded-Protocol https if https
-
-    # Disable CONNECT globally (identified as vulnerability by audit tools such as Qualys)
-    acl ::disabled_http_methods method CONNECT
-    http-request block if ::disabled_http_methods
-
-    # Consul-agent ACL
-    acl consul-agent-http-acl hdr_dom(host) -i  consul-agent-http.lb-ty5.crto.in consul-agent-http.lb-ty5.central.criteo.prod
-    use_backend bestatic_consul-agent-http if consul-agent-http-acl
-
     # ACLs, HTTP Host header based, following the pattern <consul_app_name>.<fqdn_suffix>
 <% backends.each_pair do |service_name, nodes|
 %>
@@ -144,10 +127,6 @@ frontend fe_main # HTTP(S) Service
 <% end %>
 
 # Backends
-backend bestatic_consul-agent-http
-    http-request deny if !METH_GET # Deny writes
-    server srv0 127.0.0.1:8500
-
 <% backends.each_pair do |service_name, nodes|
 %>
 backend be_<%= service_name %>

data/samples/metrics.erb

@@ -0,0 +1,71 @@
+# A template for exporting metrics for prometheus using consul-templaterb
+# This template can be configure the following way with environment variables
+# Environment variables to filter services/instances
+#   SERVICES_TAG_FILTER: basic tag filter for service (default HTTP)
+#   INSTANCE_MUST_TAG: Second level of filtering (optional, default to SERVICES_TAG_FILTER)
+#   INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag
+#   EXCLUDE_SERVICES: comma-separated services to exclude (example: consul-agent-http,mesos-slave,mesos-agent-watcher)
+<%
+
+  service_tag_filter = ENV['SERVICES_TAG_FILTER'] || nil
+  instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
+  instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG']
+
+  # Services to hide
+  services_blacklist = (ENV['EXCLUDE_SERVICES'] || '').split(',')
+  # Compute the health of a Service
+  def compute_state(snode)
+    states = ['passing', []]
+    snode['Checks'].each do |chk|
+      st = chk['Status']
+      states[1] << st
+      if st == 'critical'
+        states[0] = st
+      elsif st == 'warning' && states[0] == 'passing'
+        states[0] = st
+      end
+    end
+    states
+  end
+  backends = {}
+  service_count = 0
+  services(tag: service_tag_filter).each do |service_name, tags|
+    if !services_blacklist.include?(service_name) && (instance_must_tag.nil? || tags.include?(instance_must_tag))
+      service_count += 1
+      res = {
+        'passing'  => 0,
+        'warning'  => 0,
+        'critical' => 0,
+      }
+      service(service_name).each do |snode|
+        state = compute_state(snode)[0]
+        res[state] += 1
+      end
+      backends[service_name] = res
+    end
+  end
+%>
+# HELP consul_datacenters_count A gauge of number of datacenters available
+# TYPE consul_nodes_count gauge
+consul_datacenters_count <%= datacenters().count %>
+
+# HELP consul_nodes_count A gauge of number of nodes in the cluster
+# TYPE consul_nodes_count gauge
+consul_nodes_count <%= nodes().count %>
+
+# HELP consul_services_count A gauge of the number of services
+# TYPE consul_services_count gauge
+consul_services_count <%= service_count %>
+
+# HELP consul_service_count A gauge of number instances of service with their current state
+# TYPE consul_service_count gauge
+<%
+  json_backends = {}
+  backends.each_pair do |service_name, stats|
+    stats.each_pair do |state_name, state_count|
+%>consul_service_count{service="<%= service_name %>",state="<%= state_name %>"} <%= state_count %>
+<%
+    end
+  end
+%>
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.5.3
+  version: 1.5.4
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-24 00:00:00.000000000 Z
+date: 2018-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -199,6 +199,7 @@ files:
 - samples/debug/compare_connect_services.txt.erb
 - samples/ha_proxy.cfg.erb
 - samples/keys.html.erb
+- samples/metrics.erb
 - samples/nodes.html.erb
 - samples/sample_keys.html.erb
 - samples/services.html.erb