consul-templaterb 1.5.4 → 1.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 29433f0be694b9d9358498c00ac3f4c0cf2112fdabd86801d94c2afe899243f9
-  data.tar.gz: df4dcdd2a3862122ffb122e3abaad2da1691366fd672c4a111b703586161ad37
+  metadata.gz: ed586b6f530661c89df4f8f31dd1f4960742b1fcfd9ea2aed2af44da9f059c05
+  data.tar.gz: d0eb5d09751bd5a8eafac6bf67febce94601ceae65f03ede0f576627f1d179ef
 SHA512:
-  metadata.gz: b20f9e3549f5a886da874cf48b567a264269573c8fdb0204c0d6e3627adc68297411a55de04fa9ec9e8946e5141b8a7dce548d90448d843792b29b8df533d680
-  data.tar.gz: 06722bbfaf0149c3fb3ee5b071de9950fb950ebaa53167926ad8f7d24f0bb86ea43a07e0d08f4809d19e15d7607e6d4c3a5bb6ea5365c0dd3905c403b3b1d664
+  metadata.gz: 865873980923a237af55f5809eb8a9ebb484279897c5dd3f37c6b7a8a19a780ed1bde203a566883f921fb14c4dc8c3e750abdf10807f86546c47d681da7a9483
+  data.tar.gz: 96dfd1d681f5760897609a6ec69b66fde406745ee4fb718d94e8faa566f97581637e08bd57fc91dc520be109221d426da0e9c6d327480e04fc8f0ae5b5dafdbc

data/.travis.yml

@@ -1,6 +1,8 @@
 language: ruby
 rvm:
 - 2.3
+- 2.4
+- 2.5
 deploy:
   provider: rubygems
   api_key:

data/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## (UNRELEASED)
 
+## 1.5.4 (September 27, 2018)
+
+IMPROVEMENTS:
+
+ * Improved [Prometheus template](samples/metrics.erb) with many new metrics
+ * Documentation improvements regarding statistics and endpoints
+ * Code cleanups, code style, remove unused source file
+ * Travis build now runs rubocop and test with Ruby version from 2.3.x to 2.5.x
+
 ## 1.5.4 (September 26, 2018)
 
 IMPROVEMENTS:

data/TemplateAPI.md

@@ -19,12 +19,29 @@ Have a look to [samples/](samples/) directory to start writing your own template
 All objects returned by those functions described below all share the same structure:
 
 * `.result` : handle the result
-* `.endpoint` : get technical information about how data was retrieved and statistics
+* `.endpoint` : get technical information about how data was retrieved and statistics.
+
+## Accessing to statistics to monitor your cluster
+
+All endpoints implement technical interface that allow to get information about what
+is going on in your Consul Cluster.
+
+On each object, you can access the `.endpoint` object that includes several informations about the endpoint being queried:
+
+* `myresult.endpoint.x_consul_index` return the current index on blocking query
+* `myresult.endpoint.stats` a object with interresting fields: `bytes_per_sec`,
+  `bytes_per_sec_human`, `successes`, `errors`, `body_bytes`. All stats details
+  are available in the file [lib/consul/async/stats.rb](lib/consul/async/stats.rb).
+
+Using those statistics might be useful to trigger alerts very easily when something
+is going on. Have a look to [samples/metrics.erb](samples/metrics.erb) that exposes
+most of those metrics to [Prometheus](https://prometheus.io/).
 
 ## Common re-implemented functions for all objects
 
 Most objects returned by all those functions are contained within a `.result` object. However, in order
 to avoid having to write .result in all templates, some shortcuts have been added:
+
 * `[]` allow to either access values for map-based data or arrays
 * for all objects: `.each`, `sort`, `.select`, `.each_value`, `.count`, `.empty?`
 * additionnaly, for map based results, the following methods are available: `.keys`,  `.values`, `.each_pair`,

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.5.4'.freeze
+    VERSION = '1.5.5'.freeze
   end
 end

data/samples/metrics.erb

@@ -41,7 +41,10 @@
         state = compute_state(snode)[0]
         res[state] += 1
       end
-      backends[service_name] = res
+      backends[service_name] = {
+        state: res,
+        stats: service(service_name).endpoint.stats,
+      }
     end
   end
 %>
@@ -59,13 +62,31 @@ consul_services_count <%= service_count %>
 
 # HELP consul_service_count A gauge of number instances of service with their current state
 # TYPE consul_service_count gauge
+# HELP consul_service_stats_requests_total A counter of requests performed by consul-templaterb, show activity on a service
+# TYPE consul_service_stats_requests_total counter
+# HELP consul_service_stats_requests_bytes A counter of bytes transfered from Consul Agent to consul-templaterb
+# TYPE consul_service_stats_requests_bytes counter
+# HELP consul_service_stats_requests_bytes_per_sec A counter of bytes transfered from Consul Agent to consul-templaterb
+# TYPE consul_service_stats_requests_bytes_per_sec gauge
+
 <%
   json_backends = {}
-  backends.each_pair do |service_name, stats|
-    stats.each_pair do |state_name, state_count|
+  backends.each_pair do |service_name, service_info|
+    service_info[:state].each_pair do |state_name, state_count|
 %>consul_service_count{service="<%= service_name %>",state="<%= state_name %>"} <%= state_count %>
 <%
     end
+    current_stats = service_info[:stats]
+%>consul_service_stats_requests_total{service="<%= service_name %>",type="success"} <%= current_stats.successes %>
+<%
+    if current_stats.errors > 0
+%>consul_service_stats_requests_total{service="<%= service_name %>",type="errors"} <%= current_stats.errors %>
+<%
+    end
+%>consul_service_stats_requests_bytes{service="<%= service_name %>"} <%= service_info[:stats].body_bytes %>
+consul_service_stats_requests_bytes_per_sec{service="<%= service_name %>"} <%= service_info[:stats].bytes_per_sec %>
+
+<%
   end
 %>
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.5.4
+  version: 1.5.5
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-26 00:00:00.000000000 Z
+date: 2018-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -165,7 +165,6 @@ files:
 - lib/consul/async/consul_template.rb
 - lib/consul/async/consul_template_engine.rb
 - lib/consul/async/consul_template_render.rb
-- lib/consul/async/endpoint.rb
 - lib/consul/async/process_handler.rb
 - lib/consul/async/stats.rb
 - lib/consul/async/utilities.rb

data/lib/consul/async/endpoint.rb

@@ -1,133 +0,0 @@
-require 'consul/async/utilities'
-
-class Endpoint
-  attr_reader :conf, :path, :x_consul_index, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
-  def initialize(conf, path, enforce_json_200 = true, query_params = {}, default_value = '[]')
-    @conf = conf.create(path)
-    @default_value = default_value
-    @path = path
-    @queue = EM::Queue.new
-    @s_callbacks = []
-    @e_callbacks = []
-    @enforce_json_200 = enforce_json_200
-    @start_time = Time.now.utc
-    @consecutive_errors = 0
-    @query_params = query_params
-    @stopping = false
-    @stats = EndPointStats.new
-    @last_result = ConsulResult.new(default_value, false, HttpResponse.new(nil), 0, stats, 1)
-    on_response { |result| @stats.on_reponse result }
-    on_error { |http| @stats.on_error http }
-    _enable_network_debug if conf.debug && conf.debug[:network]
-    fetch
-    queue << 0
-  end
-
-  def _enable_network_debug
-    on_response do |result|
-      state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
-      stats = result.stats
-      STDERR.puts "[DEBUG]#{state}#{result.modified? ? '[MODIFIED]' : '[NO DIFF]'}" \
-          "[s:#{stats.successes},err:#{stats.errors}]" \
-          "[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
-          " #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
-    end
-    on_error { |http| STDERR.puts "[ERROR]: #{path}: #{http.error}" }
-  end
-
-  def on_response(&block)
-    @s_callbacks << block
-  end
-
-  def on_error(&block)
-    @e_callbacks << block
-  end
-
-  def ready?
-    @ready
-  end
-
-  def terminate
-    @stopping = true
-  end
-
-  private
-
-  def build_request(headers = {}, query_params = {})
-    req = {
-      head: headers,
-      path: path,
-      query: query_params,
-      keepalive: true,
-      callback: method(:on_response)
-    }
-    @query_params.each_pair do |k, v|
-      req[:query][k] = v
-    end
-    req
-  end
-
-  def _handle_error(http, consul_index)
-    retry_in = [600, conf.retry_duration + 2**@consecutive_errors].min
-    STDERR.puts "[ERROR][#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
-    @consecutive_errors += 1
-    http_result = HttpResponse.new(http)
-    EventMachine.add_timer(retry_in) do
-      yield
-      queue.push(consul_index)
-    end
-    @e_callbacks.each { |c| c.call(http_result) }
-  end
-
-  def fetch
-    options = {
-      connect_timeout: 5, # default connection setup timeout
-      inactivity_timeout: conf.wait_duration + 1, # default connection inactivity (post-setup) timeout
-    }
-    connection = EventMachine::HttpRequest.new(conf.base_url, options)
-    cb = proc do |consul_index|
-      http = connection.get(build_request(consul_index))
-      http.callback do
-        # Dirty hack, but contrary to other path, when key is not present, Consul returns 404
-        is_kv_empty = path.start_with?('/v1/kv') && http.response_header.status == 404
-        if !is_kv_empty && enforce_json_200 && http.response_header.status != 200 && http.response_header['Content-Type'] != 'application/json'
-          _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
-        else
-          n_consul_index = find_x_consul_token(http)
-          @consecutive_errors = 0
-          http_result = if is_kv_empty
-                          HttpResponse.new(http, default_value)
-                        else
-                          HttpResponse.new(http)
-                        end
-          new_content = http_result.response.freeze
-          modified = @last_result.nil? ? true : @last_result.data != new_content
-          if n_consul_index == consul_index || n_consul_index.nil?
-            retry_in = modified ? conf.missing_index_retry_time_on_diff : conf.missing_index_retry_time_on_unchanged
-            n_consul_index = consul_index
-          else
-            retry_in = modified ? conf.min_duration : conf.retry_on_non_diff
-          end
-          retry_in = 0.1 if retry_in < 0.1
-          unless @stopping
-            EventMachine.add_timer(retry_in) do
-              queue.push(n_consul_index)
-            end
-          end
-          result = ConsulResult.new(new_content, modified, http_result, n_consul_index, stats, retry_in)
-          @last_result = result
-          @ready = true
-          @s_callbacks.each { |c| c.call(result) }
-        end
-      end
-
-      http.errback do
-        unless @stopping
-          _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
-        end
-      end
-      queue.pop(&cb)
-    end
-    queue.pop(&cb)
-  end
-end