consul-templaterb 1.8.8 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c516359f510134a8471b94ba58a0a762d38b45cd7058d3b8d45ff67837047139
-  data.tar.gz: b24b6f18c7d33561bb82b9df4ca6422052762e08b6ffb21fc76b82b55659cf3d
+  metadata.gz: 8efd0da9a415f6e58d2da8e712ea082efd08b51631b38c31db287cbbdf3a3bbe
+  data.tar.gz: 9e1fdd39625a7213dfb431dc89660fdeb0da382c4459766f5a2d34803313b9fd
 SHA512:
-  metadata.gz: 281fa6348ebac8e6b3c143979a9b21dd182b43d3933ac6ed6f177cb4e3ab86f1db290406de8150abf77c580e17cbaaadac01673efa169c675a83e426fab165ab
-  data.tar.gz: 86421aa3c59e9fc1073c46346cf61dde419ed389f40b549643d1f078723d3a587a4735d6c9882c44e33bd0d0958f20314572d5ead61988f0c9ac9c53b6b88b29
+  metadata.gz: 5f7c6e9187971dda5aadab619fe58facf7c76fe6cf7dde9a5d9c3708d525e8a3fc4bd8b8b558ccfcd8a3b2fafaf923acb7f5f673e9428635ef9b391ec50f6f79
+  data.tar.gz: 8031be2c311b9e1f12127a5e15922d13a12f86c86e2e0ec1ea92c1282c7f382fdab2f851c6bbb43b6d63dba91a8eb731df7825d93aaae7164407db1b1c48ebcf

data/.rubocop.yml

@@ -16,7 +16,7 @@ Metrics/ClassLength:
   Max: 225
 
 Metrics/CyclomaticComplexity:
-  Max: 15
+  Max: 20
 
 Metrics/LineLength:
   Max: 175
@@ -28,7 +28,7 @@ Metrics/ParameterLists:
   Max: 12
 
 Metrics/PerceivedComplexity:
-  Max: 18
+  Max: 20
 
 Style/Documentation:
   Enabled: false

data/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## (UNRELEASED)
 
+## 1.9.0
+
+OPTIMIZATIONS:
+ * Better network issue because of X-Consul-Index parsing bug
+
+NEW FEATURES:
+ * value.endpoint.x_consul_index now works as expected
+
+IMPROVEMENTS:
+ * timeline now works well behing load balancer without stickyness
+ * Now hide service column when filtering on a service
+ * apply row limits immediately
+ * auto-refresh duration is now configurable by `TIMELINE_REFRESH_MS`
+   environment variable
+
 ## 1.8.6 / 1.8.7 / 1.8.8 (December 20, 2018)
 
 OPTIMIZATIONS:

data/lib/consul/async/consul_endpoint.rb

@@ -181,8 +181,8 @@ module Consul
         res
       end
 
-      def find_x_consul_token(http)
-        http.response_header['X_CONSUL_INDEX']
+      def find_x_consul_index(http)
+        http.response_header['X-Consul-Index']
       end
 
       def _handle_error(http, consul_index)
@@ -211,7 +211,8 @@ module Consul
             if !is_kv_empty && enforce_json_200 && http.response_header.status != 200 && http.response_header['Content-Type'] != 'application/json'
               _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
             else
-              n_consul_index = find_x_consul_token(http)
+              n_consul_index = find_x_consul_index(http)
+              @x_consul_index = n_consul_index.to_i if n_consul_index
               @consecutive_errors = 0
               http_result = if is_kv_empty
                               HttpResponse.new(http, default_value)

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.8.8'.freeze
+    VERSION = '1.9.0'.freeze
   end
 end

data/samples/consul-ui/consul-timeline-ui.html.erb

@@ -1,6 +1,6 @@
 <% datasource = ENV['TIMELINE_DATASOURCE'] || 'timeline.json'
    # Time to wait before reloading configuration again in seconds (0 = never)
-   refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Services Timeline') %>
+   refresh = ENV['TIMELINE_REFRESH_MS'] || '10000' %><%= render_file('common/header.html.erb', title: 'Services Timeline') %>
     <div class="main">
       <div class="row mx-0">
         <div id="filter-menu" class="col-2 col-m-3 px-4 pt-4">
@@ -27,7 +27,10 @@
                     <span id="numRowsDisplayed" class="lookup badge badge-pill badge-primary">loading&hellip;</span>
                   </span>
                </div>
-                <input id="maxRows" title="max rows to display" placeholder="max rows" style="width: 5em !important;" type="number" min="100" max="<%= (ENV['CONSUL_TIMELINE_BUFFER'] || 10000) %>" step="25" value="5000" class="form-control"/>
+               <button class="btn btn-primary btn-sm" id="firstPage" title="Newest entries" disabled="disabled" onclick="serviceTimeline.firstPage()">⇤</button>
+               <button class="btn btn-primary btn-sm" id="prevPage" title="Newer entries" disabled="disabled" onclick="serviceTimeline.prevPage()">←</button>
+               <input id="maxRows" title="max rows to display" placeholder="max rows" style="width: 5.5em" type="number" min="25" max="<%= (ENV['CONSUL_TIMELINE_BUFFER'] || 10000) %>" step="25" value="250" class="form-control" onchange="serviceTimeline.displayEvents(false)"/>
+               <button class="btn btn-primary btn-sm" id="nextPage" disabled="disabled" onclick="serviceTimeline.nextPage()">→</button>
               </div>
             </div>
           </div>

data/samples/consul-ui/css/style.css

@@ -22,6 +22,7 @@
 }
 .checkName {
   padding-left: 5px;
+  max-width: 12em;
   overflow: hidden;
   white-space: nowrap;
   text-overflow: ellipsis;
@@ -93,11 +94,11 @@ td.serviceName {
   border-left: 0px;
   border-right: 0px;
 }
-.service-status {
-  width: 10em;
+td.evt-service-status {
+  min-width: 10em;
 }
-.all-instances {
-  width: 14em;
+td.all-instances {
+  min-width: 10em;
   text-align: right;
 }
 #node-statuses span {
@@ -221,4 +222,4 @@ pre, code {
 }
 .connect-disabled{
   display: none;
-}
+}

data/samples/consul-ui/js/timeline.js

@@ -17,13 +17,17 @@ class ServiceTimeline {
       this.lastEntryLoaded = null;
       this.presentServices = {}
       this.currentlyUpdating = false;
+      this.start = null;
+      this.next = null;
+      this.prev = null;
       var sT = this;
     }
 
     fetchRessource(firstReload) {
-        $.ajax({url: this.ressourceURL, cache: false, dataType: "json", sourceObject: this,
+        $.ajax({url: this.ressourceURL, cache: true, dataType: "json", sourceObject: this,
                 success: function(result){
                            serviceTimeline.initRessource(result, firstReload);
+                           serviceTimeline.currentlyUpdating = false;
                 },
                 error: function(err) {
                            console.log("Error updating data", err);
@@ -32,7 +36,69 @@ class ServiceTimeline {
          });
     }
 
+    nextPage() {
+        this.start = this.next;
+        console.log("nextPage() at ", this.start);
+        this.doFilter();
+    }
+
+    prevPage() {
+        this.start = this.prev;
+        console.log("prevPage() at ", this.start);
+        this.doFilter();
+    }
+
+    firstPage() {
+        this.start = null;
+        this.doFilter();
+    }
+
     initRessource(data, firstReload) {
+        if (!data || data.length < 1) {
+            console.log("No new data to load ", data);
+            return;
+        }
+        if (this.data != null) {
+            var previousCount = this.data.length;
+            if (previousCount > 0) {
+                var previousMaxIndex = indexOfTimelineEvent(this.data[previousCount - 1]);
+                var newIndex = indexOfTimelineEvent(data[data.length - 1]);
+                if (newIndex < previousMaxIndex) {
+                    console.log("New index " + newIndex + " is lower than previous one", previousMaxIndex, " no need to reload");
+                    return;
+                } else if (newIndex == previousMaxIndex) {
+                    if (this.data.length >= data.length) {
+                        //console.log("new := (idx=", newIndex, ",len=", data.length, ") old := (idx:=", previousMaxIndex, this.data.length, "), no need to reload");
+                        return;
+                    }
+                } else {
+                    // Lets merge old data with new one
+                    if (this.data.length > data.length) {
+                        var diff = this.data.length - data.length;
+                        var firstNewItem = indexOfTimelineEvent(data[0]);
+                        var maxItemsInMemory = 50000;
+                        if (maxItemsInMemory > data.length) {
+                            console.log("Looking for matches with ", firstNewItem);
+                            for (var i = 0 ; i < this.data.length; i++){
+                                var oldVal = indexOfTimelineEvent(this.data[i]);
+                                if (oldVal == firstNewItem) {
+                                    // We found the common start
+                                    var maxItemsToAdd = maxItemsInMemory - data.length;
+                                    var start = 0;
+                                    if (i > maxItemsToAdd) {
+                                        start = i - maxItemsToAdd;
+                                    }
+                                    console.log("Did preprend ", (i - start), "items to data");
+                                    data = this.data.slice(start, i).concat(data);
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        console.log("Loading", data.length, "items...");
         this.data = data;
         this.reloadTimeline(firstReload);
     }
@@ -132,7 +198,8 @@ class ServiceTimeline {
                 if (!found) {
                     sT.selectService($('#anyService', false));
                 }
-                setInterval(sT.reloadDataFromJSON, 10000);
+                console.log("Starting autorefresh every ", serviceTimeline.refresh, "ms.")
+                setInterval(sT.reloadDataFromJSON, serviceTimeline.refresh);
             }, 150);
         }
     }
@@ -232,13 +299,16 @@ class ServiceTimeline {
         var filterValue = $('#instance-filter')[0].value;
         var serviceName = serviceTimeline.serviceInstanceFilter;
         var serviceEvaluator = function(){return true};
+        var stylesheetText = '';
         if (serviceName != '' && serviceName != 'All') {
-            serviceEvaluator = function(e){ return e.service === serviceName }
+           stylesheetText = '.serviceCol { display: none; }';
+           serviceEvaluator = function(e){ return e.service === serviceName }
         }
+        // Show / hide .serviceCol content
+        document.getElementById('serviceCol').textContent = stylesheetText;
         var maxRows = document.getElementById("maxRows").value;
         //$("#service-title").html(service['name']);
         var tableBody = $('#all-events > tbody');
-        var startIndex = 0;
         var newDoc = document.createDocumentFragment();
         var frag = document.createElement('tbody');
         newDoc.appendChild(frag);
@@ -289,12 +359,56 @@ class ServiceTimeline {
                 return false;
             }
         }
-        for (var i = this.data.length - 1 ; i >= 0 && count <= maxRows; i--) {
+        var startIdx = this.start;
+        if (startIdx == null) {
+            startIdx = (this.data[this.data.length - 1].idx);
+        } else {
+            //console.log("startIdx := ", startIdx);
+        }
+        var previousElements = [];
+        document.getElementById('prevPage').setAttribute('disabled', 'disabled');
+        this.next = null;
+        document.getElementById('firstPage').setAttribute('disabled', 'disabled');
+        this.prev = null;
+        document.getElementById('nextPage').setAttribute('disabled', 'disabled');
+        var totalSkipped = 0;
+        for (var i = this.data.length - 1 ; i >= 0; i--) {
+            if (count >= maxRows) {
+                this.next = this.data[i].idx;
+                //console.log("next is ", this.data[i].ts, ' at ', this.next);
+                var nextTs = this.data[i].ts;
+                setTimeout(function(){
+                  nextPage = document.getElementById('nextPage');
+                  nextPage.removeAttribute('disabled');
+                  nextPage.setAttribute('title', 'Older entries at ' + nextTs);
+                }, 1);
+                break;
+            }
             var e = this.data[i];
             if (!serviceEvaluator(e)) {
                 continue;
             }
+            if (startIdx < e.idx) {
+                totalSkipped++;
+                previousElements.push(e);
+                var previousCounter = previousElements.length;
+                if (previousCounter == 1) {
+                    setTimeout(function(){
+                      document.getElementById('prevPage').removeAttribute('disabled');
+                      document.getElementById('firstPage').removeAttribute('disabled');
+                    }, 1);
+                } else if (previousCounter >= maxRows){
+                    previousElements.shift();
+                }
+                continue;
+            }
             count++;
+            if (count == 1) {
+                if (previousElements.length > 0) {
+                    //console.log("Current is ", e.ts, ' aka  ',  e.idx, ", Had ", previousElements.length, 'elements before, skipped := ', totalSkipped, "skipped elements", previousElements);
+                    this.prev = previousElements.shift().idx;
+                }
+            }
             var row = document.createElement('tr');
             row.setAttribute("class", 'srv-' + e.service);
             var timestamp;
@@ -312,7 +426,7 @@ class ServiceTimeline {
             {
               var timeElement = this.appChild('time', document.createTextNode(timestamp));
               timeElement.setAttribute('datetime', fullTimestamp);
-              timeElement.setAttribute('title', fullTimestamp);
+              timeElement.setAttribute('title', fullTimestamp + '\nX-Consul-Index: ' +e.idx);
               this.buildCell(row, 'td', 'ts', timeElement);
             }
             this.buildCell(row, 'td', 'lookup serviceName serviceCol', document.createTextNode(e.service));
@@ -352,10 +466,10 @@ class ServiceTimeline {
             }
             {
                 if (e.new_state == e.old_state) {
-                    var td = this.buildCell(row, 'td', 'service-status', this.createBadge(e['new_state']));
+                    var td = this.buildCell(row, 'td', 'evt-service-status', this.createBadge(e['new_state']));
                 } else {
                     var old = e.old_state == null ? this.createBadge('new', 'primary') : this.createBadge(e.old_state);
-                    var td = this.buildCell(row, 'td', 'service-status', old);
+                    var td = this.buildCell(row, 'td', 'evt-service-status', old);
                     td.appendChild(document.createTextNode('→'));
                     var newState = (e.new_state == null) ? this.createBadge('removed', 'dark') : this.createBadge(e.new_state);
                     td.appendChild(newState);
@@ -375,25 +489,28 @@ class ServiceTimeline {
                     allInstances.appendChild(elem);
                 }
                 var elem = this.createBadge(nCritical);
-                var percent = Math.round(nSuccess * 100 / e['stats']['total']);
-                var clazz = 'secondary'
-                if (percent > 90) {
-                    clazz = 'success';
-                } else if (percent < 20) {
-                    clazz = 'danger';
-                } else if (percent < 50) {
-                    clazz = 'warning';
+                if (e['stats']['total'] > 0) {
+                    var percent = Math.round(nSuccess * 100 / e['stats']['total']);
+                    var clazz = 'secondary'
+                    if (percent > 90) {
+                        clazz = 'success';
+                    } else if (percent < 20) {
+                        clazz = 'danger';
+                    } else if (percent < 50) {
+                        clazz = 'warning';
+                    }
+                    this.buildCell(row, 'td', 'ipercents', this.createBadge(percent + " %", clazz));
+                } else {
+                    this.buildCell(row, 'td', 'ipercents', this.createBadge('none', 'danger'));
                 }
-                this.buildCell(row, 'td', 'ipercents', this.createBadge(percent + " %", clazz));
             }
             frag.append(row);
         }
-        $('#numRowsDisplayed').html(count + ' / ' + this.data.length);
+        $('#numRowsDisplayed').html(totalSkipped + '…' + (totalSkipped + count) + ' / ' + this.data.length);
         var tbody = tableBody[0];
         tbody.parentNode.replaceChild(frag, tbody);
         if (this.data.length > 1) {
           this.lastEntryLoaded = this.data[this.data.length - 1];
-          console.log("Last entry loaded: ", indexOfTimelineEvent(this.lastEntryLoaded));
         }
     }
-}
+}

data/samples/consul-ui/timeline.json.erb

@@ -4,6 +4,8 @@ require 'set'
 services_blacklist = (ENV['EXCLUDE_SERVICES'] || 'consul-agent-http,mesos-slave,mesos-agent-watcher,mesos-exporter-slave').split(',')
 
 @current_time = Time.now.utc.iso8601
+@last_service_info = {}
+@newest_index = 0 unless @newest_index
 cur_state = services.map do |service_name, _tags|
   next if services_blacklist.include?(service_name)
 
@@ -14,6 +16,9 @@ cur_state = services.map do |service_name, _tags|
     'critical' => 0,
     'total'    => snodes.count
   }
+  cur_index = snodes.endpoint.x_consul_index.to_i
+  @newest_index = cur_index if @newest_index < cur_index
+  @last_service_info[service_name] = { 'idx' => cur_index, 'stats' => cur_stats }
   snodes.each do |snode|
     case snode.status.downcase
     when 'passing'
@@ -32,9 +37,9 @@ cur_state = services.map do |service_name, _tags|
        'address' => instance.service_address,
        'node' => instance['Node']['Node'],
        'port' => instance['Service']['Port'],
-       'idx'  => instance['Service']['ModifyIndex'],
+       'idx'  => (instance['Service']['ModifyIndex'] || cur_index).to_i,
        'status' => instance.status,
-       'stats'  => cur_stats,
+       'stats' => cur_stats,
        'checks' => instance['Checks'].map { |check| [check['CheckID'], { 'name' => check['Name'], 'status' => check['Status'], 'output' => check['Output'] }] }.to_h
      }]
   end.to_h
@@ -54,26 +59,27 @@ class RingBuffer < Array
 
   def initialize(max_size:, enum: nil)
     @max_size = max_size
+    warn "Ringbuffer initialized with #{max_size}"
     enum&.each { |e| self << e }
   end
 
   def <<(element)
-    if size < @max_size || @max_size.nil?
-      super
-    else
+    return unless element
+    if size >= @max_size
       shift
+    end
+    previous_e = last
+    after = []
+    while !previous_e.nil? && (previous_e['idx'] > element['idx'])
+      after.insert(0, pop)
       previous_e = last
-      after = []
-      while !previous_e.nil? && previous_e['idx'] > element['idx']
-        after << pop
-        previous_e = last
-      end
-      push(element)
-      push after
     end
+    push(element)
+    after.each do |x|
+      push(x)
+    end
+    self
   end
-
-  alias push <<
 end
 
 def diff(old, new_e)
@@ -92,16 +98,12 @@ def log_event(line)
   puts "#{Time.now.to_i} #{line}" if ENV['DEBUG_TIMELINE']
 end
 
-def store_event(service: service_name, instance: nil, old_state: nil, new_state: nil, instance_info: nil, checks: [])
-  STDERR.puts "empty instance_info for  #{service} ; #{instance} ; #{new_state}" unless instance_info
-  @new_events << { 'service' => service, 'instance' => instance, 'idx' => instance_info['idx'], 'old_state' => old_state, 'new_state' => new_state,
+def store_event(service: nil, instance: nil, old_state: nil, new_state: nil, instance_info: nil, checks: [])
+  @new_events << { 'service' => service, 'instance' => instance, 'old_state' => old_state, 'new_state' => new_state,
                'ts' => @current_time, 'instance_info' => instance_info }.tap do |ev|
     ev['checks'] = checks if checks
-    ev['stats'] = if instance_info
-                    instance_info['stats']
-                  else
-                    {}
-                  end
+    ev['stats'] = instance_info['stats']
+    ev['idx'] = instance_info['idx']
   end
 end
 
@@ -133,11 +135,19 @@ service_diff = diff(old_state.keys, cur_state.keys)
 service_diff.disappeared.each do |service_name|
   old_state[service_name].each do |instance_name, instance_info|
     checks = compute_checks(old_state, cur_state, service_name, instance_name)
+    the_info = {}.merge(instance_info)
+    the_info['idx'] = @newest_index
+    the_info['stats'] = {
+      'passing'  => 0,
+      'warning'  => 0,
+      'critical' => 0,
+      'total'    => 0,
+    }
     store_event(service: service_name,
                 instance: instance_name,
                 old_state: old_state[service_name][instance_name]['status'],
                 new_state: nil,
-                instance_info: instance_info,
+                instance_info: the_info,
                 checks: checks)
   end
 end
@@ -156,11 +166,14 @@ end
 
   instance_diff.disappeared.each do |instance_name|
     checks = compute_checks(old_state, cur_state, service_name, instance_name)
+    the_info = {}.merge(old_state[service_name][instance_name])
+    the_info['idx'] = @last_service_info[service_name]['idx']
+    the_info['stats'] = @last_service_info[service_name]['stats']
     store_event(service: service_name,
                 old_state: old_state[service_name][instance_name]['status'],
                 new_state: nil,
                 instance: instance_name,
-                instance_info: old_state[service_name][instance_name],
+                instance_info: the_info,
                 checks: checks)
   end
   instance_diff.appeared.each do |instance_name|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.8.8
+  version: 1.9.0
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-20 00:00:00.000000000 Z
+date: 2019-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request