consul-templaterb 1.2.1 → 1.3.0

data/lib/consul/async/stats.rb

@@ -0,0 +1,40 @@
+require 'consul/async/utilities'
+
+module Consul
+  module Async
+    class EndPointStats
+      attr_reader :successes, :errors, :start, :body_bytes
+
+      def initialize
+        @start = Time.now.utc
+        @successes = 0
+        @errors = 0
+        @body_bytes = 0
+      end
+
+      def on_response(res)
+        @successes += 1
+        @body_bytes = body_bytes + res.http.response.bytesize
+      end
+
+      def on_error(_http)
+        @errors += 1
+      end
+
+      def bytes_per_sec
+        diff = (Time.now.utc - start)
+        diff = 1 if diff < 1
+        (body_bytes / diff).round(0)
+      end
+
+      def bytes_per_sec_human
+        "#{Utilities.bytes_to_h(bytes_per_sec)}/s"
+      end
+
+      def body_bytes_human
+        Utilities.bytes_to_h(body_bytes)
+      end
+    end
+  end
+end
+

data/lib/consul/async/vault_endpoint.rb

@@ -0,0 +1,249 @@
+require 'consul/async/utilities'
+require 'consul/async/stats'
+require 'em-http'
+require 'net/http'
+require 'thread'
+require 'json'
+
+module Consul
+  module Async
+    class VaultConfiguration
+      attr_reader :base_url, :token, :token_renew, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
+                  :lease_duration_factor, :debug
+
+      def initialize(base_url: 'http://localhost:8200',
+                     debug: { network: false },
+                     token: nil,
+                     token_renew: true,
+                     retry_duration: 10,
+                     min_duration: 0.1,
+                     lease_duration_factor: 0.5,
+                     max_retry_duration: 600,
+                     paths: {})
+        @base_url = base_url
+        @token_renew = token_renew
+        @debug = debug
+        @retry_duration = retry_duration
+        @min_duration = min_duration
+        @max_retry_duration = max_retry_duration
+        @lease_duration_factor = lease_duration_factor
+        @paths = paths
+        @token = token
+
+      end
+
+      def ch(path, symbol)
+        sub = @paths[path.to_sym]
+        if sub && sub[symbol]
+          STDERR.puts "[INFO] Overriding #{symbol}:=#{sub[symbol]} for #{path}"
+          sub[symbol]
+        else
+          method(symbol).call
+        end
+      end
+
+      def create(path)
+        return self unless @paths[path.to_sym]
+        VaultConfiguration.new(base_url: ch(path, :base_url),
+                               debug: ch(path, :debug),
+                               token: ch(path, :token),
+                               retry_duration: ch(path, :retry_duration),
+                               min_duration: ch(path, :min_duration),
+                               max_retry_duration: ch(path, :max_retry_duration),
+                               lease_duration_factor: ch(path, :lease_duration_factor),
+                               paths: @paths)
+      end
+    end
+    class VaultResult
+      attr_reader :data, :http, :stats, :retry_in
+
+      def initialize(result, modified, stats, retry_in)
+        @data = result.response
+        @modified = modified
+        @http = result
+        @data_json = result.json
+        @last_update = Time.now.utc
+        @next_update = Time.now.utc + retry_in
+        @stats = stats
+        @retry_in = retry_in
+      end
+
+      def modified?
+        @modified
+      end
+
+      def mutate(new_data)
+        @data = new_data.dup
+        @data_json = nil
+      end
+
+      def [](path)
+        json[path]
+      end
+
+      def json
+        @data_json = JSON.parse(data) if @data_json.nil?
+        @data_json
+      end
+
+    end
+    class VaultHttpResponse
+      attr_reader :response_header, :response, :error, :json
+
+      def initialize(http, override_nil_response = nil)
+        if http.nil?
+          @response_header = nil
+          @response = override_nil_response
+          @error = 'Not initialized yet'
+        else
+          @response_header = http.response_header.nil? ? nil : http.response_header.dup.freeze
+          @response = http.response.nil? || http.response.empty? ? override_nil_response : http.response.dup.freeze
+          @error = http.error.nil? ? nil : http.error.dup.freeze
+        end
+        @json = JSON[response]
+      end
+    end
+
+    class VaultEndpoint
+      attr_reader :conf, :path, :http_method, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
+
+      def initialize(conf, path, http_method = 'GET', enforce_json_200 = true, query_params = {}, default_value = '{}', post_data ={})
+        @conf = conf.create(path)
+        @default_value = default_value
+        @path = path
+        @http_method = http_method
+        @queue = EM::Queue.new
+        @x_consul_index = 0
+        @s_callbacks = []
+        @e_callbacks = []
+        @enforce_json_200 = enforce_json_200
+        @start_time = Time.now.utc
+        @consecutive_errors = 0
+        @query_params = query_params
+        @post_data = post_data
+        @stopping = false
+        @stats = EndPointStats.new
+        @last_result = VaultResult.new(VaultHttpResponse.new(nil, default_value), false, stats ,1)
+        on_response { |result| @stats.on_response result }
+        on_error { |http| @stats.on_error http }
+        _enable_network_debug if conf.debug && conf.debug[:network]
+        fetch
+        queue.push 0
+      end
+
+      def _enable_network_debug
+        on_response do |result|
+          state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
+          stats = result.stats
+          STDERR.puts "[DBUG]#{state}#{result.modified? ? '[MODFIED]' : '[NO DIFF]'}" \
+          "[s:#{stats.successes},err:#{stats.errors}]" \
+          "[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
+          " #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
+        end
+        on_error { |http| STDERR.puts "[ERROR]: #{path}: #{http.error}" }
+      end
+
+      def on_response(&block)
+        @s_callbacks << block
+      end
+
+      def on_error(&block)
+        @e_callbacks << block
+      end
+
+      def ready?
+        @ready
+      end
+
+      def terminate
+        @stopping = true
+      end
+
+      private
+
+      def build_request()
+        res = {
+          head: {
+            'Accept' => 'application/json',
+            'X-Vault-Token' => conf.token
+          },
+          query: {},
+          path: path,
+          keepalive: true,
+          callback: method(:on_response)
+        }
+        # if @post_data
+        #   res[:body] = JSON.generate(@post_data)
+        # end
+        @query_params.each_pair do |k, v|
+          res[:query][k] = v
+        end
+        res
+      end
+
+      def get_lease_duration(result)
+        result.json['lease_duration'] || conf.min_duration
+      end
+
+      def _get_errors(http)
+        return [http.error] if http.error
+        unless http.json.nil?
+          return http.json['errors'] if http.json.key?('errors')
+        end
+        ['unknown error']
+      end
+
+      def _handle_error(http)
+        retry_in = [conf.max_retry_duration, conf.retry_duration + 2**@consecutive_errors].min
+        STDERR.puts "[ERROR][#{path}][#{http_method}] Code: #{http.response_header.status} #{_get_errors(http).join(' - ')} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        @consecutive_errors += 1
+        http_result = VaultHttpResponse.new(http, default_value)
+        EventMachine.add_timer(retry_in) do
+          yield
+          queue.push()
+        end
+        @e_callbacks.each { |c| c.call(http_result) }
+      end
+
+      def fetch
+        options = {
+          connect_timeout: 5, # default connection setup timeout
+          inactivity_timeout: 1, # default connection inactivity (post-setup) timeout
+        }
+        connection = EventMachine::HttpRequest.new(conf.base_url, options)
+        cb = proc do |_|
+          http = connection.send(http_method.downcase, build_request) # Under the hood: c.send('get', {stuff}) === c.get({stuff})
+          http.callback do
+            http_result = VaultHttpResponse.new(http.dup.freeze, default_value)
+            if enforce_json_200 && http.response_header.status != 200
+              _handle_error(http_result) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
+            else
+              @consecutive_errors = 0
+              modified = @last_result.nil? ? true : @last_result.data != http_result.response # Leaving it do to stats with this later
+              retry_in = get_lease_duration(http_result) * conf.lease_duration_factor
+              retry_in = [retry_in, conf.max_retry_duration].min
+              retry_in = [retry_in, conf.min_duration].max
+              result = VaultResult.new(http_result, modified, stats, retry_in)
+              unless @stopping
+                EventMachine.add_timer(retry_in) do
+                  queue.push(0)
+                end
+              end
+              @last_result = result
+              @ready = true
+              @s_callbacks.each { |c| c.call(result) }
+            end
+          end
+
+          http.errback do
+            unless @stopping
+              _handle_error(http) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
+            end
+          end
+          queue.pop(&cb)
+        end
+        queue.pop(&cb)
+      end
+    end
+  end
+end

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.2.1'.freeze
+    VERSION = '1.3.0'.freeze
   end
 end

data/samples/consul-ui/common/header.html.erb

@@ -3,13 +3,26 @@
   # CONSUL_TOOLS_SUFFIX: suffix for the address of consul tools
   # CONSUL_TOOLS_PREFIX: prefix for the address of consul tools
   # CONSUL_TOOLS: comma sperated list of consul tools
-  tools = (ENV['CONSUL_TOOLS'] || 'services').split(",")
+  tools = (ENV['CONSUL_TOOLS'] || 'services,nodes,keys').split(",")
   tools_suffix = ENV['CONSUL_TOOLS_PREFIX'] || '-ui.html'
   tools_prefix = ENV['CONSUL_TOOLS_SUFFIX'] || 'consul-'
 
   dc_suffix = ENV['CONSUL_DC_SUFFIX'] || ''
   dc_prefix = ENV['CONSUL_DC_PREFIX'] || '#'
 
+  # CUSTOM_LINKS: A comma separated list of URLs to add to the header in the format: [linkname](actuallink)
+  # Example: [Consul template erb](https://github.com/criteo/consul-templaterb),[anotherlink](#)
+  custom_links = (ENV['CUSTOM_LINKS'] || '').split(",")
+  links_to_display = {}
+  if !custom_links.empty?
+    custom_links.each do |link|
+      if /\[.*\]\(.*\)/ =~ link
+        link_data = /\[(.*)\]\((.*)\)/.match(link)
+        links_to_display[link_data[1]] = link_data[2]
+      end
+    end
+  end
+
 %><!DOCTYPE html>
 <html lang="en">
   <head>
@@ -21,6 +34,7 @@
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
     <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.11/css/all.css" integrity="sha384-p2jx59pefphTFIpeqCcISO9MdVfIm4pNnsL08A6v5vaQc4owkQqxMV8kg4Yvhaw/" crossorigin="anonymous">
     <link rel="stylesheet" href="css/style.css">
+    <link rel="stylesheet" href="vendors/highlight/atom-one-dark.css">
     <style id="css-states">
       .service-tags { display: none; }
     </style>
@@ -44,6 +58,11 @@
             <a class="nav-link" href="<%= tools_prefix + tool + tools_suffix %>"><%= tool.gsub('_', ' ').capitalize %></a>
           </li>
           <% end %>
+          <% links_to_display.each do |linkname,link| %>
+          <li class="nav-item">
+            <a class="nav-link" target="_blank" href="<%= link %>"><%= linkname %></a>
+          </li>
+          <% end %>
         </ul>
       </div>
     </nav>

data/samples/consul-ui/consul-keys-ui.html.erb

@@ -0,0 +1,39 @@
+<% datasource = ENV['SERVICE_DATASOURCE'] || 'consul_keys.json'
+   # Time to wait before reloading configuration again in seconds (0 = never)
+   refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Keys') %>
+    <div class="main">
+      <div class="row mx-0">
+        <div id="filter-menu" class="col-4 col-m-3 px-4 pt-4">
+          <div class="form-group">
+            <div class="input-group">
+            <input id="keys-filter" type="text" placeholder="filter keys by name" class="form-control" />
+              <div class="input-group-append">
+                <span class="input-group-text" id="keys-counter"></span>
+              </div>
+            </div>
+          </div>
+          <div id="keys-wrapper" >
+            <ul id="keys-list" class="list-group">
+            </ul>
+          </div>
+        </div>
+        <div class="col-8 col-m-9">
+          <h2 class="text-center" id="kv-title"></h2>
+          <div id='data-wrapper'>
+            <pre><code id="kv-data">...</code></pre>
+          </div>
+        </div>
+      </div>
+    </div>
+  <!-- Optional JavaScript -->
+  <!-- JavaScript Dependencies: jQuery, Popper.js, Bootstrap JS, Shards JS -->
+  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+  <script src="js/utils.js"></script>
+  <script src="js/keys.js"></script>
+  <script src="vendors/highlight/highlight.pack.js"></script>
+  <script type="text/javascript">
+    consulKeys = new ConsulKeys('<%= datasource %>','<%= refresh %>');
+  </script>
+  </body>
+</html>

data/samples/consul-ui/consul-nodes-ui.html.erb

@@ -0,0 +1,35 @@
+<% datasource = ENV['SERVICE_DATASOURCE'] || 'consul_nodes.json'
+   # Time to wait before reloading configuration again in seconds (0 = never)
+   refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Nodes') %>
+    <div id='nodes' class="main">
+      <div class="row mx-0">
+        <div class="col-12 col-m-12">
+          <h2 class="text-center" id="service-title"></h2>
+          <div class="row mb-2">
+            <div class="input-group float-left col-10">
+              <input id="instance-filter" type="text" placeholder="filter nodes by name, service or tags" class="form-control" />
+            </div>
+            <div id="instance-statuses" class="col-2">
+              <span id="service-status-passing"  status="passing" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-success passing service-status">0</span>
+              <span id="service-status-warning"  status="warning" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-warning warning service-status">0</span>
+              <span id="service-status-critical" status="critical" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-danger critical service-status">0</span>
+              <span id="service-status-total" status="total" onclick="consulNodes.onClickFilter(this)" class="badge mx-1 badge-dark service-status">0</span>
+            </div>
+          </div>
+          <div id="instances-wrapper">
+            <div id="instances-list" class="list-group"></div>
+          </div>
+        </div>
+      </div>
+    </div>
+  <!-- Optional JavaScript -->
+  <!-- JavaScript Dependencies: jQuery, Popper.js, Bootstrap JS, Shards JS -->
+  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+  <script src="js/utils.js"></script>
+  <script src="js/nodes.js"></script>
+  <script type="text/javascript">
+    consulNodes = new ConsulNodes('<%= datasource %>','<%= refresh %>');
+  </script>
+  </body>
+</html>

data/samples/consul-ui/consul-services-ui.html.erb

@@ -1,6 +1,6 @@
-<% datasource = ENV['SERVICE_DATASOURCE'] || 'consul-template.json'
+<% datasource = ENV['SERVICE_DATASOURCE'] || 'consul_services.json'
    # Time to wait before reloading configuration again in seconds (0 = never)
-   refresh = ENV['REFRESH'] || '600' %><%= render_file('common/header.html.erb', title: 'Services') %>
+   refresh = ENV['REFRESH'] || '3600' %><%= render_file('common/header.html.erb', title: 'Services') %>
     <div class="main">
       <div class="row mx-0">
         <div id="filter-menu" class="col-4 col-m-3 px-4 pt-4">
@@ -12,7 +12,7 @@
               </label>
             </div>
             <div class="input-group">
-            <input id="service-filter" type="text" placeholder="filter by name or tags" class="form-control" />
+            <input id="service-filter" type="text" placeholder="filter services by name or tags" class="form-control" />
               <div class="input-group-append">
                 <span class="input-group-text" id="service-counter"></span>
               </div>
@@ -25,6 +25,11 @@
         </div>
         <div class="col-8 col-m-9">
           <h2 class="text-center" id="service-title"></h2>
+          <div class="row mb-2">
+            <div class="input-group float-left col-12">
+              <input id="instance-filter" type="text" placeholder="filter nodes by name or tags" class="form-control" />
+            </div>
+          </div>
           <div class="progress">
             <div id="service-progress-passing" status="passing" onclick="consulService.onClickFilter(this)" class="progress-status progress-bar bg-success" role="progressbar" style="width: 100%">passing</div>
             <div id="service-progress-warning" status="warning" onclick="consulService.onClickFilter(this)" class="progress-status progress-bar bg-warning" role="progressbar" style="width: 0%">warning</div>
@@ -40,6 +45,7 @@
   <!-- JavaScript Dependencies: jQuery, Popper.js, Bootstrap JS, Shards JS -->
   <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
   <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+  <script src="js/utils.js"></script>
   <script src="js/service.js"></script>
   <script type="text/javascript">
     consulService = new ConsulService('<%= datasource %>','<%= refresh %>');