consul-templaterb 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ecbf300312175e355f65232450dc9303db8bbb1d2d2af473480a2fde3997112
-  data.tar.gz: 08abe3b2ef2e812810e88d29db8a44c1191af395a3e5b94c91a9d2a54b0c5906
+  metadata.gz: 875f33bec50159a51a9fb9e59b07fbe37a78a0587cb3a496cc42a1806b5c8c9d
+  data.tar.gz: 08f091608892658b875d716c6935c9fe24f19b72b0c37dcde07eee4bcbdea838
 SHA512:
-  metadata.gz: 8042decc8b615eff3080f06330347f6596f7fbdb5b8e732eaf9afc00b79a7b2ddffb161d3c9c63d986f7eed2d54a467cca0d39d9ee247af65edad71453fcbaf5
-  data.tar.gz: b88e62b6884d6dfc3e0dad669943124e6ec0c28af3da482c6a9614f2b23325bd9d57fe9afeb0ebd08c57808a62003ab3a4dd4cb4da1b008043ec865bf49e054f
+  metadata.gz: 481afae4f8b4df204897584ebc0ad4559c54ea5cbdd4a693977c33a573acc67e3e726df8b55a6ed3d2506b01c82f3092d98d27208e0de51c1c2367df01447cb3
+  data.tar.gz: 361f0786469ff9c305f047824b42a8c801109ffe6c7283e0a4325c2f89c3d8dcfa989b3a52bbef982e8cc78bbad17505b2c5c2b57223b67ba0343c0c9722143a

data/CHANGELOG.md

@@ -2,8 +2,20 @@
 
 ## (UNRELEASED)
 
+## 1.3.0 (June 7, 2018)
+
 IMPROVEMENTS:
 
+ * samples/consul-ui/ now supports keys as well as nodes thans to [@geobeau](https://github.com/geobeau)
+
+NEW FEATURES
+
+ * EXPERIMENTAL Vault support thanks to [@uepock](https://github.com/uepoch)
+
+BUG FIXES:
+
+ * Properly shutdown connections when receiving Posix signal to kill app
+
 ## 1.2.1 (May 28, 2018)
 
 BUG FIXES:

data/README.md

@@ -276,7 +276,7 @@ Please consult [CHANGELOG.md](CHANGELOG.md) for fixed bugs.
 
 ## TODO
 
-* [ ] Hashi's Vault support
+* [x] Hashi's Vault support (EXPERIMENTAL)
 * [ ] Implement automatic dynamic rate limit
 * [ ] More samples: apache, nginx, full website displaying consul information...
 * [ ] Optimize rendering speed at start-up: an iteration is done very second by default, but it would be possible to speed

data/bin/consul-templaterb

@@ -17,6 +17,24 @@ def compute_default_output(source)
 end
 
 options = {
+  vault: {
+    debug: {
+      network: false
+    },
+    base_url: ENV['VAULT_ADDR'] || 'http://localhost:8200',
+    token: ENV['VAULT_TOKEN'] || nil,
+    token_renew: true,
+    retry_duration: 10,
+    lease_duration_factor: 0.5, # The time it waits before actualizing data based on the lease time: 2h lease * 0.5 = Fetch data every 1h
+    min_duration: 60,
+    max_retry_duration: 86_400, # Much higher than consul's because some dynamic secrets could be used for a day.
+    paths: {
+      '/v1/sys/mounts' => {
+        max_retry_duration: 600,
+        min_duration: 600
+      }
+    }
+  },
   consul: {
     debug: {
       network: false
@@ -71,10 +89,27 @@ optparse = OptionParser.new do |opts|
     options[:consul][:base_url] = consul_url
   end
 
-  opts.on('-t', '--consul-token=<token>', String, 'Use a token to connect to Consul') do |consul_token|
+  opts.on('--consul-token=<token>', String, 'Use a token to connect to Consul') do |consul_token|
     options[:consul][:token] = consul_token
   end
 
+  opts.on('-V', '--vault-addr=<address>', String, 'Address of Vault, eg: http://localhost:8200') do |vault_url|
+    options[:vault][:base_url] = vault_url
+  end
+
+  opts.on('-T', '--vault-token=<token>', String, 'Token used to authenticate against vault.') do |vault_token|
+    options[:vault][:token] = vault_token
+  end
+
+  options[:vault][:token_renew] = true
+  opts.on('--[no-]vault-renew', 'Control auto-renewal of the Vault token. Default: activated') do |vault_renew|
+    options[:vault][:token_renew] = vault_renew
+  end
+
+  opts.on('--vault-lease-duration-factor=<factor>', Float, 'Wait at least <factor> * lease time before updating a Vault secret. Default: 0.5') do |factor|
+    options[:vault][:lease_duration_factor] = factor
+  end
+
   opts.on('-w', '--wait=<min_duration>', Float, 'Wait at least n seconds before each template generation') do |min_duration|
     options[:consul][:min_duration] = min_duration
   end
@@ -213,8 +248,10 @@ def find_max_descriptors(max_descripors)
   max_size
 end
 
-unless options[:consul][:base_url].start_with? 'http'
-  options[:consul][:base_url] = "http://#{options[:consul][:base_url]}"
+%i[consul vault].each do |type|
+  unless options[type][:base_url].start_with? 'http', 'https'
+    options[type][:base_url] = "http://#{options[type][:base_url]}"
+  end
 end
 
 # Since we might be using a lots of descriptors, document this
@@ -227,7 +264,8 @@ STDERR.puts "Max number of descriptors set to #{new_size}" if options[:consul][:
 EM.epoll
 
 consul_conf = Consul::Async::ConsulConfiguration.new(options[:consul])
-template_manager = Consul::Async::ConsulEndPointsManager.new(consul_conf)
+vault_conf = Consul::Async::VaultConfiguration.new(options[:vault])
+template_manager = Consul::Async::EndPointsManager.new(consul_conf, vault_conf)
 
 ARGV.each do |tpl|
   dest = compute_default_output(tpl)
@@ -239,6 +277,7 @@ end
 %w[INT PIPE TERM].each do |sig|
   Signal.trap(sig) do
     STDERR.puts "[KILL] received #{sig}, stopping myself"
+    template_manager.terminate
     kill_program
   end
 end

data/lib/consul/async/consul_endpoint.rb

@@ -1,4 +1,5 @@
 require 'consul/async/utilities'
+require 'consul/async/stats'
 require 'em-http'
 require 'thread'
 require 'json'
@@ -56,38 +57,6 @@ module Consul
                                 paths: @paths)
       end
     end
-    class ConsulEndPointStats
-      attr_reader :successes, :errors, :start, :body_bytes
-      def initialize
-        @start = Time.now.utc
-        @successes = 0
-        @errors = 0
-        @body_bytes = 0
-      end
-
-      def on_reponse(res)
-        @successes += 1
-        @body_bytes = body_bytes + res.http.response.bytesize
-      end
-
-      def on_error(_http)
-        @errors += 1
-      end
-
-      def bytes_per_sec
-        diff = (Time.now.utc - start)
-        diff = 1 if diff < 1
-        (body_bytes / diff).round(0)
-      end
-
-      def bytes_per_sec_human
-        "#{Utilities.bytes_to_h(bytes_per_sec)}/s"
-      end
-
-      def body_bytes_human
-        Utilities.bytes_to_h(body_bytes)
-      end
-    end
     class ConsulResult
       attr_reader :data, :http, :x_consul_index, :last_update, :stats, :retry_in
       def initialize(data, modified, http, x_consul_index, stats, retry_in)
@@ -106,6 +75,7 @@ module Consul
 
       def mutate(new_data)
         @data = new_data.dup
+        @data_json = nil
       end
 
       def json
@@ -146,9 +116,9 @@ module Consul
         @consecutive_errors = 0
         @query_params = query_params
         @stopping = false
-        @stats = ConsulEndPointStats.new
+        @stats = EndPointStats.new
         @last_result = ConsulResult.new(default_value, false, HttpResponse.new(nil), 0, stats, 1)
-        on_response { |result| @stats.on_reponse result }
+        on_response { |result| @stats.on_response result }
         on_error { |http| @stats.on_error http }
         _enable_network_debug if conf.debug && conf.debug[:network]
         fetch

data/lib/consul/async/consul_template.rb

@@ -19,10 +19,11 @@ module Consul
       end
     end
 
-    class ConsulEndPointsManager
-      attr_reader :conf, :net_info, :start_time
-      def initialize(consul_configuration)
-        @conf = consul_configuration
+    class EndPointsManager
+      attr_reader :consul_conf, :vault_conf, :net_info, :start_time
+      def initialize(consul_configuration, vault_configuration)
+        @consul_conf = consul_configuration
+        @vault_conf = vault_configuration
         @endpoints = {}
         @iteration = 1
         @start_time = Time.now.utc
@@ -35,6 +36,11 @@ module Consul
           current_erb_path: nil,
           params: {}
         }
+
+        unless @vault_conf.token.nil? || !@vault_conf.token_renew
+          #Setup token renewal
+          vault_setup_token_renew
+        end
       end
 
       # https://www.consul.io/api/health.html#list-nodes-for-service
@@ -45,7 +51,7 @@ module Consul
         query_params[:dc] = dc if dc
         query_params[:passing] = passing if passing
         query_params[:tag] = tag if tag
-        create_if_missing(path, query_params) { ConsulTemplateService.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params) { ConsulTemplateService.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
       end
 
       # https://www.consul.io/api/health.html#list-checks-for-service
@@ -55,7 +61,7 @@ module Consul
         query_params = {}
         query_params[:dc] = dc if dc
         query_params[:passing] = passing if passing
-        create_if_missing(path, query_params) { ConsulTemplateChecks.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params) { ConsulTemplateChecks.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
       end
 
       # https://www.consul.io/api/catalog.html#list-nodes
@@ -63,7 +69,7 @@ module Consul
         path = '/v1/catalog/nodes'
         query_params = {}
         query_params[:dc] = dc if dc
-        create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
       end
 
       # https://www.consul.io/api/catalog.html#list-services-for-node
@@ -71,7 +77,7 @@ module Consul
         path = "/v1/catalog/node/#{name_or_id}"
         query_params = {}
         query_params[:dc] = dc if dc
-        create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(conf, path, true, query_params, '{}')) }
+        create_if_missing(path, query_params) { ConsulTemplateNodes.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '{}')) }
       end
 
       # https://www.consul.io/api/agent.html#read-configuration
@@ -79,7 +85,7 @@ module Consul
         path = '/v1/agent/self'
         query_params = {}
         default_value = '{"Config":{}, "Coord":{}, "Member":{}, "Meta":{}, "Stats":{}}'
-        create_if_missing(path, query_params) { ConsulAgentSelf.new(ConsulEndpoint.new(conf, path, true, query_params, default_value)) }
+        create_if_missing(path, query_params) { ConsulAgentSelf.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value)) }
       end
 
       # https://www.consul.io/api/agent.html#view-metrics
@@ -87,7 +93,7 @@ module Consul
         path = '/v1/agent/metrics'
         query_params = {}
         default_value = '{"Gauges":[], "Points":[], "Member":{}, "Counters":[], "Samples":{}}'
-        create_if_missing(path, query_params) { ConsulAgentMetrics.new(ConsulEndpoint.new(conf, path, true, query_params, default_value)) }
+        create_if_missing(path, query_params) { ConsulAgentMetrics.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value)) }
       end
 
       # Return a param of template
@@ -105,14 +111,14 @@ module Consul
         query_params[:dc] = dc if dc
         # Tag filtering is performed on client side
         query_params[:tag] = tag if tag
-        create_if_missing(path, query_params) { ConsulTemplateServices.new(ConsulEndpoint.new(conf, path, true, query_params, '{}')) }
+        create_if_missing(path, query_params) { ConsulTemplateServices.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '{}')) }
       end
 
       # https://www.consul.io/api/catalog.html#list-datacenters
       def datacenters
         path = '/v1/catalog/datacenters'
         query_params = {}
-        create_if_missing(path, query_params) { ConsulTemplateDatacenters.new(ConsulEndpoint.new(conf, path, true, query_params, '[]')) }
+        create_if_missing(path, query_params) { ConsulTemplateDatacenters.new(ConsulEndpoint.new(consul_conf, path, true, query_params, '[]')) }
       end
 
       # https://www.consul.io/api/kv.html#read-key
@@ -123,7 +129,23 @@ module Consul
         query_params[:recurse] = recurse if recurse
         query_params[:keys] = keys if keys
         default_value = '[]'
-        create_if_missing(path, query_params) { ConsulTemplateKV.new(ConsulEndpoint.new(conf, path, true, query_params, default_value), name) }
+        create_if_missing(path, query_params) { ConsulTemplateKV.new(ConsulEndpoint.new(consul_conf, path, true, query_params, default_value), name) }
+      end
+
+      def secrets(path = '')
+        raise "You need to provide a vault token to use 'secret' keyword" if vault_conf.token.nil?
+        path = "/v1/#{path}".gsub(/\/{2,}/, '/')
+        query_params = {list: "true"}
+        create_if_missing(path, query_params) { ConsulTemplateVaultSecretList.new(VaultEndpoint.new(vault_conf, path, 'GET',true, query_params,JSON.generate(data: {keys: []}))) }
+      end
+
+      def secret(path = '', post_data = nil )
+        puts post_data
+        raise "You need to provide a vault token to use 'secrets' keyword" if vault_conf.token.nil?
+        path = "/v1/#{path}"
+        query_params = {}
+        method = post_data ? "POST" : "GET"
+        create_if_missing(path, query_params) { ConsulTemplateVaultSecret.new(VaultEndpoint.new(vault_conf, path, method, true, query_params, JSON.generate(data: {}))) }
       end
 
       # render a relative file with the given params accessible from template
@@ -204,6 +226,12 @@ module Consul
         @endpoints = {}
       end
 
+      def vault_setup_token_renew
+        path = 'v1/auth/token/renew-self'
+        STDERR.print "[INFO] Setting up vault token renewal"
+        VaultEndpoint.new(vault_conf, path, :POST, {}, {})
+      end
+
       def create_if_missing(path, query_params)
         fqdn = path.dup
         query_params.each_pair do |k, v|
@@ -384,5 +412,18 @@ module Consul
         end
       end
     end
+
+    class ConsulTemplateVaultSecret < ConsulTemplateAbstractMap
+      def initialize(vault_endpoint)
+        super(vault_endpoint)
+      end
+    end
+    class ConsulTemplateVaultSecretList < ConsulTemplateAbstractArray
+      def parse_result(res)
+        return res if res.data.nil?
+        res.mutate(JSON.generate(res.json['data']['keys']))
+        res
+      end
+    end
   end
 end

data/lib/consul/async/consul_template_engine.rb

@@ -1,5 +1,6 @@
 require 'consul/async/utilities'
 require 'consul/async/consul_endpoint'
+require 'consul/async/vault_endpoint'
 require 'consul/async/consul_template'
 require 'consul/async/consul_template_render'
 require 'em-http'
@@ -55,7 +56,7 @@ module Consul
               template_manager.terminate
               EventMachine.stop
             rescue StandardError => e
-              STDERR.puts "[ERROR] Fatal error occured: #{e.inspect} - #{e.backtrace}"
+              STDERR.puts "[ERROR] Fatal error occured: #{e.inspect} - #{e.backtrace.join("\n\t")}"
               template_manager.terminate
               EventMachine.stop
             end

data/lib/consul/async/endpoint.rb

@@ -0,0 +1,137 @@
+require 'consul/async/utilities'
+
+class Endpoint
+  attr_reader :conf, :path, :x_consul_index, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
+  def initialize(conf, path, enforce_json_200 = true, query_params = {}, default_value = '[]')
+    @conf = conf.create(path)
+    @default_value = default_value
+    @path = path
+    @queue = EM::Queue.new
+    @s_callbacks = []
+    @e_callbacks = []
+    @enforce_json_200 = enforce_json_200
+    @start_time = Time.now.utc
+    @consecutive_errors = 0
+    @query_params = query_params
+    @stopping = false
+    @stats = EndPointStats.new
+    @last_result = ConsulResult.new(default_value, false, HttpResponse.new(nil), 0, stats, 1)
+    on_response { |result| @stats.on_reponse result }
+    on_error { |http| @stats.on_error http }
+    _enable_network_debug if conf.debug && conf.debug[:network]
+    fetch
+    queue << 0
+  end
+
+  def _enable_network_debug
+    on_response do |result|
+      state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
+      stats = result.stats
+      STDERR.puts "[DEBUG]#{state}#{result.modified? ? '[MODIFIED]' : '[NO DIFF]'}" \
+          "[s:#{stats.successes},err:#{stats.errors}]" \
+          "[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
+          " #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
+    end
+    on_error { |http| STDERR.puts "[ERROR]: #{path}: #{http.error}" }
+  end
+
+  def on_response(&block)
+    @s_callbacks << block
+  end
+
+  def on_error(&block)
+    @e_callbacks << block
+  end
+
+  def ready?
+    @ready
+  end
+
+  def terminate
+    @stopping = true
+  end
+
+  private
+
+  def build_request(headers = {}, query_params = {})
+    req = {
+        head: headers,
+        path: path,
+        query: query_params,
+        keepalive: true,
+        callback: method(:on_response)
+    }
+    @query_params.each_pair do |k, v|
+      req[:query][k] = v
+    end
+    req
+  end
+
+  def log(level, message)
+
+  end
+
+  def _handle_error(http, consul_index)
+    retry_in = [600, conf.retry_duration + 2**@consecutive_errors].min
+    STDERR.puts "[ERROR][#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+    @consecutive_errors += 1
+    http_result = HttpResponse.new(http)
+    EventMachine.add_timer(retry_in) do
+      yield
+      queue.push(consul_index)
+    end
+    @e_callbacks.each { |c| c.call(http_result) }
+  end
+
+  def fetch
+    options = {
+        connect_timeout: 5, # default connection setup timeout
+        inactivity_timeout: conf.wait_duration + 1, # default connection inactivity (post-setup) timeout
+    }
+    connection = EventMachine::HttpRequest.new(conf.base_url, options)
+    cb = proc do |consul_index|
+      http = connection.get(build_request(consul_index))
+      http.callback do
+        # Dirty hack, but contrary to other path, when key is not present, Consul returns 404
+        is_kv_empty = path.start_with?('/v1/kv') && http.response_header.status == 404
+        if !is_kv_empty && enforce_json_200 && http.response_header.status != 200 && http.response_header['Content-Type'] != 'application/json'
+          _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
+        else
+          n_consul_index = find_x_consul_token(http)
+          @consecutive_errors = 0
+          http_result = if is_kv_empty
+                          HttpResponse.new(http, default_value)
+                        else
+                          HttpResponse.new(http)
+                        end
+          new_content = http_result.response.freeze
+          modified = @last_result.nil? ? true : @last_result.data != new_content
+          if n_consul_index == consul_index || n_consul_index.nil?
+            retry_in = modified ? conf.missing_index_retry_time_on_diff : conf.missing_index_retry_time_on_unchanged
+            n_consul_index = consul_index
+          else
+            retry_in = modified ? conf.min_duration : conf.retry_on_non_diff
+          end
+          retry_in = 0.1 if retry_in < 0.1
+          unless @stopping
+            EventMachine.add_timer(retry_in) do
+              queue.push(n_consul_index)
+            end
+          end
+          result = ConsulResult.new(new_content, modified, http_result, n_consul_index, stats, retry_in)
+          @last_result = result
+          @ready = true
+          @s_callbacks.each { |c| c.call(result) }
+        end
+      end
+
+      http.errback do
+        unless @stopping
+          _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
+        end
+      end
+      queue.pop(&cb)
+    end
+    queue.pop(&cb)
+  end
+end