consul-templaterb 1.0.3

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/consul-templaterb

@@ -0,0 +1,246 @@
+#!/usr/bin/env ruby
+# This script can be launched to get a uniq id for this instance
+require 'consul/async/consul_template_engine'
+require 'consul/async/process_handler'
+require 'consul/async/version'
+require 'optparse'
+require 'optparse/uri'
+
+def usage_text
+  "USAGE: #{__FILE__} [[options]]"
+end
+
+def compute_default_output(source)
+  dest = source.gsub(/\.erb$/, '')
+  raise "Source and destination cannot be the same in #{source}" if source == dest || dest.empty?
+  dest
+end
+
+options = {
+  consul: {
+    debug: {
+      network: false
+    },
+    base_url: ENV['CONSUL_HTTP_ADDR'] || 'http://locahost:8500',
+    token: nil,
+    retry_duration: 10,      # On error, retry after n seconds
+    min_duration: 5,         # On sucess and when differences are found
+    retry_on_non_diff: 3,    # On success but when there are not differences
+    wait_duration: 600,      # Delay to block in Consul
+    max_retry_duration: 600, # On consecutive errors, delay will increase, max value
+    missing_index_retry_time_on_diff: 15, # On endpoints without X-Consul-Index => next request
+    missing_index_retry_time_on_unchanged: 60, # Endpoints with X-Consul index and no diff
+    paths: {
+      '/v1/catalog/services': {
+        min_duration: 30,    # Since services change a lot, refresh services every 30 seconds
+      },
+      '/v1/catalog/nodes': {
+        min_duration: 30,    # Do not wake up before 30 seconds when node appear/disappear
+      },
+      '/v1/catalog/datacenters': {
+        min_duration: 60,    # Datacenters are not added every minute, right?
+      },
+      '/v1/agent/metrics': {
+        min_duration: 60,    # Refresh metrics only minute max
+      },
+      '/v1/agent/self': {
+        min_duration: 60,    # Refresh self info every minute max
+      }
+    }
+  }
+}
+consul_engine = Consul::Async::ConsulTemplateEngine.new
+@programs = {}
+cur_sig_reload = 'HUP'.freeze
+cur_sig_term = 'TERM'.freeze
+
+optparse = OptionParser.new do |opts|
+  opts.banner = usage_text
+
+  opts.on('-h', '--help', 'Show help') do
+    STDERR.puts opts
+    exit 0
+  end
+
+  opts.on('-v', '--version', 'Show Version') do
+    STDERR.puts Consul::Async::VERSION
+    exit 0
+  end
+
+  opts.on('-c', '--consul-addr=<address>', String, 'Address of Consul, eg: http://locahost:8500') do |consul_url|
+    options[:consul][:base_url] = consul_url
+  end
+
+  opts.on('-t', '--consul-token=<token>', String, 'Use a token to connect to Consul') do |consul_token|
+    options[:consul][:token] = consul_token
+  end
+
+  opts.on('-w', '--wait=<min_duration>', Float, 'Wait at least n seconds before each template generation') do |min_duration|
+    options[:consul][:min_duration] = min_duration
+  end
+
+  opts.on('-r', '--retry-delay=<min_duration>', Float, 'Min Retry delay on Error/Missing Consul Index') do |min_duration|
+    options[:consul][:min_duration] = min_duration
+  end
+
+  opts.on('-k', '--hot-reload=<behavior>', String, 'Control hot reload behaviour, one of :'\
+                                                  '[die (kill daemon on hot reload failure), '\
+                                                  'keep (on error, keep running), '\
+                                                  'disable (hot reload disabled)] ') do |hot_reload_behaviour|
+    consul_engine.hot_reload_failure = hot_reload_behaviour == 'die' ? nil : hot_reload_behaviour
+  end
+
+  def compute_signal(val, none_value)
+    valid_signals = Signal.list.keys
+    raise "Please #{val} specifiy a signal: #{valid_signals.inspect}" unless val
+    return nil if val == none_value
+    raise "Invalid signal, valid signals: #{valid_signals.inspect}" unless valid_signals.include? val
+    val
+  end
+
+  opts.on('-K', '--sig-term=kill_signal', String,
+          "Signal to sent to next --exec command on kill, default=#{cur_sig_term}") do |sig|
+    cur_sig_term = compute_signal(sig, nil)
+  end
+
+  opts.on('-R', '--sig-reload=reload_signal', String,
+          "Signal to sent to next --exec command on reload (NONE supported), default=#{cur_sig_reload}") do |sig|
+    cur_sig_reload = compute_signal(sig, 'NONE')
+  end
+
+  opts.on('-e', '--exec=<command>', String, 'Execute the following command') do |cmd|
+    sig_reload = cur_sig_reload
+    sig_term = cur_sig_term
+    consul_engine.add_template_callback do |all_ready, template_manager, results|
+      if all_ready
+        modified = results.reduce(false) { |a, e| a || (e.ready? && e.modified) }
+        if modified
+          if @programs[cmd].nil?
+            STDERR.puts "[EXEC] Starting process: #{cmd}... on_reload=#{sig_reload ? sig_reload : 'NONE'} on_term=#{sig_term}"
+            @programs[cmd] = Consul::Async::ProcessHandler.new(cmd, sig_reload: sig_reload, sig_term: sig_term)
+            @programs[cmd].start
+          else
+            @programs[cmd].reload
+          end
+        elsif !@programs[cmd].nil?
+          begin
+            @programs[cmd].process_status
+          rescue Consul::Async::ProcessDoesNotExist => e
+            STDERR.puts "[ERROR] The process is dead, aborting run: #{e.inspect}"
+            template_manager.terminate
+            EventMachine.stop
+          end
+        end
+      end
+    end
+  end
+
+  opts.on('-d', '--debug-network-usage', 'Debug the network usage') do
+    options[:consul][:debug][:network] = true
+    consul_engine.add_template_callback do |all_ready, template_manager, results|
+      if all_ready
+        mod = false
+        results = results.map do |res|
+          mod ||= res.modified
+          STDERR.puts "[INFO] Hot reload of template #{res.template_file} with success" if res.hot_reloaded
+          "#{res.modified ? 'WRITTEN' : 'UNCHANGED'}[#{res.output_file}]"
+        end.join(' ')
+        if mod
+          STDERR.puts("[INFO] File written: #{results} #{template_manager.net_info.inspect}")
+        else
+          STDERR.print "[DBUG] Files not changed #{results} #{template_manager.net_info.inspect}\r"
+        end
+      else
+        STDERR.print "[DBUG] Still waiting for data #{template_manager.net_info.inspect}...\r"
+      end
+    end
+  end
+
+  opts.on('-t', '--template erb_file:[output]:[command]', String, 'Add a erb template, its output and optional reload command') do |tpl|
+    splitted = tpl.split(':')
+    source = splitted[0]
+    dest = splitted[1]
+    unless dest
+      dest = compute_default_output(source)
+      STDERR.puts "-t --template #{tpl} : Since output has not been set, using #{dest}"
+    end
+    raise "Source and destination cannot be the same in #{tpl}" if source == dest || dest.empty?
+    command = splitted[2]
+    consul_engine.add_template(source, dest)
+
+    if command
+      consul_engine.add_template_callback do |_all_ready, _template_manager, results|
+        results.each do |res|
+          next unless res.ready? && res.modified && res.output_file == dest && res.template_file == source
+          # Our template has been fully rendered
+          system(command)
+        end
+      end
+    end
+  end
+
+  opts.on(nil, '--once', 'Do not run the process as a daemon') do
+    consul_engine.add_template_callback do |all_ready, template_manager, _|
+      if all_ready
+        STDERR.puts '[INFO] Program ends since daemon mode has been disabled, file(s) has been written'
+        template_manager.terminate
+        EventMachine.stop
+      end
+    end
+  end
+end
+
+def kill_program
+  @programs.each do |k, v|
+    STDERR.puts "Killing process #{k}..."
+    v.kill
+  end
+  @programs = {}
+  exit 0
+end
+
+optparse.parse!
+
+# Find the max descriptors for our system
+def find_max_descriptors(max_descripors)
+  i = max_descripors
+  max_size = 1024
+  while i != max_size && i > 1024
+    max_size = EM.set_descriptor_table_size i
+    i /= 2 if max_size < i
+  end
+  max_size
+end
+
+# Since we might be using a lots of descriptors, document this
+new_size = find_max_descriptors(65_536)
+STDERR.puts "Max number of descriptors set to #{new_size}" if options[:consul][:debug][:network]
+
+# This is needed to avoid EM not to crash on some Linux Hosts
+# When using a very large number of Consul Endpoints
+# See https://github.com/eventmachine/eventmachine/issues/636#issuecomment-143313282
+EM.epoll
+
+consul_conf = Consul::Async::ConsulConfiguration.new(options[:consul])
+template_manager = Consul::Async::ConsulEndPointsManager.new(consul_conf)
+
+ARGV.each do |tpl|
+  dest = compute_default_output(tpl)
+  puts "Using #{dest} output for #{tpl}"
+  consul_engine.add_template(tpl, dest)
+end
+
+# Ensure to kill child process if any
+%w[INT PIPE TERM].each do |sig|
+  Signal.trap(sig) do
+    STDERR.puts "[KILL] received #{sig}, stopping myself"
+    kill_program
+  end
+end
+
+consul_engine.run(template_manager)
+
+# Kill possible child process if consul_engine.run did stop
+kill_program
+
+exit 0

data/consul-templaterb.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'consul/async/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'consul-templaterb'
+
+  spec.version       = Consul::Async::VERSION
+  spec.authors       = ['SRE Core Services']
+  spec.email         = ['sre-core-services@criteo.com']
+
+  spec.summary       = 'Implementation of Consul template using Ruby and .erb templating language'
+  spec.homepage      = 'https://github.com/criteo/consul-templaterb'
+  spec.description   = 'A ruby implementation of Consul Template with support of erb templating'
+
+  spec.license       = 'Apache v2'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.require_paths = ['lib']
+  spec.add_runtime_dependency 'em-http-request', '>= 1.1.5'
+
+  spec.add_development_dependency 'bundler', '>= 1.14'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec_junit_formatter'
+  spec.add_development_dependency 'rubocop', '0.49.0'
+  spec.add_development_dependency 'rubocop-junit-formatter'
+  spec.add_development_dependency 'webmock'
+end

data/lib/consul/async/consul_endpoint.rb

@@ -0,0 +1,279 @@
+require 'consul/async/utilities'
+require 'em-http'
+require 'thread'
+require 'json'
+module Consul
+  module Async
+    class ConsulConfiguration
+      attr_reader :base_url, :token, :retry_duration, :min_duration, :wait_duration, :max_retry_duration, :retry_on_non_diff,
+                  :missing_index_retry_time_on_diff, :missing_index_retry_time_on_unchanged, :debug
+      def initialize(base_url: 'http://locahost:8500',
+                     debug: { network: false },
+                     token: nil,
+                     retry_duration: 10,
+                     min_duration: 0.1,
+                     retry_on_non_diff: 5,
+                     wait_duration: 600,
+                     max_retry_duration: 600,
+                     missing_index_retry_time_on_diff: 15,
+                     missing_index_retry_time_on_unchanged: 60,
+                     paths: {})
+        @base_url = base_url
+        @token = token
+        @debug = debug
+        @retry_duration = retry_duration
+        @min_duration = min_duration
+        @wait_duration = wait_duration
+        @max_retry_duration = max_retry_duration
+        @retry_on_non_diff = retry_on_non_diff
+        @missing_index_retry_time_on_diff = missing_index_retry_time_on_diff
+        @missing_index_retry_time_on_unchanged = missing_index_retry_time_on_unchanged
+        @paths = paths
+      end
+
+      def ch(path, symbol)
+        sub = @paths[path.to_sym]
+        if sub && sub[symbol]
+          STDERR.puts "[INFO] Overriding #{symbol}:=#{sub[symbol]} for #{path}"
+          sub[symbol]
+        else
+          method(symbol).call
+        end
+      end
+
+      def create(path)
+        return self unless @paths[path.to_sym]
+        ConsulConfiguration.new(base_url: ch(path, :base_url),
+                                debug: ch(path, :debug),
+                                token: ch(path, :token),
+                                retry_duration: ch(path, :retry_duration),
+                                min_duration: ch(path, :min_duration),
+                                retry_on_non_diff: ch(path, :retry_on_non_diff),
+                                wait_duration: ch(path, :wait_duration),
+                                max_retry_duration: ch(path, :max_retry_duration),
+                                missing_index_retry_time_on_diff: ch(path, :missing_index_retry_time_on_diff),
+                                missing_index_retry_time_on_unchanged: ch(path, :missing_index_retry_time_on_unchanged),
+                                paths: @paths)
+      end
+    end
+    class ConsulEndPointStats
+      attr_reader :successes, :errors, :start, :body_bytes
+      def initialize
+        @start = Time.now.utc
+        @successes = 0
+        @errors = 0
+        @body_bytes = 0
+      end
+
+      def on_reponse(res)
+        @successes += 1
+        @body_bytes = body_bytes + res.http.response.bytesize
+      end
+
+      def on_error(_http)
+        @errors += 1
+      end
+
+      def bytes_per_sec
+        diff = (Time.now.utc - start)
+        diff = 1 if diff < 1
+        (body_bytes / diff).round(0)
+      end
+
+      def bytes_per_sec_human
+        "#{Utilities.bytes_to_h(bytes_per_sec)}/s"
+      end
+
+      def body_bytes_human
+        Utilities.bytes_to_h(body_bytes)
+      end
+    end
+    class ConsulResult
+      attr_reader :data, :http, :x_consul_index, :last_update, :stats, :retry_in
+      def initialize(data, modified, http, x_consul_index, stats, retry_in)
+        @data = data
+        @modified = modified
+        @http = http
+        @x_consul_index = x_consul_index
+        @last_update = Time.now.utc
+        @stats = stats
+        @retry_in = retry_in
+      end
+
+      def modified?
+        @modified
+      end
+
+      def mutate(new_data)
+        @data = new_data.dup
+      end
+
+      def json
+        @data_json = JSON.parse(data) if @data_json.nil?
+        @data_json
+      end
+
+      def next_retry_at
+        next_retry + last_update
+      end
+    end
+    class HttpResponse
+      attr_reader :response_header, :response, :error
+      def initialize(http, override_nil_response = nil)
+        if http.nil?
+          @response_header = nil
+          @response = override_nil_response
+          @error = 'Not initialized yet'
+        else
+          @response_header = http.response_header.nil? ? nil : http.response_header.dup.freeze
+          @response = http.response.nil? || http.response.empty? ? override_nil_response : http.response.dup.freeze
+          @error = http.error.nil? ? nil : http.error.dup.freeze
+        end
+      end
+    end
+    class ConsulEndpoint
+      attr_reader :conf, :path, :x_consul_index, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
+      def initialize(conf, path, enforce_json_200 = true, query_params = {}, default_value = '[]')
+        @conf = conf.create(path)
+        @default_value = default_value
+        @path = path
+        @queue = EM::Queue.new
+        @x_consul_index = 0
+        @s_callbacks = []
+        @e_callbacks = []
+        @enforce_json_200 = enforce_json_200
+        @start_time = Time.now.utc
+        @consecutive_errors = 0
+        @query_params = query_params
+        @stopping = false
+        @stats = ConsulEndPointStats.new
+        @last_result = ConsulResult.new(default_value, false, HttpResponse.new(nil), 0, stats, 1)
+        on_response { |result| @stats.on_reponse result }
+        on_error { |http| @stats.on_error http }
+        _enable_network_debug if conf.debug && conf.debug[:network]
+        fetch
+        queue << 0
+      end
+
+      def _enable_network_debug
+        on_response do |result|
+          state = result.x_consul_index.to_i < 1 ? '[WARN]' : '[ OK ]'
+          stats = result.stats
+          STDERR.puts "[DBUG]#{state}#{result.modified? ? '[MODFIED]' : '[NO DIFF]'}" \
+          "[s:#{stats.successes},err:#{stats.errors}]" \
+          "[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
+          " #{path.ljust(48)} idx:#{result.x_consul_index}, next in #{result.retry_in} s"
+        end
+        on_error { |http| STDERR.puts "[ERROR]: #{path}: #{http.error}" }
+      end
+
+      def on_response(&block)
+        @s_callbacks << block
+      end
+
+      def on_error(&block)
+        @e_callbacks << block
+      end
+
+      def ready?
+        @ready
+      end
+
+      def terminate
+        @stopping = true
+      end
+
+      private
+
+      def build_request(consul_index)
+        res = {
+          head: {
+            'Accept'         => 'application/json',
+            'X-Consul-Index' => consul_index,
+            'X-Consul-Token' => conf.token
+          },
+          path: path,
+          query: {
+            wait: "#{conf.wait_duration}s",
+            index: consul_index,
+            stale: 'stale'
+          },
+          keepalive: true,
+          callback: method(:on_response)
+        }
+        @query_params.each_pair do |k, v|
+          res[:query][k] = v
+        end
+        res
+      end
+
+      def find_x_consul_token(http)
+        http.response_header['X_CONSUL_INDEX']
+      end
+
+      def _handle_error(http, consul_index)
+        retry_in = [600, conf.retry_duration + 2**@consecutive_errors].min
+        STDERR.puts "[ERROR][#{path}] X-Consul-Index:#{consul_index} - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        @consecutive_errors += 1
+        http_result = HttpResponse.new(http)
+        EventMachine.add_timer(retry_in) do
+          yield
+          queue.push(consul_index)
+        end
+        @e_callbacks.each { |c| c.call(http_result) }
+      end
+
+      def fetch
+        options = {
+          connect_timeout: 5, # default connection setup timeout
+          inactivity_timeout: conf.wait_duration + 1, # default connection inactivity (post-setup) timeout
+        }
+        connection = EventMachine::HttpRequest.new(conf.base_url, options)
+        cb = proc do |consul_index|
+          http = connection.get(build_request(consul_index))
+          http.callback do
+            # Dirty hack, but contrary to other path, when key is not present, Consul returns 404
+            is_kv_empty = path.start_with?('/v1/kv') && http.response_header.status == 404
+            if !is_kv_empty && enforce_json_200 && http.response_header.status != 200 && http.response_header['Content-Type'] != 'application/json'
+              _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
+            else
+              n_consul_index = find_x_consul_token(http)
+              @consecutive_errors = 0
+              http_result = if is_kv_empty
+                              HttpResponse.new(http, default_value)
+                            else
+                              HttpResponse.new(http)
+                            end
+              new_content = http_result.response.freeze
+              modified = @last_result.nil? ? true : @last_result.data != new_content
+              if n_consul_index == consul_index || n_consul_index.nil?
+                retry_in = modified ? conf.missing_index_retry_time_on_diff : conf.missing_index_retry_time_on_unchanged
+                n_consul_index = consul_index
+              else
+                retry_in = modified ? conf.min_duration : conf.retry_on_non_diff
+              end
+              retry_in = 0.1 if retry_in < 0.1
+              unless @stopping
+                EventMachine.add_timer(retry_in) do
+                  queue.push(n_consul_index)
+                end
+              end
+              result = ConsulResult.new(new_content, modified, http_result, n_consul_index, stats, retry_in)
+              @last_result = result
+              @ready = true
+              @s_callbacks.each { |c| c.call(result) }
+            end
+          end
+
+          http.errback do
+            unless @stopping
+              _handle_error(http, consul_index) { connection = EventMachine::HttpRequest.new(conf.base_url, options) }
+            end
+          end
+          queue.pop(&cb)
+        end
+        queue.pop(&cb)
+      end
+    end
+  end
+end