consul-templaterb 1.17.4 → 1.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc1a1a088369364bcc3a1db3bbb1418312575858268189a15632be9b9fd59edf
-  data.tar.gz: d0db07806bd2ba9bb597490cd34763c912d796385166a9e5a4b153e75cc8bd97
+  metadata.gz: 11600fd1fbd0d94c1cc32e9fdcc5734f5f8caa1a543adaa876b5831f4e0a8303
+  data.tar.gz: 69ce9c611cf41505b43d743adfca4f453857cea9f1df8a4e907a98c694468ed0
 SHA512:
-  metadata.gz: cb8c4269065695d6fa08da55d7b60669b8c91d037bb2fd196a150872e02e67d24dd12b6a54390c27b28d59843703b64f82e7ea642d8a3c818c12d92abe695b1a
-  data.tar.gz: 68be729f62bf5b5f6f9933ad660c3f9c2e6d44b17de1a17837952c6930af939ba70f0513c2a55bd73def8194506daebb77b40cf9f25f79ed9f510d817b37b468
+  metadata.gz: 89ef2d0efa08adc140735b8df89ed45b5074e9a2ef90ccdbf96eb94f5cfb8a81d4ff807859f137263b1dcc1ae09458819e036da4fe08a557ef27ee76d106dfdc
+  data.tar.gz: 1668c8e3814b857af84f3f0cdd6011c3cb93733c4c62ae43424f268cd4bc9b0c9958c0d5568fc3466b47b84044619093b5ff58a136de4041e901953d8c944444

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## (UNRELEASED)
 
+## 1.18.0 (July 27, 2019)
+
+NEW FEATURES:
+
+ * Support for `remote_resource` provided by @kamaradclimber
+ * Added support for `remote_resource.as_json` to fetch JSON remote resource from a web server
+ * Added `samples/list_ruby_versions_from_rubygems.txt.erb` to demonstrate usage
+
 ## 1.17.3 (July 18, 2019)
 
 BUGFIX:

data/README.md

@@ -256,6 +256,7 @@ examples:
    datacenters and nodes and export it to prometheus easily to trigger alerts.
 10. [List all services/Nodes with their statuses for all datacenters](samples/all_services.txt.erb)
 11. [Show all services/instances not passing on all DCs](samples/tools/find_all_failing_services.txt.erb)
+12. [List all rubygems consul versions from remote server JSON](samples/list_ruby_versions_from_rubygems.txt.erb)
 
 If you want to test it quickly, you might try with (assuming your consul agent is listening on
 `http://localhost:8500`):

data/TemplateAPI.md

@@ -434,6 +434,19 @@ secret('secret/foo', [force_ttl: intInSecond])
 </div>
 </details>
 
+## remote_resource
+
+### as_json(url, default_value, [refresh_delay_secs: intInSecond])
+
+Fetch json data from any url. This allows to create templates with consul/vault data mixed in with data coming from other services/api.
+Polling interval can be controlled with refresh_delay_secs.
+
+```erb
+remote_resource.as_json('http://my-api.dev/fridge/list.json', [])
+```
+
+Full example: [samples/list_ruby_versions_from_rubygems.txt.erb](samples/list_ruby_versions_from_rubygems.txt.erb)
+
 ## template_info()
 
 It returns information about current template being rendered.

data/lib/consul/async/consul_template.rb

@@ -21,6 +21,22 @@ module Consul
       end
     end
 
+    class RemoteResource
+      def initialize(endpoints_manager)
+        @endp_manager = endpoints_manager
+      end
+
+      def as_json(url, default_value, refresh_delay_secs: 10)
+        conf = JSONConfiguration.new(url: url, min_duration: refresh_delay_secs, retry_on_non_diff: refresh_delay_secs)
+        ret = if default_value.is_a?(Array)
+                ConsulTemplateJSONArray.new(JSONEndpoint.new(conf, url, default_value))
+              else
+                ConsulTemplateJSONObject.new(JSONEndpoint.new(conf, url, default_value))
+              end
+        @endp_manager.create_if_missing(url, {}) { ret }
+      end
+    end
+
     # Encapsulation of endpoints to get coordinates
     class Coordinate
       def initialize(endpoints_manager)
@@ -62,7 +78,7 @@ module Consul
     end
 
     class EndPointsManager
-      attr_reader :consul_conf, :vault_conf, :net_info, :start_time, :coordinate
+      attr_reader :consul_conf, :vault_conf, :net_info, :start_time, :coordinate, :remote_resource
       def initialize(consul_configuration, vault_configuration, trim_mode = nil)
         @consul_conf = consul_configuration
         @vault_conf = vault_configuration
@@ -89,6 +105,7 @@ module Consul
           params: {}
         }
         @coordinate = Coordinate.new(self)
+        @remote_resource = RemoteResource.new(self)
 
         # Setup token renewal
         vault_setup_token_renew unless @vault_conf.token.nil? || !@vault_conf.token_renew
@@ -380,6 +397,11 @@ module Consul
       end
     end
 
+    # technically this class could be also an array, a simple string or any simple json object other than a hash.
+    class ConsulTemplateAbstractJSONObject < ConsulTemplateAbstractMap; end
+
+    class ConsulTemplateAbstractJSONArray < ConsulTemplateAbstractArray; end
+
     class ServiceInstance < Hash
       def initialize(obj)
         merge!(obj)
@@ -478,6 +500,9 @@ module Consul
       end
     end
 
+    class ConsulTemplateJSONObject < ConsulTemplateAbstractJSONObject; end
+    class ConsulTemplateJSONArray < ConsulTemplateAbstractJSONArray; end
+
     class ConsulAgentSelf < ConsulTemplateAbstractMap
       def initialize(consul_endpoint)
         super(consul_endpoint)

data/lib/consul/async/consul_template_engine.rb

@@ -1,5 +1,6 @@
 require 'consul/async/utilities'
 require 'consul/async/consul_endpoint'
+require 'consul/async/json_endpoint'
 require 'consul/async/vault_endpoint'
 require 'consul/async/consul_template'
 require 'consul/async/consul_template_render'

data/lib/consul/async/json_endpoint.rb

@@ -0,0 +1,215 @@
+require 'consul/async/utilities'
+require 'consul/async/stats'
+require 'em-http'
+require 'json'
+
+module Consul
+  module Async
+    class JSONConfiguration
+      attr_reader :url, :retry_duration, :min_duration, :retry_on_non_diff,
+                  :debug, :enable_gzip_compression
+      def initialize(url:,
+                     debug: { network: false },
+                     retry_duration: 10,
+                     min_duration: 10,
+                     retry_on_non_diff: 10,
+                     enable_gzip_compression: true)
+        @url = url
+        @debug = debug
+        @enable_gzip_compression = enable_gzip_compression
+        @retry_duration = retry_duration
+        @min_duration = min_duration
+        @retry_on_non_diff = retry_on_non_diff
+      end
+
+      def create(_url)
+        # here we assume we don't need to cache configuration
+        self
+      end
+    end
+    class JSONResult
+      attr_reader :data, :http, :last_update, :stats, :retry_in
+      def initialize(data, modified, http, stats, retry_in, fake: false)
+        @data = data
+        @modified = modified
+        @http = http
+        @last_update = Time.now.utc
+        @stats = stats
+        @retry_in = retry_in
+        @fake = fake
+      end
+
+      def fake?
+        @fake
+      end
+
+      def modified?
+        @modified
+      end
+
+      def mutate(new_data)
+        @data = new_data.dup
+        @json = nil
+      end
+
+      def json
+        @json ||= JSON.parse(data)
+      end
+
+      def next_retry_at
+        next_retry + last_update
+      end
+    end
+    class HttpResponse
+      attr_reader :response_header, :response, :error
+      def initialize(http, override_nil_response = nil)
+        if http.nil?
+          @response_header = nil
+          @response = override_nil_response
+          @error = 'Not initialized yet'
+        else
+          @response_header = http.response_header.nil? ? nil : http.response_header.dup.freeze
+          @response = http.response.nil? || http.response.empty? ? override_nil_response : http.response.dup.freeze
+          @error = http.error.nil? ? nil : http.error.dup.freeze
+        end
+      end
+    end
+    class JSONEndpoint
+      attr_reader :conf, :url, :queue, :stats, :last_result, :enforce_json_200, :start_time, :default_value, :query_params
+      def initialize(conf, url, default_value, enforce_json_200 = true, query_params = {})
+        @conf = conf.create(url)
+        @default_value = default_value
+        @url = url
+        @queue = EM::Queue.new
+        @s_callbacks = []
+        @e_callbacks = []
+        @enforce_json_200 = enforce_json_200
+        @start_time = Time.now.utc
+        @consecutive_errors = 0
+        @query_params = query_params
+        @stopping = false
+        @stats = EndPointStats.new
+        @last_result = JSONResult.new(default_value.to_json, false, HttpResponse.new(nil), stats, 1, fake: true)
+        on_response { |result| @stats.on_response result }
+        on_error { |http| @stats.on_error http }
+        _enable_network_debug if conf.debug && conf.debug[:network]
+        fetch
+        queue << Object.new
+      end
+
+      def _enable_network_debug
+        on_response do |result|
+          stats = result.stats
+          warn "[DBUG][ OK ]#{result.modified? ? '[MODFIED]' : '[NO DIFF]'}" \
+          "[s:#{stats.successes},err:#{stats.errors}]" \
+          "[#{stats.body_bytes_human.ljust(8)}][#{stats.bytes_per_sec_human.ljust(9)}]"\
+          " #{url.ljust(48)}, next in #{result.retry_in} s"
+        end
+        on_error { |http| warn "[ERROR]: #{url}: #{http.error.inspect}" }
+      end
+
+      def on_response(&block)
+        @s_callbacks << block
+      end
+
+      def on_error(&block)
+        @e_callbacks << block
+      end
+
+      def ready?
+        @ready
+      end
+
+      def terminate
+        @stopping = true
+      end
+
+      private
+
+      def build_request
+        res = {
+          head: {
+            'Accept' => 'application/json'
+          },
+          url: url,
+          keepalive: true,
+          callback: method(:on_response)
+        }
+        res[:head]['accept-encoding'] = 'identity' unless conf.enable_gzip_compression
+        @query_params.each_pair do |k, v|
+          res[:query][k] = v
+        end
+        res
+      end
+
+      def _compute_retry_in(retry_in)
+        retry_in / 2 + Consul::Async::Utilities.random.rand(retry_in)
+      end
+
+      def _handle_error(http)
+        retry_in = _compute_retry_in([600, conf.retry_duration + 2**@consecutive_errors].min)
+        ::Consul::Async::Debug.puts_error "[#{url}] - #{http.error} - Retry in #{retry_in}s #{stats.body_bytes_human}"
+        @consecutive_errors += 1
+        http_result = HttpResponse.new(http)
+        EventMachine.add_timer(retry_in) do
+          yield
+          queue.push(Object.new)
+        end
+        @e_callbacks.each { |c| c.call(http_result) }
+      end
+
+      def fetch
+        options = {
+          connect_timeout: 5, # default connection setup timeout
+          inactivity_timeout: 60 # default connection inactivity (post-setup) timeout
+        }
+        connection = {
+          conn: EventMachine::HttpRequest.new(conf.url, options)
+        }
+        cb = proc do
+          http = connection[:conn].get(build_request)
+          http.callback do
+            if enforce_json_200 && http.response_header.status != 200 && http.response_header['Content-Type'] != 'application/json'
+              _handle_error(http) do
+                warn "[RETRY][#{url}] (#{@consecutive_errors} errors)" if (@consecutive_errors % 10) == 1
+              end
+            else
+              @consecutive_errors = 0
+              http_result = HttpResponse.new(http)
+              new_content = http_result.response.freeze
+              modified = @last_result.fake? || @last_result.data != new_content
+              retry_in = modified ? conf.min_duration : conf.retry_on_non_diff
+              retry_in = _compute_retry_in(retry_in)
+              retry_in = 0.1 if retry_in < 0.1
+              unless @stopping
+                EventMachine.add_timer(retry_in) do
+                  queue.push(Object.new)
+                end
+              end
+              result = JSONResult.new(new_content, modified, http_result, stats, retry_in, fake: false)
+              @last_result = result
+              @ready = true
+              @s_callbacks.each { |c| c.call(result) }
+            end
+          end
+
+          http.errback do
+            unless @stopping
+              _handle_error(http) do
+                if (@consecutive_errors % 10) == 1
+                  add_msg = http.error
+                  if Gem.win_platform? && http.error.include?('unable to create new socket: Too many open files')
+                    add_msg += "\n *** Windows does not support more than 2048 watches, watch less endpoints ***"
+                  end
+                  ::Consul::Async::Debug.puts_error "[RETRY][#{url}] (#{@consecutive_errors} errors) due to #{add_msg}"
+                end
+              end
+            end
+          end
+          queue.pop(&cb)
+        end
+        queue.pop(&cb)
+      end
+    end
+  end
+end

data/lib/consul/async/version.rb

@@ -1,5 +1,5 @@
 module Consul
   module Async
-    VERSION = '1.17.4'.freeze
+    VERSION = '1.18.0'.freeze
   end
 end

data/samples/list_ruby_versions_from_rubygems.txt.erb

@@ -0,0 +1,7 @@
+All sorted consul-templaterb versions
+<%
+  remote_resource
+    .as_json('https://rubygems.org/api/v1/versions/consul-templaterb.json', [], refresh_delay_secs: 1800)
+    .sort { |a, b| Gem::Version.create(a['number']) <=> Gem::Version.create(b['number']) }.each do |gem_version|
+%> * Version: <%= gem_version['number'] %> (<%= gem_version['created_at'] %>) with <%= gem_version['downloads_count'] %> downloads
+<% end %>

data/samples/prometheus_consul_coordinates.erb

@@ -22,7 +22,6 @@ unless @consul_node_settings
   if @consul_node_settings[:num_cpus] < 0
     require 'etc'
     @consul_node_settings[:num_cpus] = Etc.nprocessors - 1
-    STDERR.puts "Autodetected #{@consul_node_settings[:num_cpus]} CPUs"
   end
 end
 
@@ -135,4 +134,4 @@ all_nodes.each do |node, node_info|
 <%
   end
 end
-%>
+%>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: consul-templaterb
 version: !ruby/object:Gem::Version
-  version: 1.17.4
+  version: 1.18.0
 platform: ruby
 authors:
 - SRE Core Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-18 00:00:00.000000000 Z
+date: 2019-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -183,6 +183,7 @@ files:
 - lib/consul/async/consul_template_engine.rb
 - lib/consul/async/consul_template_render.rb
 - lib/consul/async/debug.rb
+- lib/consul/async/json_endpoint.rb
 - lib/consul/async/process_handler.rb
 - lib/consul/async/stats.rb
 - lib/consul/async/utilities.rb
@@ -224,6 +225,7 @@ files:
 - samples/haproxy_dns.cfg.erb
 - samples/keys.html.erb
 - samples/kv_yaml_to_json.json.erb
+- samples/list_ruby_versions_from_rubygems.txt.erb
 - samples/metrics.erb
 - samples/nodes.html.erb
 - samples/prometheus_consul_coordinates.erb
@@ -257,7 +259,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubyforge_project: 
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Implementation of Consul template using Ruby and .erb templating language