consul-templaterb 1.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c3c5dba956cece1ad77f8eb4edaee447c86bbade6442adb8b1a1dc21577a62ae
+  data.tar.gz: 8b0e94328289d7529074b608da3284d7fc9ec97d51167d90ad92b817e6582bc3
+SHA512:
+  metadata.gz: 7e0e3f61b3cfac004babf0aba88cd36eedb5bba31dc029ed014eb0c1c5d5e92c4a9eb17be31dc9ade62a1ca23269ea1cc69e593e8d55041a62386e2041963d81
+  data.tar.gz: c4bd56c30874d36f7aaefe4b4abd488fae373a1b7275252a3208424ff47b2770c9666654f3254d4652429c8a3c9c1f1775ed0ca16721de6a167f50a3563b2a6a

data/.gitignore

@@ -0,0 +1,25 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/samples/*.json
+/samples/*.xml
+/samples/*.html
+/samples/*.txt
+/samples/*.cfg
+/samples/criteo/haproxy.cfg
+/samples/consul_template
+/samples/find_blocked_choregraphies
+/spec/consul/async/resources/templates/*.txt
+/spec/consul/async/resources/templates/*.result
+/out.txt
+
+# rspec failure tracking
+.rspec_status
+# Max OS
+.DS_Store

data/.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=review.criteois.lan
+port=29418
+project=ruby-gems/consul-templaterb
+defaultbranch=master

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,43 @@
+AllCops:
+  Exclude:
+    - bundle/**/*
+  TargetRubyVersion: 2.4
+
+Metrics/AbcSize:
+  Max: 80
+
+Metrics/BlockLength:
+  Max: 120
+
+Metrics/BlockNesting:
+  Max: 4
+
+Metrics/ClassLength:
+  Max: 160
+
+Metrics/CyclomaticComplexity:
+  Max: 15
+
+Metrics/LineLength:
+  Max: 150
+
+Metrics/MethodLength:
+  Max: 50
+
+Metrics/ParameterLists:
+  Max: 12
+
+Metrics/PerceivedComplexity:
+  Max: 18
+
+Style/Documentation:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/MultilineBlockChain:
+  Enabled: false
+
+Style/VariableNumber:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+- 2.3
+deploy:
+  provider: rubygems
+  api_key:
+    secure: l78H6Ow9HCeEz0hcAP2XCXaGGBNRXZ/QLFPNzwyNmHToko9WqZj6mbiNIcVwZ8aCcZEzFmtcB7m1lC1l+mvot3Snfh4OUErK7EXMLiNxUWybtA4qJqmsb2J8JGS69pTohPKagi1YyRbyVs2tJpM5OVazcNxEQDAVkWVxJuPVz6tsn4rXZsL77nTSVyB6YGKgguRGhtS/3M0ufk1ymOpuk8pxXKaotlYGRXmht2OHW+FDQczTMgqu6jiZubjWlNHVxtYn+CsLYNj1BswzD6fF1uous6nchV0if2QW5N4OJVyZCx3yEQ0RswfI8G1kG/Jm8wd2lSknY2wpnd1qkRnAcFlKBpUv8aQd6SdbSO84GTAOvucgK7Mcf0LRvYSE2FyagaAPFSfMybWd+GS26g5AuRPpkhtsS9ijfdIzpudiF18pvnJ+wQj+hWM/THWowYNyKv9NeckxdrULaHPisA346r5eM/EADOhyEUtPm11PFgcheGjff1H4qXGc9gcm50u6mNNHkWv38oGhAPpRk/bU7qybTKqRKNGWGP0MiT4WVIFXp62/JMY7AmufOdJx5Ca5JVsaWoN4fsS9r3+R+FxvRtV9NOrRJVKKaQhzi4UHEUDQ2hQjpLDijD5L0vUsl6arl+nufTt5G5jp9QvdY7fhYT1mpGgWj0hTBd8Pxsl3N8U=
+  gem: consul-templaterb
+  edge:
+    branch: v1.8.47
+  on:
+    tags: true
+    repo: criteo/consul-templaterb

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in consul-templaterb.gemspec
+gemspec
+gem 'bootstrap', '~> 4.0.0.alpha6'

data/LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,270 @@
+# Consul::Templaterb
+
+This GEM is both a library and an executable that allows to generate files
+using data from Consul (Discovery and Key/Value Store) easily using ruby's
+erb templates. It also support launching programs and babysitting processes
+when rendering the files, thus notifying programs when data do change.
+
+It is intended for user accustomed to expressiveness or Ruby templating (ERB),
+allowing for more flexibility and than Go Templating.
+
+It also allow to use all of ruby language, especially usefull for generating
+files in several formats (JSON, XML) where text substitutions are hard to get
+right.
+
+It also focuses on good performance and lightweight usage of bandwidth,
+especially for very large clusters and watching lots of services.
+
+For complicated rendering of templates and large Consul Clusters, it usually
+render faster with a more predictible way the template than the original
+consul-template.
+
+## Differences with HashiCorp's consul-template
+
+[Hashicorp's Consul Template](https://github.com/hashicorp/consul-template)
+inspired strongly the creation of this GEM and this executable wants
+to achieve better results in some use cases, especially for very large
+Consul clusters with many nodes and servers.
+
+consul-template has more features regarding Consul support (for instance, it
+has support for Hashicorp's Vault), but consul-templaterb focuses on getting
+more power with the generation of templates and more performance.
+
+Consul Template uses Go templates which is very limited in its set of
+features is quite limited: it is complicated to sort, apply real transformations
+using code and even interact with the OS (ex: get the current date, format
+timestamps...).
+
+The sort feature for instances allow you to create predictible output (i.e: meaning
+that the order of nodes is predictible), thus it might offer better performance
+since the reload of processes if happening ONLY when the files are binary
+different. Thus, if using consul-templaterb, you will reload less your haproxy or
+load-balancer than you would do with consul-template.
+
+Compared to consul-template, consul-templaterb offers the following features:
+
+* Hot-Reload of template files
+* Bandwith limitation per endpoint (will soon support dynamic bandwith limiter)
+* Supports baby sitting of multiple processes
+* Supports all Ruby features (ex: base64, real JSON/XML generation...)
+* Information about bandwidth
+
+The executable supports close semantics to Consul template, it also supports
+commands when files are modified and babysitting of multiple processes with
+ability to send signals to those processes whenever the files do change.
+
+## Installation
+
+You might either use the executable direcly OR use this GEM as a library by
+adding this line to your application's Gemfile:
+
+```ruby
+gem 'consul-templaterb'
+```
+
+And then execute:
+
+```shell
+    $ bundle
+[...]
+```
+
+Or install it yourself as:
+
+```shell
+$ gem install consul-templaterb
+[...]
+```
+
+## Usage of consul-templaterb
+
+### Show help
+
+```shell
+$ consul-templaterb --help
+USAGE: bin/consul-templaterb [[options]]
+    -h, --help                       Show help
+    -v, --version                    Show Version
+    -c, --consul-addr=<address>      Address of Consul, eg: http://locahost:8500
+        --consul-token=<token>       Use a token to connect to Consul
+    -w, --wait=<min_duration>        Wait at least n seconds before each template generation
+    -r, --retry-delay=<min_duration> Min Retry delay on Error/Missing Consul Index
+    -k, --hot-reload=<behavior>      Control hot reload behaviour, one of :[die (kill daemon on hot reload failure), keep (on error, keep running), disable (hot reload disabled)]
+    -K, --sig-term=kill_signal       Signal to sent to next --exec command on kill, default=#{cur_sig_term}
+    -R, --sig-reload=reload_signal   Signal to sent to next --exec command on reload (NONE supported), default=#{cur_sig_reload}
+    -e, --exec=<command>             Execute the following command
+    -d, --debug-network-usage        Debug the network usage
+    -t erb_file:[output]:[command],  Add a erb template, its output and optional reload command
+        --template
+        --once                       Do not run the process as a daemon
+```
+
+### Generate multiple templates
+
+In the same way as consul-template, consul-templaterb supports multiple templates and executing
+commands when the files do change. The parameter `--template <ERB>:<DEST>:[reload_command]` works
+in the following way:
+
+* ERB : the ERB file to use as a template
+* DEST: the destination file
+* reload_command: optional shell command executed whenever the file has been modified
+
+The argument can be specified multiple times, ex:
+
+Example of usage:
+
+```shell
+$ consul-templaterb \\
+  --template "samples/ha_proxy.cfg.erb:/opt/haproxy/etc/haproxy.cfg:sudo service haproxy reload"
+  --template "samples/consul_template.erb:consul-summary.txt"
+```
+
+### Process management and signalisation of configuration files
+
+With the --exec argument (can be specified multiple times), consul-templaterb will launch
+the process specified when all templates have been generated and will send a reload signal
+if the content of any of the files do change (the signal will be sent atomically however,
+meaning that if 2 results of templates are modified at the same time, the signal will be
+sent only once (it is helpful for instance if your app is using several configurations
+files that must be consistent all together).
+
+Signals can be customized per process. Two signals are supported with options --sig-reload and
+--sig-term. When the option is added, the next --exec options to start a process will use the
+given signal. By default, HUP will be sent to reload events (you can use NONE to avoid sending any
+reload signal), TERM will be used when leaving consul-templaterb.
+
+### Bandwidth limitation
+
+This is actually the original reason for the creation of this GEM: on Criteo's large clusters,
+consul-template generated several hundreds of Mb/s to the Consul-Agent which also
+generated several hundreds of Mb/s with the Consul servers.
+
+By design, the GEM supports limiting the number of requests per endpoints (see code in
+`bin/consul-templaterb` file). It avoids using too much network to fetch data from Consul
+in large Consul Clusters (especially when watching lots of files).
+
+The limitation is currently static, but fair dynamic bandwidth allocation will allow to limit
+the bandwidth used to get information for all services by capping the global bandwidth used
+by consul-templaterb.
+
+### Samples
+
+Have a look into the [samples/](samples/) directory to browse example files.
+
+If you want to test it quickly, you might try with (assuming you consul agent is listening on
+http://localhost:8500):
+
+```
+$ be bin/consul-templaterb -c 'http://localhost:8500' samples/*.html.erb
+```
+
+It will generate a full website in samples/ directory with lots of Consul information ready to
+use (website updated automagically when values to change).
+
+## Template development
+
+Here are the various functions you might use in your templates.
+
+For each function, mandatory arguments are specified at the begining while optional ones are marked with `[]`.
+Most of them support the optional dc attribute to access data from another datacenter. If the `dc`
+attribute is not specified, the function will output data from the current datacenter.
+
+To ease template development, `consul-templaterb` supports HOT reload of templates, thus it is possible to
+develop the templates interactivelly. While developping, it is possible to use the switch `--hot-reload=keep`,
+thus the application will display a warning if the template is invalid and won't stop
+(`--hot-reload=die` is the default, thus if the hot-reloaded template has issue, the application will die).
+
+### datacenters()
+
+[Get the list of datacenters as string array](https://www.consul.io/api/catalog.html#list-datacenters).
+
+### services([dc: datacenter], [tag: tagToFilterWith])
+
+[List the services matching the optional tag filter](https://www.consul.io/api/catalog.html#list-services),
+if tag is not specified, will match all the services. Note that this endpoint performs client side tag
+filtering for services to ease templates development since this feature is not available on Consul's endpoint.
+
+### service(serviceName, [dc: datacenter], [tag: tagToFilterWith], [passing: true])
+
+[List the instances](https://www.consul.io/api/health.html#list-nodes-for-service) of a service having the given
+optional tag. If no tag is specified, will return all instances of service. By default, it will return all the
+well services that are passing or not. An optional argument passing might be used to retrieve only passing instances.
+
+### nodes([dc: datacenter])
+
+[List all the nodes of selected datacenter](https://www.consul.io/api/catalog.html#list-nodes). No filtering is
+applied.
+
+### node(nodeNameOrId, [dc: datacenter])
+
+[List all the services of a given Node](https://www.consul.io/api/catalog.html#list-services-for-node) using its
+name or its ID. If DC is specified, will lookup for given node in another datacenter.
+
+### checks_for_service(name, dc: nil, passing: false, tag: nil)
+
+[Find all the checks](https://www.consul.io/api/health.html#list-checks-for-service) of a given service.
+
+### kv(name = nil, dc: nil, keys: nil, recurse: false)
+
+[Read keys from KV Store](https://www.consul.io/api/kv.html#read-key). It can be used for both listing the keys and
+getting the values. See the file in samples [keys.html.erb](samples/keys.html.erb) for a working example.
+
+### agent_metrics()
+
+[Get the metrics of Consul Agent](https://www.consul.io/api/agent.html#view-metrics). Since this endpoint does
+not support blocking queries, data will be refreshed every few seconds, but will not use blocking queries
+mechanism.
+
+### agent_self()
+
+[Get the configuration of Consul Agent](https://www.consul.io/api/agent.html#read-configuration). Since this
+endpoint does not support blocking queries, data will be refreshed every few seconds, but will not use blocking
+queries mechanism.
+
+### render_file RELATIVE_PATH_TO_ERB_FILE
+
+This allow to include a template file into another one. Beware, it does not check for infinite recursion!
+The template can be either a static file either another template. The file has to be a valid template, but
+can also be raw text (if it is a valid template.
+
+Example:
+
+```erb
+<%= render_file 'header.html.erb' %>
+```
+
+## Development
+
+We recommend using bundle using `bundle install`, you can now run `bundle exec bin/consul-templaterb`.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
+version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Known bugs
+
+Here are the known bugs of the application:
+
+* [ ] On Mac OS X, with many services, consul-templaterb sometimes crashes when watching lots of changes (more than 2k
+  watches) at the same time. This bug is a race condition, probably in `em-http-request`. Only visible on very large
+  clusters.
+* [ ] render_file might create an infinite recusion if a template includes itself indirectly.
+
+## TODO
+
+* [ ] Hashi's Vault support
+* [ ] Implement automatic dynamic rate limit
+* [ ] More samples: apache, nginx, full website displaying consul information...
+* [ ] Optimize rendering speed at startup: an iteration is done very second by default, but it would be possible to speed
+      up rendering by iterating with higher frequency until the first write of result has been performed.
+* [ ] Allow to tune bandwith using a simple config file (while it should not be necessary for 90% of use-cases)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/criteo/consul-templaterb.
+This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the
+[Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the Apache v2 license. See LICENSE.txt file.