consult 0.7.3 → 0.10.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d0e9e0e2f63b060fe1fae95c3a95c93d7cf89654dcf18c5cd7c7928a346db2b
-  data.tar.gz: 6c5316a1093c3806deb091aa74f38a1f65101f8fcc1d4525f4f60a50731737f9
+  metadata.gz: 04de1a20dd8943a9c3740bcdd00a2b087c4dd8c706deb8ad47896f72a643ca6b
+  data.tar.gz: 03211311d716648cee6d9d42fb834919b616428015c860e48cbc0e04ae111e4b
 SHA512:
-  metadata.gz: fc4a191a7b1b362b9655cecbfcb83b63354f10659d4a84d899bb271ce0013b82f2c1a9e08f008cbade94ac4132e47438bd3dd3f3fd7ce40cad4c5d54a26d4702
-  data.tar.gz: 05c1543f7f774dae4c7fdeb25e2dbc51f732ea63d09b276e663b29aca2f40b88df80a6e6c83fb4a10b51e2e91217f5c3c0e52ef0d8b9630669f032aa386fade2
+  metadata.gz: c21b28fc79d2480d57e16aad107ecaf6d3e1f38609c781222263b1c98d6cd6e9dd903fc822a5e99ac89bd22f981ae19b9eb74f418ea62f1ec989ac9c6544948c
+  data.tar.gz: a9fa5fc6d4af723125ed276a69391036eef91a2eef94ffe4e46d1587fa5154b8ad6f5411ec51f6bc74ba0edfcf97a99b950b361d3a76462bea75b5df9a19202c

data/.circleci/config.yml

@@ -0,0 +1,78 @@
+version: 2.1
+
+jobs:
+  test:
+    parameters:
+      ruby_version:
+        type: string
+
+    docker:
+      - image: circleci/ruby:<< parameters.ruby_version >>
+      - image: consul:1.3.0
+      - image: vault:0.11.3
+        environment:
+          - VAULT_DEV_ROOT_TOKEN_ID=94e1a9ed-5d72-5677-27ab-ebc485cca368
+          # - SKIP_SETCAP=skip
+        # cap_add:
+        #   - IPC_LOCK
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          name: restore gem cache
+          keys:
+            - v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-{{ checksum "consult.gemspec" }}
+            - v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-
+            - v1-bundle-<< parameters.ruby_version >>-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle check --path ./vendor/bundle || bundle install --jobs=3 --retry=3 --path vendor/bundle
+            bundle clean
+
+      - save_cache:
+          name: save gem cache
+          paths:
+            - ./vendor/bundle
+          key: v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-{{ checksum "consult.gemspec" }}
+
+      - run:
+          name: setup fixture data
+          command: |
+            ./spec/support/populate_consul.sh
+
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | \
+              circleci tests split --split-by=timings)"
+
+            bundle exec rspec \
+              --format progress \
+              --format RspecJunitFormatter \
+              --out /tmp/test-results/rspec.xml \
+              --format progress \
+              $TEST_FILES
+
+      - store_test_results:
+          path: /tmp/test-results
+
+      - store_artifacts:
+          path: ./coverage
+          destination: coverage
+
+workflows:
+  test_supported_ruby_versions:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby_version:
+                - '2.5'
+                - '2.7'

data/.gitignore

@@ -5,7 +5,7 @@
 /doc/
 /pkg/
 /spec/reports/
-/spec/support/rendered/*.yml
+/spec/support/rendered/*
 /tmp/
 
 # rspec failure tracking

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+#### Unreleased
+
+#### 0.10.0
+
+* Switch from Travis to CircleCI, and set up testing for multiple Ruby versions ([#25](https://github.com/veracross/consult/pull/25) & [#34](https://github.com/veracross/consult/pull/34))
+* Improve development on Windows ([#21](https://github.com/veracross/consult/pull/21))
+* Normalize string encodings to UTF-8 ([#22](https://github.com/veracross/consult/pull/22))
+* Remove dependency on ActiveSupport ([#27](https://github.com/veracross/consult/pull/27))
+
+#### 0.9.0
+
+* Add a CLI to render templates on demand ([#20](https://github.com/veracross/consult/pull/20))
+
+#### 0.8.2
+
+* Use `X-Consul-Token` header for Consul authentication. See the [Consul docs](https://www.consul.io/api/index.html#authentication) for details. ([#19](https://github.com/veracross/consult/pull/19))
+
+#### 0.8.1
+
+* Obey template location order as specified in consult's config
+
+#### 0.8.0
+
+* Add support for multiple sources for a single template ([#14](https://github.com/veracross/consult/pull/14))
+* Add support for Consul-sourced templates ([#15](https://github.com/veracross/consult/pull/15))
+* Don't crash on rendering errors ([#16](https://github.com/veracross/consult/pull/16))
+
 #### 0.7.3
 
 * Add `key` function to templates to pull kv data from Consul

data/README.md

@@ -3,7 +3,7 @@
 Generate configuration and secrets for Rails apps automatically from [Consul](https://github.com/hashicorp/consul) & [Vault](https://github.com/hashicorp/vault).
 
 [![Gem Version](https://badge.fury.io/rb/consult.svg)](https://badge.fury.io/rb/consult)
-[![Build Status](https://travis-ci.org/veracross/consult.svg?branch=master)](https://travis-ci.org/veracross/consult)
+[![CircleCI](https://circleci.com/gh/veracross/consult/tree/master.svg?style=svg)](https://circleci.com/gh/veracross/consult/tree/master)
 [![Maintainability](https://api.codeclimate.com/v1/badges/d7b048b7edd9f27c83b9/maintainability)](https://codeclimate.com/github/veracross/consult/maintainability)
 
 ## Background
@@ -14,8 +14,6 @@ We use Consul Template for server level configuration, but application level con
 
 With Consult the process is the same in all environments.
 
-This gem is considered _beta_. At Veracross, we are just beginning to use it in staging environments.
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -40,6 +38,16 @@ Pre-existing copies of files generated by Consult (such as `secrets.yml`, `datab
 
 If this gem is included in a Rails, the templates will render on Rails boot. Configuration or credential changes can be picked up by restarting your app.
 
+### CLI
+
+Render templates on demand with the CLI. By default, this will bypass template TTLs to force rendering and provide verbose output. See `consult --help` for options.
+
+```bash
+$ bundle exec consult
+Consult: Rendered my_config
+Consult: Rendered secrets
+```
+
 ### Configuration
 
 ```yaml
@@ -88,14 +96,20 @@ test:
       dest: config/secrets.yml
 
 production:
-  # example: override vault token in production
-  vault:
-    token: 1397af7aede2-8923-412d-3eb9-8fcd5aed
   templates:
-    # excluded from non-production environments
-    should_be_excluded:
-      path: config/templates/fake.yml
-      dest: config/fake.yml
+    # You can concatenate multiple files together
+    my_config:
+      paths:
+        - config/templates/one.yml
+        - config/templates/two.yml
+      dest: config/my_config.yml
+
+    # Templates can come from Consul
+    your_config:
+      consul_keys:
+        - some/consul/key
+        - another/consul/key
+      dest: config/your_config.txt
 ```
 
 ### Templates
@@ -283,7 +297,7 @@ production:
 Then reference secrets in your app with `Rails.application.secrets`.
 
 ```ruby
-# config/intiializers/rollbar.rb
+# config/initializers/rollbar.rb
 Rollbar.configure do |config|
   config.access_token = Rails.application.secrets.rollbar_token
 end

data/bin/consult

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/consult/cli'
+
+cli = Consult::CLI.instance
+cli.parse
+cli.render

data/consult.gemspec

@@ -20,20 +20,19 @@ Gem::Specification.new do |spec|
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = 'bin'
+  spec.executables   = ['consult']
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '> 4', '< 6'
   spec.add_dependency 'diplomat', '~> 2.0.2'
   spec.add_dependency 'vault', '>= 0.10.0', '< 1.0.0'
 
-  spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'byebug'
+  spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'guard'
   spec.add_development_dependency 'guard-rspec'
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4.1'
+  spec.add_development_dependency 'simplecov', '~> 0.16.1'
 end

data/docker-compose.yml

@@ -1,11 +1,11 @@
 version: '3'
 services:
   consul:
-    image: consul:1.2.0
+    image: consul:1.3.0
     ports:
       - "8500:8500"
   vault:
-    image: vault:0.10.3
+    image: vault:0.11.3
     environment:
       - VAULT_DEV_ROOT_TOKEN_ID=94e1a9ed-5d72-5677-27ab-ebc485cca368
     cap_add:

data/lib/consult.rb

@@ -2,7 +2,6 @@
 
 require 'pathname'
 require 'yaml'
-require 'active_support/core_ext/hash'
 require 'erb'
 require 'vault'
 require 'diplomat'
@@ -10,6 +9,7 @@ require 'diplomat'
 require 'consult/version'
 require 'consult/utilities'
 require 'consult/template'
+require_relative './support/hash_extensions'
 
 module Consult
   @config = {}
@@ -18,17 +18,18 @@ module Consult
   CONSUL_DISK_TOKEN = Pathname.new("#{Dir.home}/.consul-token").freeze
 
   class << self
-    attr_reader :config, :templates
+    attr_reader :config, :templates, :force_render
 
-    def load(config_dir: nil)
+    def load(config_dir: nil, force_render: false, verbose: nil)
       root directory: config_dir
       yaml = root.join('config', 'consult.yml')
 
-      @all_config = yaml.exist? ? YAML.safe_load(ERB.new(yaml.read).result, [], [], true).to_h : {}
-      @all_config.deep_symbolize_keys!
+      @all_config = yaml.exist? ? YAML.safe_load(ERB.new(yaml.read).result, [], [], true, symbolize_names: true).to_h : {}
 
       @config = @all_config[:shared].to_h.deep_merge @all_config[env&.to_sym].to_h
-      @templates = @config[:templates]&.map { |name, config| Template.new(name, config) } || []
+      @templates = @config[:templates]&.map { |name, config| Template.new(name, config.merge(verbose: verbose)) } || []
+
+      @force_render = force_render
 
       configure_consul
       configure_vault
@@ -41,7 +42,9 @@ module Consult
       # Additionally: prefer env vars over explicit config
       configured_address = @config[:consul].delete(:address)
       @config[:consul][:url] = ENV['CONSUL_HTTP_ADDR'] || configured_address || @config[:consul][:url]
-      @config[:consul][:acl_token] = consul_token
+      # If a consul token exists, treat it as special
+      # See https://github.com/WeAreFarmGeek/diplomat/pull/160
+      (@config[:consul][:options] ||= {}).merge!(headers: {'X-Consul-Token' => consul_token}) if consul_token.to_s.length > 0
 
       Diplomat.configure do |c|
         @config[:consul].each do |opt, val|
@@ -70,7 +73,7 @@ module Consult
 
     # Return only the templates that are relevant for the current environment
     def active_templates
-      templates.select(&:should_render?)
+      force_render ? templates : templates.select(&:should_render?)
     end
 
     # Render templates.
@@ -83,7 +86,7 @@ module Consult
     def consul_token
       ENV['CONSUL_HTTP_TOKEN'] ||
         @config[:consul].delete(:token) ||
-        @config[:consul][:acl_token] ||
+        @config[:consul].delete(:acl_token) ||
         (CONSUL_DISK_TOKEN.exist? ? CONSUL_DISK_TOKEN.read.chomp : nil)
     end
   end

data/lib/consult/cli.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+
+require_relative '../consult'
+
+module Consult
+  class CLI
+    include Singleton
+
+    attr_reader :opts
+
+    def parse(args = ARGV)
+      @opts = parse_options(args)
+      Consult.load @opts
+    end
+
+    def render
+      Consult.render!
+    end
+
+    def parse_options(argv)
+      opts = {
+        config_dir: Dir.pwd,
+        force_render: true,
+        verbose: true
+      }
+
+      @parser = OptionParser.new do |o|
+        o.on '-d', '--directory=DIR', 'Path to directory containing the config directory' do |arg|
+          opts[:config_dir] = arg
+        end
+
+        o.on '-f', '--[no-]force', TrueClass, 'Ignore template TTLs and force rendering' do |arg|
+          opts[:force_render] = arg
+        end
+
+        o.on '-v', '--quiet', FalseClass, 'Silence output' do |arg|
+          opts[:verbose] = arg
+        end
+      end
+
+      @parser.on_tail "-h", "--help", "Show help" do
+        puts @parser
+        exit 1
+      end
+
+      @parser.parse! argv
+      opts
+    end
+  end
+end

data/lib/consult/template.rb

@@ -7,6 +7,8 @@ module Consult
     include Utilities
     include TemplateFunctions
 
+    LOCATIONS = %i[path paths consul_key consul_keys]
+
     attr_reader :name, :config
 
     def initialize(name, config)
@@ -15,18 +17,29 @@ module Consult
     end
 
     def render(save: true)
-      renderer = ERB.new(File.read(path, encoding: 'utf-8'), nil, '-')
+      # Attempt to render
+      renderer = ERB.new(contents, nil, '-')
       result = renderer.result(binding)
 
-      File.open(dest, 'w') { |f| f << result } if save
+      File.open(dest, 'wb') { |f| f << result } if save
+      puts "Consult: Rendered #{name}" if verbose?
       result
     rescue StandardError => e
-      puts "Error rendering template: #{name}"
-      raise e
+      STDERR.puts "Error rendering template: #{name}"
+      STDERR.puts e
+      nil
     end
 
     def path
-      resolve @config.fetch(:path)
+      resolve @config[:path]
+    end
+
+    def paths
+      @config.fetch(:paths, []).map { |path| resolve(path) }
+    end
+
+    def vars
+      @config[:vars]
     end
 
     def dest
@@ -42,5 +55,34 @@ module Consult
       return true if !config.key?(:ttl) || !dest.exist?
       dest.mtime < (Time.now - @config[:ttl].to_i)
     end
+
+    def verbose?
+      @config[:verbose]
+    end
+
+    def ordered_locations
+      @config.keys & LOCATIONS
+    end
+
+    private
+
+    # Concatenate all the source templates together, in the order provided
+    def contents
+      ordered_locations.map do |location|
+        location.to_s.start_with?('consul') ? consul_contents(location) : disk_contents(location)
+      end.join
+    end
+
+    def consul_contents(location)
+      [@config[location]].compact.flatten.map do |key|
+        Diplomat::Kv.get(key, options: nil, not_found: :return, found: :return).force_encoding 'utf-8'
+      end.join
+    end
+
+    def disk_contents(location)
+      [public_send(location)].compact.flatten.map do |file_path|
+        File.read file_path, encoding: 'utf-8'
+      end.join
+    end
   end
 end

data/lib/consult/utilities.rb

@@ -3,6 +3,7 @@
 module Consult
   module Utilities
     def resolve(path)
+      return unless path
       pathname = Pathname.new(path)
       pathname.relative? ? Consult.root.join(pathname) : pathname
     end

data/lib/consult/version.rb

@@ -1,3 +1,3 @@
 module Consult
-  VERSION = '0.7.3'
+  VERSION = '0.10.0.pre'
 end

data/lib/support/hash_extensions.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# ActiveSupport's corresponding methods
+#  - https://github.com/rails/rails/tree/master/activesupport/lib/active_support/core_ext/hash
+module HashExtension
+  def deep_merge(other_hash, &block)
+    merge(other_hash) do |key, this_val, other_val|
+      if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+        this_val.deep_merge other_val, &block
+      elsif block_given?
+        block.call key, this_val, other_val
+      else
+        other_val
+      end
+    end
+  end
+end
+
+Hash.send(:include, HashExtension)

metadata

@@ -1,35 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: consult
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.10.0.pre
 platform: ruby
 authors:
 - Jeff Fraser
-autorequire: 
-bindir: exe
+autorequire:
+bindir: bin
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
 - !ruby/object:Gem::Dependency
   name: diplomat
   requirement: !ruby/object:Gem::Requirement
@@ -68,18 +48,18 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,7 +73,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard
+  name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -107,7 +87,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,7 +101,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -135,58 +115,60 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rspec_junit_formatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 0.4.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.16.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.16.1
 description: Manage consul/vault backed template files in Ruby.
 email:
 - jeff.fraser@veracross.com
-executables: []
+executables:
+- consult
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
+- ".ruby-version"
 - CHANGELOG.md
 - Gemfile
 - Guardfile
@@ -195,21 +177,24 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/consult
 - bin/setup
 - consult.gemspec
 - docker-compose.yml
 - lib/consult.rb
+- lib/consult/cli.rb
 - lib/consult/rails/engine.rb
 - lib/consult/template.rb
 - lib/consult/template_functions.rb
 - lib/consult/utilities.rb
 - lib/consult/version.rb
+- lib/support/hash_extensions.rb
 homepage: https://github.com/veracross/consult
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -220,13 +205,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.6
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Manage consul/vault backed template files in Ruby.
 test_files: []

data/.travis.yml

@@ -1,15 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.5
-  - 2.4
-  - 2.3
-
-services:
-  - docker
-
-before_install:
-  - gem install bundler -v 1.16.2
-  - docker-compose --version
-  - docker-compose up -d
-  - docker ps