constancy 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3119c8dc521f42f974a1f708273026eb49f59c19e25867408aae150dc83261ac
-  data.tar.gz: f6c729bfd5bbf17e29ecd21570076dcf212ae5af999b5f5d89f629216d1c33b0
+  metadata.gz: 4f4ece9e3ea596821b57e8e4e2f4e11d70833df49f3480082a4438a1a018a69b
+  data.tar.gz: 5cead608d291729245435375774093004032877b5e4e447aa7ebf0ff8a7b0e2a
 SHA512:
-  metadata.gz: 1928aead12f4a255cc9e8595d3c024b115a30b950fdcefe5df3ab50d35b122ba8340ed7a95f4db15a36b04f8b8f124d10e00467000088d78a1e6e88e72d0c3e0
-  data.tar.gz: f1d991706a3e0883bf2b22f9e53588a1c6472a55376fb424c5ead455a5c704d99bc6966d378deda7d07c648c606941fb5499fa23affd837104e4e1e513362265
+  metadata.gz: 7b40eb6bc0491fce5719d10c44cd11b154f71ea6415a5f5564b116ed7814a8d796deb046cf18e6d74cf17b8da15a5726a094141ae491506d1e5a3739c192f6b6
+  data.tar.gz: c45737ecc9a1c1aa92b0bfb46594dbe5ce8be6dac55f4e9efb3e30655b65bec9ec1cccec1895a0805da86c9b8bd6193f5116b11798c3e2a1a24d2f6d3b159d24

data/README.md

@@ -20,14 +20,14 @@ synchronize the changes from the filesystem to Consul.
     local:consul/config => consul:dc1:config/myapp
       Keys scanned: 80
 
-    UPDATE config/myapp/prod/ip-whitelist.json
+    UPDATE config/myapp/prod/ip-allowlist.json
     -------------------------------------------------------------------------------------
     -["10.8.0.0/16"]
     +["10.8.0.0/16","10.9.10.0/24"]
     -------------------------------------------------------------------------------------
 
     Keys to update: 1
-    ~ config/myapp/prod/ip-whitelist.json
+    ~ config/myapp/prod/ip-allowlist.json
 
 You can also limit your command to specific synchronization targets by using
 the `--target` flag:
@@ -38,19 +38,19 @@ the `--target` flag:
     local:consul/config => consul:dc1:config/myapp
       Keys scanned: 80
 
-    UPDATE config/myapp/prod/ip-whitelist.json
+    UPDATE config/myapp/prod/ip-allowlist.json
     -------------------------------------------------------------------------------------
     -["10.8.0.0/16"]
     +["10.8.0.0/16","10.9.10.0/24"]
     -------------------------------------------------------------------------------------
 
     Keys to update: 1
-    ~ config/myapp/prod/ip-whitelist.json
+    ~ config/myapp/prod/ip-allowlist.json
 
     Do you want to push these changes?
       Enter 'yes' to continue: yes
 
-    UPDATE config/myapp/prod/ip-whitelist.json   OK
+    UPDATE config/myapp/prod/ip-allowlist.json   OK
 
 Run `constancy --help` for additional options and commands.
 
@@ -127,6 +127,9 @@ required. An example `constancy.yml` is below including explanatory comments:
       #   'none': expect no Consul token (although env vars will be used if they are set)
       #   'env': expect Consul token to be set in CONSUL_TOKEN or CONSUL_HTTP_TOKEN
       #   'vault': read Consul token from Vault based on settings in the 'vault' section
+      #   'vault.<label>': a named Vault token source, eg `vault.us-east-1` or `vault.dev`
+      #      NOTE: labels must begin with a letter and may contain only (ASCII) letters,
+      #            numbers, hyphens, and underscores
 
     # the vault section is only necessary if consul.token_source is set to 'vault'
     vault:
@@ -146,6 +149,15 @@ required. An example `constancy.yml` is below including explanatory comments:
       #   but can be set to something else for static values.
       consul_token_field: token
 
+    # You can define one or more 'vault.<label>' sections to define alternative Vault
+    # token sources for use in individual sync targets.
+    vault.other:
+      url: https://your.vault.example
+      consul_token_path: consul/creds/my-other-role
+    vault.dev:
+      url: https://dev.vault.example
+      consul_token_path: consul/creds/my-dev-role
+
     sync:
       # sync is an array of hashes of sync target configurations
       #   Fields:
@@ -167,6 +179,10 @@ required. An example `constancy.yml` is below including explanatory comments:
       #       containing a hash of remote keys if this sync target has
       #       type=file. This path is calculated relative to the directory
       #       containing the configuration file.
+      #     token_source - An alternative token source other than the
+      #       default. Potential values are the same as for the
+      #       consul.token_source config value: 'none', 'env', 'vault',
+      #       or 'vault.<label>'.
       #     delete - Whether or not to delete remote keys that do not exist
       #       in the local filesystem. This inherits the setting from the
       #       `constancy` section, or if not specified, defaults to `false`.
@@ -180,6 +196,9 @@ required. An example `constancy.yml` is below including explanatory comments:
       #       ignored. At this time there is no provision for specifying
       #       prefixes or patterns. Each key must be fully and explicitly
       #       specified.
+      #     erb_enabled - Whether or not to run the local content through
+      #       ERB parsing before attempting to sync to the remote. Defaults
+      #       to `false`.
       - name: myapp-config
         prefix: config/myapp
         datacenter: dc1
@@ -192,6 +211,7 @@ required. An example `constancy.yml` is below including explanatory comments:
         type: dir
         datacenter: dc1
         path: consul/private
+        token_source: vault.dev
         delete: true
       - name: yourapp-config
         prefix: config/yourapp
@@ -199,6 +219,7 @@ required. An example `constancy.yml` is below including explanatory comments:
         datacenter: dc1
         path: consul/yourapp.yml
         delete: true
+        erb_enabled: true
 
 You can run `constancy config` to get a summary of the defined configuration
 and to double-check config syntax.
@@ -222,33 +243,98 @@ If the file `yourapp.yml` has the following content:
     prod/dbname: yourapp
     prod/message: |
       Hello, world. This is a multiline message.
-      I hope you like it.
-      Thanks,
-      YourApp
+      Thanks.
     prod/app/config.json: |
       {
         "port": 8080,
-        "listen": "0.0.0.0",
         "enabled": true
       }
 
 Then `constancy push` will attempt to create and/or update the following keys
 with the corresponding content from `yourapp.yml`:
 
-    config/yourapp/prod/dbname
-    config/yourapp/prod/message
-    config/yourapp/prod/app/config.json
+| Key  | Value  |
+|:-----|:-------|
+| `config/yourapp/prod/dbname` | `yourapp` |
+| `config/yourapp/prod/message` | `Hello, world. This is a multiline message.\nThanks.` |
+| `config/yourapp/prod/app/config.json` | `{\n  "port": 8080,\n  "enabled": true\n}` |
+
+In addition to specifying the entire relative path in each key, you may also
+reference paths via your file's YAML structure directly. For example:
+
+    ---
+    prod:
+      redis:
+        port: 6380
+        host: redis.example.com
+
+When pushed, this document will create and/or update the following keys:
+
+| Key  | Value  |
+|:-----|:-------|
+| `config/yourapp/prod/redis/port` | `6380` |
+| `config/yourapp/prod/redis/host` | `redis.example.com` |
+
+You may mix and match relative paths and document hierarchy to build paths as
+you would like. And you may also use the special key `_` to embed a value for
+a particular prefix while also nesting values underneath it. For example, given
+this local file target content:
+
+    ---
+    prod/postgres:
+      host: db.myproject.example.com
+      port: 10001
+
+    prod:
+      redis:
+        _: Embedded Value
+        port: 6380
+
+    prod/redis/host: cache.myproject.example.com
 
-Likewise, a `constancy pull` operation will work in reverse, and pull values
-from any keys under `config/yourapp/` into the file `yourapp.yml`, overwriting
-whatever values are there.
+This file target content would correspond to the following values, when pushed:
 
-Note that JSON is also supported for this file for `push` operations, given that
-YAML parsers will correctly parse JSON. However, `constancy pull` will only
-write out YAML in the current version.
+| Key  | Value  |
+|:-----|:-------|
+| `config/yourapp/prod/postgres/host` | `db.myproject.example.com` |
+| `config/yourapp/prod/postgres/port` | `10001` |
+| `config/yourapp/prod/redis` | `Embedded Value` |
+| `config/yourapp/prod/redis/port` | `6380` |
+| `config/yourapp/prod/redis/host` | `cache.myproject.example.com` |
 
-Also important to note that any comments in the YAML file will be lost on a
-`pull` operation that updates a file sync target.
+A `constancy pull` operation against a file type target will work in reverse,
+and pull values from any keys under `config/yourapp/` into the file
+`yourapp.yml`, overwriting whatever values are there.
+
+**NOTE**: Values in local file targets are converted to strings before comparing
+with or uploading to the remote Consul server. However, because YAML parsing
+converts some values (such as `yes` or `no`) to boolean types, the effective
+value of a key with a value of a bare `yes` will be `true` when converted to a
+string. If you need the actual values `yes` or `no`, use quotes around the value
+to force the YAML parser to interpret it as a string.
+
+
+#### IMPORTANT NOTES ABOUT PULL MODE WITH FILE TARGETS
+
+Against a file target, the structure of the local file can vary in a number
+of ways while still producing the same remote structure. Thus, in pull mode,
+Constancy must necessarily choose one particular rendering format, and will not
+be able to retain the exact structure of the local file if you alternate push
+and pull operations.
+
+Specifically, the following caveats are important to note, when pulling a target
+to a local file:
+
+* The local file will be written out as YAML, even if it was originally
+  provided locally as a JSON file, and even if the extension is `.json`.
+
+* Any existing comments in the local file will be lost.
+
+* The document structure will be that of a flat hash will fully-specified
+  relative paths as the keys.
+
+Future versions of Constancy may provide options to modify the behavior for pull
+operations on a per-target basis. Pull requests are always welcome.
 
 
 ### Dynamic configuration
@@ -270,6 +356,12 @@ It's a good idea to sanity-check your ERB by running `constancy config` after
 making any changes.
 
 
+### Dynamic content
+
+You can also choose to enable ERB parsing for local content as well, by setting
+`erb_enabled: true` on any sync targets you wish to populate in this way.
+
+
 ### Environment configuration
 
 Constancy may be partially configured using environment variables:
@@ -279,10 +371,10 @@ Constancy may be partially configured using environment variables:
   interacting with the API. Otherwise, by default the agent's `acl_token`
   setting is used implicitly.
 * `VAULT_ADDR` and `VAULT_TOKEN` - if `consul.token_source` is set to `vault`
-  these variables are used to authenticate to Vault. If `VAULT_TOKEN` is not
-  set, Constancy will attempt to read a token from `~/.vault-token`. If the
-  `url` field is set, it will take priority over the `VAULT_ADDR` environment
-  variable, but one or the other must be set.
+  or `vault.<label>`, these variables are used to authenticate to Vault. If
+  `VAULT_TOKEN` is not set, Constancy will attempt to read a token from
+  `~/.vault-token`. If the `url` field is set, it will take priority over the
+  `VAULT_ADDR` environment variable, but one or the other must be set.
 
 
 ## Roadmap
@@ -291,6 +383,7 @@ Constancy is relatively new software. There's more to be done. Some ideas, which
 may or may not ever be implemented:
 
 * Using CAS to verify the key has not changed in the interim before updating/deleting
+* Options for file target pull-mode rendering
 * Automation support for running non-interactively
 * Pattern- and prefix-based exclusions
 * Logging of changes to files, syslog, other services

data/bin/constancy

@@ -3,6 +3,6 @@
 # This software is public domain. No rights are reserved. See LICENSE for more information.
 #
 
-require 'constancy/cli'
+require_relative '../lib/constancy/cli'
 
 Constancy::CLI.run

data/lib/constancy.rb

@@ -4,12 +4,14 @@ require 'erb'
 require 'imperium'
 require 'fileutils'
 require 'ostruct'
+require 'vault'
 require 'yaml'
 
-require 'constancy/version'
-require 'constancy/config'
-require 'constancy/diff'
-require 'constancy/sync_target'
+require_relative 'constancy/version'
+require_relative 'constancy/config'
+require_relative 'constancy/token_source'
+require_relative 'constancy/diff'
+require_relative 'constancy/sync_target'
 
 class Constancy
   class InternalError < RuntimeError; end

data/lib/constancy/cli.rb

@@ -1,11 +1,12 @@
 # This software is public domain. No rights are reserved. See LICENSE for more information.
 
-require 'constancy'
+require_relative '../constancy'
 require 'diffy'
-require 'constancy/cli/check_command'
-require 'constancy/cli/push_command'
-require 'constancy/cli/pull_command'
-require 'constancy/cli/config_command'
+require_relative 'cli/check_command'
+require_relative 'cli/push_command'
+require_relative 'cli/pull_command'
+require_relative 'cli/config_command'
+require_relative 'cli/targets_command'
 
 class Constancy
   class CLI
@@ -56,6 +57,7 @@ Commands:
   push         Push changes from filesystem to Consul
   pull         Pull changes from Consul to filesystem
   config       Print a summary of the active configuration
+  targets      List target names
 
 General options:
   --help           Print help for the given command
@@ -65,6 +67,12 @@ General options:
 Options for 'check' command:
   --pull       Perform dry run in pull mode
 
+Options for 'pull' command:
+  --yes        Skip confirmation prompt
+
+Options for 'push' command:
+  --yes        Skip confirmation prompt
+
 USAGE
         exit 1
       end
@@ -122,9 +130,10 @@ USAGE
         when :command
           case self.command
           when 'check'      then Constancy::CLI::CheckCommand.run(self.extra_args)
-          when 'push'       then Constancy::CLI::PushCommand.run
-          when 'pull'       then Constancy::CLI::PullCommand.run
+          when 'push'       then Constancy::CLI::PushCommand.run(self.extra_args)
+          when 'pull'       then Constancy::CLI::PullCommand.run(self.extra_args)
           when 'config'     then Constancy::CLI::ConfigCommand.run
+          when 'targets'    then Constancy::CLI::TargetsCommand.run
           when nil          then self.print_usage
 
           else

data/lib/constancy/cli/config_command.rb

@@ -8,18 +8,25 @@ class Constancy
           Constancy::CLI.configure(call_external_apis: false)
 
           puts " Config file: #{Constancy.config.config_file}"
-          puts "  Consul URL: #{Constancy.config.consul.url}"
+          puts "  Consul URL: #{Constancy.config.consul_url}"
           puts "     Verbose: #{Constancy.config.verbose?.to_s.bold}"
-          if Constancy.config.consul_token_source == "env"
-            puts
-            puts "Token Source: CONSUL_TOKEN or CONSUL_HTTP_TOKEN environment variable"
-
-          elsif Constancy.config.consul_token_source == "vault"
+          puts
+          puts " Defined Consul Token Sources:"
+          default_src_name = Constancy.config.default_consul_token_source.name
+          srcs = Constancy.config.consul_token_sources
+          ( %w( none env ) + ( srcs.keys.sort - %w( none env ) ) ).each do |name|
             puts
-            puts "Token Source: Vault"
-            puts "   Vault URL: #{Constancy.config.vault_config.url}"
-            puts "  Token Path: #{Constancy.config.vault_config.consul_token_path}"
-            puts " Token Field: #{Constancy.config.vault_config.consul_token_field}"
+            puts "   #{name}:#{ default_src_name == name ? " (DEFAULT)".bold : ""}"
+            case name
+            when "none"
+              puts "     uses CONSUL_HTTP_TOKEN or CONSUL_TOKEN env var if available"
+            when "env"
+              puts "     requires CONSUL_HTTP_TOKEN or CONSUL_TOKEN env var"
+            when /^vault/
+              puts "     address: #{srcs[name].vault_addr}"
+              puts "        path: #{srcs[name].consul_token_path}"
+              puts "       field: #{srcs[name].consul_token_field}"
+            end
           end
           puts
           puts "Sync target defaults:"
@@ -35,16 +42,17 @@ class Constancy
             else
               print '*'
             end
-            puts " Datacenter: #{target.datacenter}"
-            puts "  Local type: #{target.type == :dir ? 'Directory' : 'Single file'}"
-            puts "   #{target.type == :dir ? " Dir" : "File"} path: #{target.path}"
-            puts "      Prefix: #{target.prefix}"
-            puts "   Autochomp? #{target.chomp?}"
-            puts "      Delete? #{target.delete?}"
+            puts "   Datacenter: #{target.datacenter}"
+            puts "    Local type: #{target.type == :dir ? 'Directory' : 'Single file'}"
+            puts "     #{target.type == :dir ? " Dir" : "File"} path: #{target.path}"
+            puts "        Prefix: #{target.prefix}"
+            puts "  Token Source: #{target.token_source.name}"
+            puts "     Autochomp? #{target.chomp?}"
+            puts "        Delete? #{target.delete?}"
             if not target.exclude.empty?
-              puts "  Exclusions:"
+              puts "    Exclusions:"
               target.exclude.each do |exclusion|
-                puts "    - #{exclusion}"
+                puts "      - #{exclusion}"
               end
             end
             puts

data/lib/constancy/cli/pull_command.rb

@@ -4,7 +4,7 @@ class Constancy
   class CLI
     class PullCommand
       class << self
-        def run
+        def run(args)
           Constancy::CLI.configure
           STDOUT.sync = true
 
@@ -22,7 +22,7 @@ class Constancy
             puts
             puts "Do you want to pull these changes?"
             print "  Enter '" + "yes".bold + "' to continue: "
-            answer = gets.chomp
+            answer = args.include?('--yes') ? 'yes' : gets.chomp
 
             if answer.downcase != "yes"
               puts

data/lib/constancy/cli/push_command.rb

@@ -4,7 +4,7 @@ class Constancy
   class CLI
     class PushCommand
       class << self
-        def run
+        def run(args)
           Constancy::CLI.configure
           STDOUT.sync = true
 
@@ -22,7 +22,7 @@ class Constancy
             puts
             puts "Do you want to push these changes?"
             print "  Enter '" + "yes".bold + "' to continue: "
-            answer = gets.chomp
+            answer = args.include?('--yes') ? 'yes' : gets.chomp
 
             if answer.downcase != "yes"
               puts

data/lib/constancy/cli/targets_command.rb

@@ -0,0 +1,21 @@
+# This software is public domain. No rights are reserved. See LICENSE for more information.
+
+class Constancy
+  class CLI
+    class TargetsCommand
+      class << self
+        def run
+          Constancy::CLI.configure(call_external_apis: false)
+
+          Constancy.config.sync_targets.each do |target|
+            if target.name
+              puts target.name
+            else
+              puts "[unnamed target] #{target.datacenter}:#{target.prefix}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/constancy/config.rb

@@ -11,6 +11,8 @@ class Constancy
   class Config
     CONFIG_FILENAMES = %w( constancy.yml )
     VALID_CONFIG_KEYS = %w( sync consul vault constancy )
+    VALID_VAULT_KEY_PATTERNS = [ %r{^vault\.[A-Za-z][A-Za-z0-9_-]*$}, %r{^vault$} ]
+    VALID_CONFIG_KEY_PATTERNS = VALID_VAULT_KEY_PATTERNS
     VALID_CONSUL_CONFIG_KEYS = %w( url datacenter token_source )
     VALID_VAULT_CONFIG_KEYS = %w( url consul_token_path consul_token_field )
     VALID_CONSTANCY_CONFIG_KEYS = %w( verbose chomp delete color )
@@ -18,7 +20,8 @@ class Constancy
     DEFAULT_CONSUL_TOKEN_SOURCE = "none"
     DEFAULT_VAULT_CONSUL_TOKEN_FIELD = "token"
 
-    attr_accessor :config_file, :base_dir, :consul, :consul_token_source, :sync_targets, :target_whitelist, :call_external_apis, :vault_config
+    attr_accessor :config_file, :base_dir, :consul_url, :default_consul_token_source,
+      :sync_targets, :target_allowlist, :call_external_apis, :consul_token_sources
 
     class << self
       # discover the nearest config file
@@ -34,6 +37,15 @@ class Constancy
 
         dir == "/" ? nil : self.discover(dir: File.dirname(dir))
       end
+
+      def only_valid_config_keys!(keylist)
+        (keylist - VALID_CONFIG_KEYS).each do |key|
+          if not VALID_CONFIG_KEY_PATTERNS.find { |pattern| key =~ pattern }
+            raise Constancy::ConfigFileInvalid.new("'#{key}' is not a valid configuration key")
+          end
+        end
+        true
+      end
     end
 
     def initialize(path: nil, targets: nil, call_external_apis: true)
@@ -51,7 +63,7 @@ class Constancy
 
       self.config_file = File.expand_path(self.config_file)
       self.base_dir = File.dirname(self.config_file)
-      self.target_whitelist = targets
+      self.target_allowlist = targets
       self.call_external_apis = call_external_apis
       parse!
     end
@@ -72,7 +84,11 @@ class Constancy
       @use_color
     end
 
-    private
+    def parse_vault_token_sources!(raw)
+      raw.keys.select { |key| VALID_VAULT_KEY_PATTERNS.find { |pattern| key =~ pattern } }.collect do |key|
+        [key, Constancy::VaultTokenSource.new(name: key, config: raw[key])]
+      end.to_h
+    end
 
     def parse!
       raw = {}
@@ -91,116 +107,40 @@ class Constancy
         raise Constancy::ConfigFileInvalid.new("Config file must form a hash")
       end
 
-      if (raw.keys - Constancy::Config::VALID_CONFIG_KEYS) != []
-        raise Constancy::ConfigFileInvalid.new("Only the following keys are valid at the top level of the config: #{Constancy::Config::VALID_CONFIG_KEYS.join(", ")}")
-      end
+      Constancy::Config.only_valid_config_keys!(raw.keys)
+
+      self.consul_token_sources = {
+        "none" => Constancy::PassiveTokenSource.new,
+        "env" => Constancy::EnvTokenSource.new,
+      }.merge(
+        self.parse_vault_token_sources!(raw),
+      )
 
       raw['consul'] ||= {}
       if not raw['consul'].is_a? Hash
         raise Constancy::ConfigFileInvalid.new("'consul' must be a hash")
       end
 
-      if (raw['consul'].keys - Constancy::Config::VALID_CONSUL_CONFIG_KEYS) != []
-        raise Constancy::ConfigFileInvalid.new("Only the following keys are valid in the consul config: #{Constancy::Config::VALID_CONSUL_CONFIG_KEYS.join(", ")}")
+      if (raw['consul'].keys - VALID_CONSUL_CONFIG_KEYS) != []
+        raise Constancy::ConfigFileInvalid.new("Only the following keys are valid in the consul config: #{VALID_CONSUL_CONFIG_KEYS.join(", ")}")
       end
 
-      consul_url = raw['consul']['url'] || Constancy::Config::DEFAULT_CONSUL_URL
-
-      # start with a token from the environment, regardless of the token_source setting
-      consul_token = ENV['CONSUL_HTTP_TOKEN'] || ENV['CONSUL_TOKEN']
-
-      self.consul_token_source = raw['consul']['token_source'] || Constancy::Config::DEFAULT_CONSUL_TOKEN_SOURCE
-      case self.consul_token_source
-      when "none"
-        # nothing to do
-
-      when "env"
-        if consul_token.nil? or consul_token == ""
-          raise Constancy::ConsulTokenRequired.new("Consul token_source is set to 'env' but neither CONSUL_TOKEN nor CONSUL_HTTP_TOKEN is set")
-        end
-
-      when "vault"
-        require 'vault'
-
-        raw['vault'] ||= {}
-        if not raw['vault'].is_a? Hash
-          raise Constancy::ConfigFileInvalid.new("'vault' must be a hash")
-        end
-
-        if (raw['vault'].keys - Constancy::Config::VALID_VAULT_CONFIG_KEYS) != []
-          raise Constancy::ConfigFileInvalid.new("Only the following keys are valid in the vault config: #{Constancy::Config::VALID_VAULT_CONFIG_KEYS.join(", ")}")
-        end
-
-        vault_path = raw['vault']['consul_token_path']
-        if vault_path.nil? or vault_path == ""
-          raise Constancy::ConfigFileInvalid.new("vault.consul_token_path must be specified to use vault as a token source")
-        end
-
-        # prioritize the config file over environment variables for vault address
-        vault_addr = raw['vault']['url'] || ENV['VAULT_ADDR']
-        if vault_addr.nil? or vault_addr == ""
-          raise Constancy::VaultConfigInvalid.new("Vault address must be set in vault.url or VAULT_ADDR")
-        end
-
-        vault_token = ENV['VAULT_TOKEN']
-        if vault_token.nil? or vault_token == ""
-          vault_token_file = File.expand_path("~/.vault-token")
-          if File.exist?(vault_token_file)
-            vault_token = File.read(vault_token_file)
-          else
-            raise Constancy::VaultConfigInvalid.new("Vault token must be set in ~/.vault-token or VAULT_TOKEN")
+      self.consul_url = raw['consul']['url'] || DEFAULT_CONSUL_URL
+      srcname = raw['consul']['token_source'] || DEFAULT_CONSUL_TOKEN_SOURCE
+      self.default_consul_token_source =
+        self.consul_token_sources[srcname].tap do |src|
+          if src.nil?
+            raise Constancy::ConfigFileInvalid.new("Consul token source '#{consul_token_source}' is not defined")
           end
         end
 
-        vault_field = raw['vault']['consul_token_field'] || Constancy::Config::DEFAULT_VAULT_CONSUL_TOKEN_FIELD
-
-        self.vault_config = OpenStruct.new(
-          url: vault_addr,
-          consul_token_path: vault_path,
-          consul_token_field: vault_field,
-        )
-
-        # don't waste time talking to Vault if this is just a config parsing run
-        if self.call_external_apis
-          ENV['VAULT_ADDR'] = vault_addr
-          ENV['VAULT_TOKEN'] = vault_token
-
-          begin
-            response = Vault.logical.read(vault_path)
-            consul_token = response.data[vault_field.to_sym]
-
-            if response.lease_id
-              at_exit {
-                begin
-                  Vault.sys.revoke(response.lease_id)
-                rescue => e
-                  # this is fine
-                end
-              }
-            end
-
-          rescue => e
-            raise Constancy::VaultConfigInvalid.new("Are you logged in to Vault?\n\n#{e}")
-          end
-
-          if consul_token.nil? or consul_token == ""
-            raise Constancy::VaultConfigInvalid.new("Could not acquire a Consul token from Vault")
-          end
-        end
-
-      else
-        raise Constancy::ConfigFileInvalid.new("Only the following values are valid for token_source: none, env, vault")
-      end
-
-      self.consul = Imperium::Configuration.new(url: consul_url, token: consul_token)
-
       raw['constancy'] ||= {}
       if not raw['constancy'].is_a? Hash
         raise Constancy::ConfigFileInvalid.new("'constancy' must be a hash")
       end
 
-      if (raw['constancy'].keys - Constancy::Config::VALID_CONSTANCY_CONFIG_KEYS) != []
-        raise Constancy::ConfigFileInvalid.new("Only the following keys are valid in the 'constancy' config block: #{Constancy::Config::VALID_CONSTANCY_CONFIG_KEYS.join(", ")}")
+      if (raw['constancy'].keys - VALID_CONSTANCY_CONFIG_KEYS) != []
+        raise Constancy::ConfigFileInvalid.new("Only the following keys are valid in the 'constancy' config block: #{VALID_CONSTANCY_CONFIG_KEYS.join(", ")}")
       end
 
       # verbose: default false
@@ -233,6 +173,7 @@ class Constancy
 
       self.sync_targets = []
       raw['sync'].each do |target|
+        token_source = self.default_consul_token_source
         if target.is_a? Hash
           target['datacenter'] ||= raw['consul']['datacenter']
           if target['chomp'].nil?
@@ -241,19 +182,31 @@ class Constancy
           if target['delete'].nil?
             target['delete'] = self.delete?
           end
+          if not target['token_source'].nil?
+            token_source = self.consul_token_sources[target['token_source']]
+            if token_source.nil?
+              raise Constancy::ConfigFileInvalid.new("Consul token source '#{target['token_source']}' is not defined")
+            end
+            target.delete('token_source')
+          end
         end
 
-        if not self.target_whitelist.nil?
-          # unnamed targets cannot be whitelisted
+        if not self.target_allowlist.nil?
+          # unnamed targets cannot be allowlisted
           next if target['name'].nil?
 
-          # named targets must be on the whitelist
-          next if not self.target_whitelist.include?(target['name'])
+          # named targets must be on the allowlist
+          next if not self.target_allowlist.include?(target['name'])
         end
 
-        self.sync_targets << Constancy::SyncTarget.new(config: target, imperium_config: self.consul, base_dir: self.base_dir)
+        # only try to fetch consul tokens if we are actually going to do work
+        consul_token = if self.call_external_apis
+                         token_source.consul_token
+                       else
+                         ""
+                       end
+        self.sync_targets << Constancy::SyncTarget.new(config: target, consul_url: consul_url, token_source: token_source, base_dir: self.base_dir, call_external_apis: self.call_external_apis)
       end
-
     end
   end
 end

data/lib/constancy/diff.rb

@@ -36,7 +36,7 @@ class Constancy
             end
           end
 
-          consul_key = [@target.prefix, key].compact.join("/")
+          consul_key = [@target.prefix, key].compact.join("/").squeeze("/")
 
           if @target.exclude.include?(key) or @target.exclude.include?(consul_key)
             op = :ignore

data/lib/constancy/sync_target.rb

@@ -2,14 +2,14 @@
 
 class Constancy
   class SyncTarget
-    VALID_CONFIG_KEYS = %w( name type datacenter prefix path exclude chomp delete )
-    attr_accessor :name, :type, :datacenter, :prefix, :path, :exclude, :consul
+    VALID_CONFIG_KEYS = %w( name type datacenter prefix path exclude chomp delete erb_enabled )
+    attr_accessor :name, :type, :datacenter, :prefix, :path, :exclude, :consul, :erb_enabled, :consul_url, :token_source, :call_external_apis
 
     REQUIRED_CONFIG_KEYS = %w( prefix )
     VALID_TYPES = [ :dir, :file ]
     DEFAULT_TYPE = :dir
 
-    def initialize(config:, imperium_config:, base_dir:)
+    def initialize(config:, consul_url:, token_source:, base_dir:, call_external_apis: true)
       if not config.is_a? Hash
         raise Constancy::ConfigFileInvalid.new("Sync target entries must be specified as hashes")
       end
@@ -46,7 +46,20 @@ class Constancy
         @do_delete = false
       end
 
-      self.consul = Imperium::KV.new(imperium_config)
+      self.call_external_apis = call_external_apis
+      self.consul_url = consul_url
+      self.token_source = token_source
+      token = if self.call_external_apis
+                self.token_source.consul_token
+              else
+                ""
+              end
+      self.consul = Imperium::KV.new(Imperium::Configuration.new(url: self.consul_url, token: token))
+      self.erb_enabled = config['erb_enabled']
+    end
+
+    def erb_enabled?
+      @erb_enabled
     end
 
     def chomp?
@@ -89,22 +102,41 @@ class Constancy
       when :dir
         self.local_files.each do |local_file|
           @local_items[local_file.sub(%r{^#{self.base_path}/?}, '')] =
-            if self.chomp?
-              File.read(local_file).chomp.force_encoding(Encoding::ASCII_8BIT)
-            else
-              File.read(local_file).force_encoding(Encoding::ASCII_8BIT)
-            end
+            load_local_file(local_file)
         end
 
       when :file
         if File.exist?(self.base_path)
-          @local_items = YAML.load_file(self.base_path)
+          @local_items = local_items_from_file
         end
       end
 
       @local_items
     end
 
+    def local_items_from_file
+      if erb_enabled?
+        loaded_file = YAML.load(ERB.new(File.read(self.base_path)).result)
+      else
+        loaded_file = YAML.load_file(self.base_path)
+      end
+
+      flatten_hash(nil, loaded_file)
+    end
+
+    def load_local_file(local_file)
+      file = File.read(local_file)
+
+      if self.chomp?
+        encoded_file = file.chomp.force_encoding(Encoding::ASCII_8BIT)
+      else
+        encoded_file = file.force_encoding(Encoding::ASCII_8BIT)
+      end
+
+      return ERB.new(encoded_file).result if erb_enabled?
+      encoded_file
+    end
+
     def remote_items
       return @remote_items if not @remote_items.nil?
       @remote_items = {}
@@ -122,5 +154,26 @@ class Constancy
     def diff(mode)
       Constancy::Diff.new(target: self, local: self.local_items, remote: self.remote_items, mode: mode)
     end
+
+    private def flatten_hash(prefix, hash)
+      new_hash = {}
+
+      hash.each do |k, v|
+        if k == '_' && !prefix.nil?
+          new_key = prefix
+        else
+          new_key = [prefix, k].compact.join('/')
+        end
+
+        case v
+        when Hash
+          new_hash.merge!(flatten_hash(new_key, v))
+        else
+          new_hash[new_key] = v.to_s
+        end
+      end
+
+      new_hash
+    end
   end
 end

data/lib/constancy/token_source.rb

@@ -0,0 +1,94 @@
+# This software is public domain. No rights are reserved. See LICENSE for more information.
+
+class Constancy
+  # use env vars if defined, but otherwise just return an empty string
+  class PassiveTokenSource
+    def name
+      "none"
+    end
+
+    def consul_token
+      ENV['CONSUL_HTTP_TOKEN'] || ENV['CONSUL_TOKEN'] || ""
+    end
+  end
+
+  # use env vars and raise an error if none is found
+  class EnvTokenSource
+    def name
+      "env"
+    end
+
+    def consul_token
+      consul_token = ENV['CONSUL_HTTP_TOKEN'] || ENV['CONSUL_TOKEN']
+      if consul_token.nil? or consul_token == ""
+        raise Constancy::ConsulTokenRequired.new("Consul token_source was set to 'env' but neither CONSUL_TOKEN nor CONSUL_HTTP_TOKEN is set")
+      end
+    end
+  end
+
+  class VaultTokenSource
+    attr_accessor :name, :vault_addr, :vault_token, :consul_token_path, :consul_token_field
+
+    def initialize(name:, config:)
+      self.name = name
+
+      config ||= {}
+      if not config.is_a? Hash
+        raise Constancy::ConfigFileInvalid.new("'#{name}' must be a hash")
+      end
+
+      if (config.keys - Constancy::Config::VALID_VAULT_CONFIG_KEYS) != []
+        raise Constancy::ConfigFileInvalid.new("Only the following keys are valid in a vault config: #{Constancy::Config::VALID_VAULT_CONFIG_KEYS.join(", ")}")
+      end
+
+      self.consul_token_path = config['consul_token_path']
+      if self.consul_token_path.nil? or self.consul_token_path == ""
+        raise Constancy::ConfigFileInvalid.new("consul_token_path must be specified to use '#{name}' as a token source")
+      end
+
+      # prioritize the config file over environment variables for vault address
+      self.vault_addr = config['url'] || ENV['VAULT_ADDR']
+      if self.vault_addr.nil? or self.vault_addr == ""
+        raise Constancy::VaultConfigInvalid.new("Vault address must be set in #{name}.vault_addr or VAULT_ADDR")
+      end
+
+      self.vault_token = ENV['VAULT_TOKEN']
+      if self.vault_token.nil? or self.vault_token == ""
+        vault_token_file = File.expand_path("~/.vault-token")
+        if File.exist?(vault_token_file)
+          self.vault_token = File.read(vault_token_file)
+        else
+          raise Constancy::VaultConfigInvalid.new("Vault token must be set in ~/.vault-token or VAULT_TOKEN")
+        end
+      end
+
+      self.consul_token_field = config['consul_token_field'] || Constancy::Config::DEFAULT_VAULT_CONSUL_TOKEN_FIELD
+    end
+
+    def consul_token
+      if @consul_token.nil?
+        begin
+          response = Vault::Client.new(address: self.vault_addr, token: self.vault_token).logical.read(self.consul_token_path)
+          @consul_token = response.data[self.consul_token_field.to_sym]
+          if response.lease_id
+            at_exit {
+              begin
+                Vault::Client.new(address: self.vault_addr, token: self.vault_token).sys.revoke(response.lease_id)
+              rescue => e
+                # this is fine
+              end
+            }
+          end
+
+        rescue => e
+          raise Constancy::VaultConfigInvalid.new("Are you logged in to Vault?\n\n#{e}")
+        end
+
+        if @consul_token.nil? or @consul_token == ""
+          raise Constancy::VaultConfigInvalid.new("Could not acquire a Consul token from Vault")
+        end
+      end
+      @consul_token
+    end
+  end
+end

data/lib/constancy/version.rb

@@ -1,5 +1,5 @@
 # This software is public domain. No rights are reserved. See LICENSE for more information.
 
 class Constancy
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: constancy
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - David Adams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-07 00:00:00.000000000 Z
+date: 2020-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: imperium
@@ -83,9 +83,11 @@ files:
 - lib/constancy/cli/config_command.rb
 - lib/constancy/cli/pull_command.rb
 - lib/constancy/cli/push_command.rb
+- lib/constancy/cli/targets_command.rb
 - lib/constancy/config.rb
 - lib/constancy/diff.rb
 - lib/constancy/sync_target.rb
+- lib/constancy/token_source.rb
 - lib/constancy/version.rb
 homepage: https://github.com/daveadams/constancy
 licenses:
@@ -106,8 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Simple filesystem-to-Consul KV synchronization