tfwrapper 0.4.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4033a083750714db82cc06f4c948aa3539c1f608
-  data.tar.gz: bda28f44bd9e007263ca8c4d378e29209a4f1ffa
+SHA256:
+  metadata.gz: 4a404a81c05bcc07e29ef34eef1d881aa9c47faa1e5ba0ed127a34b8ae1202c5
+  data.tar.gz: f0078c62815fd5d7e49c14d201d9da2154ed91f6c4786a7a7be65914582cd4d8
 SHA512:
-  metadata.gz: 7f9c16e12f2f3927003eca182bf31e8ee1c9768e921a8f328424c6c0c9c522874b48c30ca1fd0f16e6d9eea7873310886a8e217083fd0602b4d642e497684eca
-  data.tar.gz: 8481b9c9e6975710e5525b5d27849a0a94a4aec7dd53749dfe8764b719edc0cbbede1489dccc214d83128b5dbae4c62a12fc9178317e6d6377c8a1419c49cb29
+  metadata.gz: db129accb7e439c570ca8a8eb2d56a259acb98221a5acf5e451cc4aa44516627abd6e29603574abbf4dcd16c968b58607552bbec3a1b50ce7df44de9659968a7
+  data.tar.gz: f9f99bcc2e51e2aa4cedabd0e1179a1954dbc3e3e4d44149ab2d171218cc6e55a940ef2372869c6cd694c77be0282d5230f7a7d142eaea53479794c78550e810

data/.gitignore

@@ -13,6 +13,7 @@
 /vendor/
 .terraform
 build.tfvars.json
+**/*_build.tfvars.json
 **/terraform.tfstate.backup
 
 # Used by dotenv library to load environment variables.

data/.rubocop.yml

@@ -2,10 +2,10 @@ AllCops:
   TargetRubyVersion: 2.0
 
 Metrics/AbcSize:
-  Max: 40
+  Max: 50
 
 Metrics/BlockLength:
-  Max: 1300
+  Max: 1500
 
 Metrics/ClassLength:
   Max: 500
@@ -14,10 +14,10 @@ Metrics/MethodLength:
   Max: 100
 
 Metrics/PerceivedComplexity:
-  Max: 8
+  Max: 10
 
 Metrics/CyclomaticComplexity:
-  Max: 8
+  Max: 10
 
 # These are to compensate for some warnings that differ by
 # rubocop/ruby version
@@ -30,7 +30,7 @@ Style/MutableConstant:
     - 'lib/tfwrapper/version.rb'
     - 'spec/acceptance/acceptance_spec.rb'
 
-Style/MultilineMethodCallIndentation:
+Layout/MultilineMethodCallIndentation:
   Exclude:
     - 'spec/unit/raketasks_spec.rb'
 

data/.travis.yml

@@ -0,0 +1,62 @@
+---
+dist: xenial
+language: ruby
+cache: bundler
+before_install:
+  - gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true
+  - gem install bundler -v '< 2'
+install: true
+matrix:
+  include:
+  - rvm: ruby
+    install: bundle install --without landscape
+    script: ["bundle exec ruby --version", "bundle exec rake spec:unit"]
+    name: "ruby-latest unit"
+  - rvm: ruby
+    install: bundle install --without landscape
+    script: bundle exec rake rubocop
+    name: "ruby-latest rubocop"
+  - rvm: ruby
+    install: bundle install --without landscape
+    script: bundle exec rake yard:generate
+    name: "ruby-latest yard"
+  - rvm: 2.0
+    install: bundle install --without landscape
+    script: bundle exec rake spec:unit
+    name: "ruby-2.0 unit"
+  - rvm: 2.1
+    install: bundle install --without landscape
+    script: bundle exec rake spec:unit
+    name: "ruby-2.1 unit"
+  - rvm: 2.2
+    install: bundle install --without landscape
+    script: bundle exec rake spec:unit
+    name: "ruby-2.2 unit"
+  - rvm: 2.3
+    install: bundle install --without landscape
+    script: bundle exec rake spec:unit
+    name: "ruby-2.3 unit"
+  - rvm: 2.3
+    install: bundle install
+    script: bundle exec rake spec:acceptance TF_VERSION=0.10.0
+    name: "ruby-2.3 acceptance TF 0.10.0"
+  - rvm: 2.3
+    install: bundle install
+    script: bundle exec rake spec:acceptance TF_VERSION=0.10.2
+    name: "ruby-2.3 acceptance TF 0.10.2"
+  - rvm: 2.3
+    install: bundle install
+    script: bundle exec rake spec:acceptance TF_VERSION=0.11.2
+    name: "ruby-2.3 acceptance TF 0.11.2"
+  - rvm: 2.3
+    install: bundle install
+    script: bundle exec rake spec:acceptance TF_VERSION=0.11.14
+    name: "ruby-2.3 acceptance TF 0.11.14"
+  - rvm: 2.4
+    install: bundle install
+    script: bundle exec rake spec:unit
+    name: "ruby-2.4 unit"
+  - rvm: 2.4
+    install: bundle install
+    script: bundle exec rake spec:acceptance TF_VERSION=0.11.14
+    name: "ruby-2.4 acceptance TF 0.11.14"

data/ChangeLog.md

@@ -1,3 +1,34 @@
+Version 0.6.2
+
+  - Add ``tf_sensitive_vars`` option.
+
+Version 0.6.1
+
+  - Add ``allowed_empty_vars`` option.
+  - Only support terraform up to 0.11.14.
+
+Version 0.6.0
+
+  - Include full terraform output when a terraform run fails with output including ``hrottling``, ``status code: 403``, or ``status code: 401``. Previously terraform output was suppressed in these cases, which causes confusion with providers other than "aws" that include these strings in their failure output.
+  - In ``Gemfile``, for testing, pin terraform-landscape gem to 0.2.2 for Ruby 2.x compatibility.
+  - Pin ``rb-inotify`` dependency to 0.9.10 to allow build with Ruby < 2.2
+  - Switch testing from circleci.com to travis-ci.org
+  - Pin terraform_landscape dependency to 0.2.2 for compatibility with ruby < 2.5
+  - Acceptance tests:
+    - Use HashiCorp checkpoint API instead of GitHub Releases API to find latest terraform version, to work around query failures
+    - Pin consul provider versions to 1.0.0 to fix intermittent failures
+  - Switch Ruby versions used in tests to latest TravisCI versions
+  - Stop acceptance testing terraform 0.9.x, as newer versions require pinning the consul provider version but 0.9 doesn't support versioned providers.
+
+Version 0.5.1
+
+  - Fix bug where terraform plan errors were suppressed if a plan run with landscape support enabled exited non-zero.
+
+Version 0.5.0
+
+  - Add support for using terraform_landscape gem, if present, to reformat plan output; see README for usage.
+  - Add CircleCI testing under ruby 2.4.1, and acceptance tests for terraform 0.11.2.
+
 Version 0.4.1
 
   - Upgrade rubocop, yard and diplomat development dependency versions

data/Gemfile

@@ -2,3 +2,7 @@
 
 source 'https://rubygems.org'
 gemspec
+
+group :landscape do
+  gem 'terraform_landscape', '0.2.2'
+end

data/README.md

@@ -1,10 +1,10 @@
 # tfwrapper
 
-Build of master branch: [![CircleCI](https://circleci.com/gh/manheim/tfwrapper.svg?style=svg)](https://circleci.com/gh/manheim/tfwrapper)
+Build of master branch: [![TravisCI](https://api.travis-ci.org/manheim/tfwrapper.svg?branch=master)](https://travis-ci.org/manheim/tfwrapper)
 
 Documentation: [http://www.rubydoc.info/gems/tfwrapper/](http://www.rubydoc.info/gems/tfwrapper/)
 
-tfwrapper provides Rake tasks for working with [Hashicorp Terraform](https://www.terraform.io/) 0.9+, ensuring proper initialization and passing in variables from the environment or Ruby, as well as optionally pushing some information to Consul. tfwrapper also attempts to detect and retry
+tfwrapper provides Rake tasks for working with [Hashicorp Terraform](https://www.terraform.io/) 0.9 through 0.11, ensuring proper initialization and passing in variables from the environment or Ruby, as well as optionally pushing some information to Consul. tfwrapper also attempts to detect and retry
 failed runs due to AWS throttling or access denied errors.
 
 ## Overview
@@ -18,7 +18,9 @@ This Gem provides the following Rake tasks:
   one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
   terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:plan[aws_instance.foo[1]]`` or
   ``bundle exec rake tf:plan[aws_instance.foo[1],aws_instance.bar[2]]``; see the
-  [plan documentation](https://www.terraform.io/docs/commands/plan.html) for more information.
+  [plan documentation](https://www.terraform.io/docs/commands/plan.html) for more information. By default this will use
+  [terraform_landscape](https://github.com/coinbase/terraform-landscape/blob/master/README.md) for formatting the plan
+  output if the ``terraform_landscape`` gem is installed; see the [section below](#terraform-landscape) for more information.
 * __tf:apply__ - run ``terraform apply`` with all variables and configuration, and TF variables written to disk. You can specify
   one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
   terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:apply[aws_instance.foo[1]]`` or
@@ -42,12 +44,29 @@ with 1.9.3 is simply too high to justify.
 Add to your ``Gemfile``:
 
 ```ruby
-gem 'tfwrapper', '~> 0.2.0'
+gem 'tfwrapper', '~> 0.6.1'
 ```
 
 ### Supported Terraform Versions
 
-tfwrapper only supports terraform 0.9+. It is tested against multiple versions from 0.9.2 to 0.10.2 and the current release.
+tfwrapper only supports terraform 0.9-0.11. It is tested against multiple versions from 0.9.2 to 0.11.14.
+
+### Terraform Landscape
+
+The [terraform_landscape](https://github.com/coinbase/terraform-landscape/blob/master/README.md) gem provides enhanced formatting of ``terraform plan`` output including colorization of changes and human-friendly diffs (i.e. diffs of JSON rendered with pretty-printing). By default ``plan`` output will be passed through terraform_landscape if the ``terraform_landscape`` gem is available. To enable this, add ``gem 'terraform_landscape', '~> 0.1.17'`` to your ``Gemfile``. __Note__ that we rely on an undocumented internal API of terraform_landscape to achieve this; the formatting code will fall back to unformatted output in case of an error.
+
+If you wish to disable terraform_landscape output even when the gem is installed, pass ``disable_landscape: true`` as an option to ``install_tasks()``:
+
+```ruby
+TFWrapper::RakeTasks.install_tasks('.', disable_landscape: true)
+```
+
+In previous versions or when terraform_landscape is not installed, the output of all terraform commands is streamed in realtime. Since terraform_landscape requires the full and complete ``plan`` output in order to reformat it, this is no longer the case. By default when terraform_landscape is installed and not disabled the ``plan`` task will not produce any output until complete, at which point it will print the landscape-formatted output all at once. This behavior can be controlled with the ``:landscape_progress`` option of ``install_tasks()``, which takes one of the following values:
+
+* ``nil``, the default, to not produce any output until the command is complete at which point the landscape-formatted output will be shown.
+* ``:dots`` to print a dot to STDOUT for every line of ``terraform plan`` output and then print the landscape-formatted output when complete.
+* ``:lines`` to print a dot followed by a newline to STDOUT for every line of ``terraform plan`` output and then print the landscape-formatted output when complete. This is useful for systems like Jenkins that line-buffer output (and don't display anything until a newline is encountered).
+* ``:stream`` to stream the ``terraform plan`` output in realtime (as was the previous behavior) and then print the landscape-formatted output when complete. Note that this will result in the output containing the complete unformatted ``terraform plan`` output, followed by the landscape-formatted output.
 
 ## Usage
 
@@ -128,6 +147,10 @@ TFWrapper::RakeTasks.install_tasks(
 )
 ```
 
+These variables are tested to be set in the environment with a non-empty value, and will raise an error if any are
+missing or empty. If some should be allowed to be missing or empty empty, pass a ``allowed_empty_vars`` list with
+their environment variable names.
+
 ### Ruby Variables to Terraform Variables
 
 If you wish to explicitly bind values from your Ruby code to terraform variables, you can do this with
@@ -307,6 +330,26 @@ $ consul kv get terraform/inputs/foo
 {"FOO":"one", "BAR":"two"}
 ```
 
+### Sensitive Environment Variables
+If you wish for certain variables to be marked as "redacted", use the ``tf_sensitive_vars`` option. This is an array of variables that will not be printed.
+
+Note: ``aws_access_key`` and ``aws_secret_key`` will always be redacted without requiring configuration.
+
+
+Example to redact the vaule for ``secret``:
+
+Rakefile:
+
+```ruby
+require 'tfwrapper/raketasks'
+
+TFWrapper::RakeTasks.install_tasks(
+  '.',
+  tf_vars_from_env: {'foo' => 'FOO', 'bar' => 'BAR', 'secret' => 'abc'},
+  tf_sensitive_vars: ['secret']
+)
+```
+
 ## Development
 
 1. ``bundle install --path vendor``

data/lib/tfwrapper/helpers.rb

@@ -31,8 +31,20 @@ module TFWrapper
     #
     # @param cmd [String] command to run
     # @param pwd [String] directory/path to run command in
+    # @option opts [Hash] :progress How to handle streaming output. Possible
+    #  values are ``:stream`` (default) to stream each line in STDOUT/STDERR
+    #  to STDOUT, ``:dots`` to print a dot for each line, ``:lines`` to print
+    #  a dot followed by a newline for each line, or ``nil`` to not stream any
+    #  output at all.
     # @return [Array] - out_err [String], exit code [Fixnum]
-    def self.run_cmd_stream_output(cmd, pwd)
+    def self.run_cmd_stream_output(cmd, pwd, opts = {})
+      stream_type = opts.fetch(:progress, :stream)
+      unless [:dots, :lines, :stream, nil].include?(stream_type)
+        raise(
+          ArgumentError,
+          'progress option must be one of: [:dots, :lines, :stream, nil]'
+        )
+      end
       old_sync = $stdout.sync
       $stdout.sync = true
       all_out_err = ''.dup
@@ -41,7 +53,13 @@ module TFWrapper
         stdin.close_write
         begin
           while (line = stdout_and_err.gets)
-            puts line
+            if stream_type == :stream
+              puts line
+            elsif stream_type == :dots
+              STDOUT.print '.'
+            elsif stream_type == :lines
+              puts '.'
+            end
             all_out_err << line
           end
         rescue IOError => e
@@ -51,6 +69,7 @@ module TFWrapper
       end
       # rubocop:disable Style/RedundantReturn
       $stdout.sync = old_sync
+      puts '' if stream_type == :dots
       return all_out_err, exit_status
       # rubocop:enable Style/RedundantReturn
     end
@@ -59,9 +78,12 @@ module TFWrapper
     # non-empty. Raise StandardError if any aren't.
     #
     # @param required [Array] list of required environment variables
-    def self.check_env_vars(required)
+    # @param allowed_missing [Array] list of environment variables to allow to
+    #  be empty or missing.
+    def self.check_env_vars(required, allowed_missing)
       missing = []
       required.each do |name|
+        next if allowed_missing.include?(name)
         if !ENV.include?(name)
           puts "ERROR: Environment variable '#{name}' must be set."
           missing << name

data/lib/tfwrapper/raketasks.rb

@@ -6,6 +6,15 @@ require 'rake'
 require 'rubygems'
 require 'tfwrapper/version'
 
+# :nocov:
+begin
+  require 'terraform_landscape'
+  HAVE_LANDSCAPE = true
+rescue LoadError
+  HAVE_LANDSCAPE = false
+end
+# :nocov:
+
 module TFWrapper
   # Generates Rake tasks for working with Terraform.
   class RakeTasks
@@ -46,8 +55,12 @@ module TFWrapper
     #   configurations in the same Rakefile.
     # @option opts [Hash] :tf_vars_from_env hash of Terraform variables to the
     #   (required) environment variables to populate their values from
+    # @option opts [Hash] :allowed_empty_vars array of environment variable
+    #   names (specified in :tf_vars_from_env) to allow to be empty or missing.
     # @option opts [Hash] :tf_extra_vars hash of Terraform variables to their
     #   values; overrides any same-named keys in ``tf_vars_from_env``
+    # @option opts [Array] :tf_sensitive_vars list of Terraform variables
+    #   which should not be printed
     # @option opts [String] :consul_url URL to access Consul at, for the
     #   ``:consul_env_vars_prefix`` option.
     # @option opts [String] :consul_env_vars_prefix if specified and not nil,
@@ -63,6 +76,22 @@ module TFWrapper
     #   the body of each task. Called with two arguments, the String full
     #   (namespaced) name of the task being executed, and ``tf_dir``. This will
     #   not execute if the body of the task fails.
+    # @option opts [Bool] :disable_landscape By default, if the
+    #  ``terraform_landscape`` gem can be loaded, it will be used to reformat
+    #  the output of ``terraform plan``. If this is not desired, set to
+    #  ``true`` to disable landscale. Default: ``false``.
+    # @option opts [Symbol, nil] :landscape_progress The ``terraform_landscape``
+    #  code used to reformat plan output requires the full output of the
+    #  complete ``plan`` execution. By default, this means that when landscape
+    #  is used, no output will appear from the time ``terraform plan`` begins
+    #  until the command is complete. If progress output is desired, this option
+    #  can be set to one of the following: ``:dots`` to print a dot to STDOUT
+    #  for every line of ``terraform plan`` output, ``:lines`` to print a dot
+    #  followed by a newline (e.g. for systems like Jenkins that line buffer)
+    #  for every line of ``terraform plan`` output, or ``:stream`` to stream
+    #  the raw ``terraform plan`` output (which will then be followed by the
+    #  reformatted landscape output). Default is ``nil`` to show no progress
+    #  output.
     def initialize(tf_dir, opts = {})
       # find the directory that contains the Rakefile
       rakedir = File.realpath(Rake.application.rakefile)
@@ -71,9 +100,20 @@ module TFWrapper
       @ns_prefix = opts.fetch(:namespace_prefix, nil)
       @consul_env_vars_prefix = opts.fetch(:consul_env_vars_prefix, nil)
       @tf_vars_from_env = opts.fetch(:tf_vars_from_env, {})
+      @allowed_empty_vars = opts.fetch(:allowed_empty_vars, [])
+      @tf_sensitive_vars = opts.fetch(:tf_sensitive_vars, [])
       @tf_extra_vars = opts.fetch(:tf_extra_vars, {})
       @backend_config = opts.fetch(:backend_config, {})
       @consul_url = opts.fetch(:consul_url, nil)
+      @disable_landscape = opts.fetch(:disable_landscape, false)
+      @landscape_progress = opts.fetch(:landscape_progress, nil)
+      unless [:dots, :lines, :stream, nil].include?(@landscape_progress)
+        raise(
+          ArgumentError,
+          'landscape_progress option must be one of: ' \
+          '[:dots, :lines, :stream, nil]'
+        )
+      end
       @before_proc = opts.fetch(:before_proc, nil)
       if !@before_proc.nil? && !@before_proc.is_a?(Proc)
         raise(
@@ -128,7 +168,9 @@ module TFWrapper
         desc 'Run terraform init with appropriate arguments'
         task :init do |t|
           @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
-          TFWrapper::Helpers.check_env_vars(@tf_vars_from_env.values)
+          TFWrapper::Helpers.check_env_vars(
+            @tf_vars_from_env.values, @allowed_empty_vars
+          )
           @tf_version = check_tf_version
           cmd = [
             'terraform',
@@ -160,7 +202,13 @@ module TFWrapper
             args.extras
           )
 
-          terraform_runner(cmd)
+          stream_type = if HAVE_LANDSCAPE && !@disable_landscape
+                          @landscape_progress
+                        else
+                          :stream
+                        end
+          outerr = terraform_runner(cmd, progress: stream_type)
+          landscape_format(outerr) if HAVE_LANDSCAPE && !@disable_landscape
           @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
         end
       end
@@ -274,7 +322,9 @@ module TFWrapper
           tf_vars = terraform_vars
           puts 'Terraform vars:'
           tf_vars.sort.map do |k, v|
-            if %w[aws_access_key aws_secret_key].include?(k)
+            redacted_list = (%w[aws_access_key aws_secret_key] +
+                             @tf_sensitive_vars)
+            if redacted_list.include?(k)
               puts "#{k} => (redacted)"
             else
               puts "#{k} => #{v}"
@@ -296,13 +346,39 @@ module TFWrapper
       res
     end
 
+    # Given a string of terraform plan output, format it with
+    # terraform_landscape and print the result to STDOUT.
+    def landscape_format(output)
+      p = TerraformLandscape::Printer.new(
+        TerraformLandscape::Output.new(STDOUT)
+      )
+      p.process_string(output)
+    rescue StandardError, ScriptError => ex
+      STDERR.puts 'Exception calling terraform_landscape to reformat ' \
+                  "output: #{ex.class.name}: #{ex}"
+      puts output unless @landscape_progress == :stream
+    end
+
     # Run a Terraform command, providing some useful output and handling AWS
     # API rate limiting gracefully. Raises StandardError on failure. The command
     # is run in @tf_dir.
     #
     # @param cmd [String] Terraform command to run
+    # @option opts [Hash] :progress How to handle streaming output. Possible
+    #  values are ``:stream`` (default) to stream each line in STDOUT/STDERR
+    #  to STDOUT, ``:dots`` to print a dot for each line, ``:lines`` to print
+    #  a dot followed by a newline for each line, or ``nil`` to not stream any
+    #  output at all.
+    # @return [String] combined STDOUT and STDERR
     # rubocop:disable Metrics/PerceivedComplexity
-    def terraform_runner(cmd)
+    def terraform_runner(cmd, opts = {})
+      stream_type = opts.fetch(:progress, :stream)
+      unless [:dots, :lines, :stream, nil].include?(stream_type)
+        raise(
+          ArgumentError,
+          'progress option must be one of: [:dots, :lines, :stream, nil]'
+        )
+      end
       require 'retries'
       STDERR.puts "terraform_runner command: '#{cmd}' (in #{@tf_dir})"
       out_err = nil
@@ -321,25 +397,30 @@ module TFWrapper
       ) do
         # this streams STDOUT and STDERR as a combined stream,
         # and also captures them as a combined string
-        out_err, status = TFWrapper::Helpers.run_cmd_stream_output(cmd, @tf_dir)
+        out_err, status = TFWrapper::Helpers.run_cmd_stream_output(
+          cmd, @tf_dir, progress: stream_type
+        )
         if status != 0 && out_err.include?('hrottling')
-          raise StandardError, 'Terraform hit AWS API rate limiting'
+          raise StandardError, "#{out_err}\nTerraform hit AWS API rate limiting"
         end
         if status != 0 && out_err.include?('status code: 403')
-          raise StandardError, 'Terraform command got 403 error - access ' \
-            'denied or credentials not propagated'
+          raise StandardError, "#{out_err}\nTerraform command got 403 error " \
+            '- access denied or credentials not propagated'
         end
         if status != 0 && out_err.include?('status code: 401')
-          raise StandardError, 'Terraform command got 401 error - access ' \
-            'denied or credentials not propagated'
+          raise StandardError, "#{out_err}\nTerraform command got 401 error " \
+            '- access denied or credentials not propagated'
         end
       end
       # end exponential backoff
       unless status.zero?
+        # if we weren't streaming output, send it now
+        STDERR.puts out_err unless stream_type == :stream
         raise StandardError, "Errors have occurred executing: '#{cmd}' " \
           "(exited #{status})"
       end
       STDERR.puts "terraform_runner command '#{cmd}' finished and exited 0"
+      out_err
     end
     # rubocop:enable Metrics/PerceivedComplexity