tfwrapper 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4033a083750714db82cc06f4c948aa3539c1f608
-  data.tar.gz: bda28f44bd9e007263ca8c4d378e29209a4f1ffa
+  metadata.gz: 73a99a40047a490174b186e5bf5c6b58dedea114
+  data.tar.gz: f64131d4e1a68fae63dc3b86465b64480a3edae3
 SHA512:
-  metadata.gz: 7f9c16e12f2f3927003eca182bf31e8ee1c9768e921a8f328424c6c0c9c522874b48c30ca1fd0f16e6d9eea7873310886a8e217083fd0602b4d642e497684eca
-  data.tar.gz: 8481b9c9e6975710e5525b5d27849a0a94a4aec7dd53749dfe8764b719edc0cbbede1489dccc214d83128b5dbae4c62a12fc9178317e6d6377c8a1419c49cb29
+  metadata.gz: 7e3d6202be5a238dc9a358ed190aa6debbef62e5f56cac5b365935afe407a700e46a3166e24af34640f00a21667dd34adec292bb1c7125fe82129da98eefeeed
+  data.tar.gz: 1b4b83ca1147ae1955d3e571e087467f792db7107b76c2d9f421ff9fd37dd4512860ca719b32a7fd8b56e8861b1785ff45b439eea01faa454e4169b0dd1b040c

data/.gitignore

@@ -13,6 +13,7 @@
 /vendor/
 .terraform
 build.tfvars.json
+**/*_build.tfvars.json
 **/terraform.tfstate.backup
 
 # Used by dotenv library to load environment variables.

data/.rubocop.yml

@@ -2,10 +2,10 @@ AllCops:
   TargetRubyVersion: 2.0
 
 Metrics/AbcSize:
-  Max: 40
+  Max: 50
 
 Metrics/BlockLength:
-  Max: 1300
+  Max: 1400
 
 Metrics/ClassLength:
   Max: 500
@@ -14,10 +14,10 @@ Metrics/MethodLength:
   Max: 100
 
 Metrics/PerceivedComplexity:
-  Max: 8
+  Max: 10
 
 Metrics/CyclomaticComplexity:
-  Max: 8
+  Max: 10
 
 # These are to compensate for some warnings that differ by
 # rubocop/ruby version
@@ -30,7 +30,7 @@ Style/MutableConstant:
     - 'lib/tfwrapper/version.rb'
     - 'spec/acceptance/acceptance_spec.rb'
 
-Style/MultilineMethodCallIndentation:
+Layout/MultilineMethodCallIndentation:
   Exclude:
     - 'spec/unit/raketasks_spec.rb'
 

data/ChangeLog.md

@@ -1,3 +1,8 @@
+Version 0.5.0
+
+  - Add support for using terraform_landscape gem, if present, to reformat plan output; see README for usage.
+  - Add CircleCI testing under ruby 2.4.1, and acceptance tests for terraform 0.11.2.
+
 Version 0.4.1
 
   - Upgrade rubocop, yard and diplomat development dependency versions

data/Gemfile

@@ -2,3 +2,7 @@
 
 source 'https://rubygems.org'
 gemspec
+
+group :landscape do
+  gem 'terraform_landscape'
+end

data/README.md

@@ -18,7 +18,9 @@ This Gem provides the following Rake tasks:
   one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
   terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:plan[aws_instance.foo[1]]`` or
   ``bundle exec rake tf:plan[aws_instance.foo[1],aws_instance.bar[2]]``; see the
-  [plan documentation](https://www.terraform.io/docs/commands/plan.html) for more information.
+  [plan documentation](https://www.terraform.io/docs/commands/plan.html) for more information. By default this will use
+  [terraform_landscape](https://github.com/coinbase/terraform-landscape/blob/master/README.md) for formatting the plan
+  output if the ``terraform_landscape`` gem is installed; see the [section below](#terraform-landscape) for more information.
 * __tf:apply__ - run ``terraform apply`` with all variables and configuration, and TF variables written to disk. You can specify
   one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
   terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:apply[aws_instance.foo[1]]`` or
@@ -42,13 +44,30 @@ with 1.9.3 is simply too high to justify.
 Add to your ``Gemfile``:
 
 ```ruby
-gem 'tfwrapper', '~> 0.2.0'
+gem 'tfwrapper', '~> 0.5.0'
 ```
 
 ### Supported Terraform Versions
 
 tfwrapper only supports terraform 0.9+. It is tested against multiple versions from 0.9.2 to 0.10.2 and the current release.
 
+### Terraform Landscape
+
+The [terraform_landscape](https://github.com/coinbase/terraform-landscape/blob/master/README.md) gem provides enhanced formatting of ``terraform plan`` output including colorization of changes and human-friendly diffs (i.e. diffs of JSON rendered with pretty-printing). By default ``plan`` output will be passed through terraform_landscape if the ``terraform_landscape`` gem is available. To enable this, add ``gem 'terraform_landscape', '~> 0.1.17'`` to your ``Gemfile``. __Note__ that we rely on an undocumented internal API of terraform_landscape to achieve this; the formatting code will fall back to unformatted output in case of an error.
+
+If you wish to disable terraform_landscape output even when the gem is installed, pass ``disable_landscape: true`` as an option to ``install_tasks()``:
+
+```ruby
+TFWrapper::RakeTasks.install_tasks('.', disable_landscape: true)
+```
+
+In previous versions or when terraform_landscape is not installed, the output of all terraform commands is streamed in realtime. Since terraform_landscape requires the full and complete ``plan`` output in order to reformat it, this is no longer the case. By default when terraform_landscape is installed and not disabled the ``plan`` task will not produce any output until complete, at which point it will print the landscape-formatted output all at once. This behavior can be controlled with the ``:landscape_progress`` option of ``install_tasks()``, which takes one of the following values:
+
+* ``nil``, the default, to not produce any output until the command is complete at which point the landscape-formatted output will be shown.
+* ``:dots`` to print a dot to STDOUT for every line of ``terraform plan`` output and then print the landscape-formatted output when complete.
+* ``:lines`` to print a dot followed by a newline to STDOUT for every line of ``terraform plan`` output and then print the landscape-formatted output when complete. This is useful for systems like Jenkins that line-buffer output (and don't display anything until a newline is encountered).
+* ``:stream`` to stream the ``terraform plan`` output in realtime (as was the previous behavior) and then print the landscape-formatted output when complete. Note that this will result in the output containing the complete unformatted ``terraform plan`` output, followed by the landscape-formatted output.
+
 ## Usage
 
 To use the Terraform rake tasks, require the module in your Rakefile and use the

data/circle.yml

@@ -5,11 +5,12 @@ dependencies:
   pre:
     - sudo add-apt-repository -y ppa:git-core/ppa && sudo apt-get update && sudo apt-get install git
   override:
-    - 'bundle install'
-    - 'rvm 2.0.0-p598 exec bundle install'
-    - 'rvm 2.1.8 exec bundle install'
-    - 'rvm 2.2.5 exec bundle install'
-    - 'rvm 2.3.1 exec bundle install'
+    - 'bundle install --without landscape'
+    - 'rvm 2.0.0-p598 exec bundle install --without landscape'
+    - 'rvm 2.1.8 exec bundle install --without landscape'
+    - 'rvm 2.2.5 exec bundle install --without landscape'
+    - 'rvm 2.3.1 exec bundle install --without landscape'
+    - 'rvm 2.4.1 exec bundle install'
 
 test:
   override:
@@ -20,9 +21,12 @@ test:
     - 'rvm 2.1.8 exec bundle exec rake spec:unit'
     - 'rvm 2.2.5 exec bundle exec rake spec:unit'
     - 'rvm 2.3.1 exec bundle exec rake spec:unit'
+    - 'rvm 2.4.1 exec bundle exec rake spec:unit'
     - 'rvm 2.3.1 exec bundle exec rake spec:acceptance TF_VERSION=0.9.2'
     - 'rvm 2.3.1 exec bundle exec rake spec:acceptance TF_VERSION=0.9.7'
     - 'rvm 2.3.1 exec bundle exec rake spec:acceptance TF_VERSION=0.10.0'
     - 'rvm 2.3.1 exec bundle exec rake spec:acceptance TF_VERSION=0.10.2'
+    - 'rvm 2.3.1 exec bundle exec rake spec:acceptance TF_VERSION=0.11.2'
     - 'rvm 2.3.1 exec bundle exec rake spec:acceptance TF_VERSION=latest'
+    - 'rvm 2.4.1 exec bundle exec rake spec:acceptance TF_VERSION=0.11.2'
     - 'bundle exec rake yard:generate'

data/lib/tfwrapper/helpers.rb

@@ -31,8 +31,20 @@ module TFWrapper
     #
     # @param cmd [String] command to run
     # @param pwd [String] directory/path to run command in
+    # @option opts [Hash] :progress How to handle streaming output. Possible
+    #  values are ``:stream`` (default) to stream each line in STDOUT/STDERR
+    #  to STDOUT, ``:dots`` to print a dot for each line, ``:lines`` to print
+    #  a dot followed by a newline for each line, or ``nil`` to not stream any
+    #  output at all.
     # @return [Array] - out_err [String], exit code [Fixnum]
-    def self.run_cmd_stream_output(cmd, pwd)
+    def self.run_cmd_stream_output(cmd, pwd, opts = {})
+      stream_type = opts.fetch(:progress, :stream)
+      unless [:dots, :lines, :stream, nil].include?(stream_type)
+        raise(
+          ArgumentError,
+          'progress option must be one of: [:dots, :lines, :stream, nil]'
+        )
+      end
       old_sync = $stdout.sync
       $stdout.sync = true
       all_out_err = ''.dup
@@ -41,7 +53,13 @@ module TFWrapper
         stdin.close_write
         begin
           while (line = stdout_and_err.gets)
-            puts line
+            if stream_type == :stream
+              puts line
+            elsif stream_type == :dots
+              STDOUT.print '.'
+            elsif stream_type == :lines
+              puts '.'
+            end
             all_out_err << line
           end
         rescue IOError => e
@@ -51,6 +69,7 @@ module TFWrapper
       end
       # rubocop:disable Style/RedundantReturn
       $stdout.sync = old_sync
+      puts '' if stream_type == :dots
       return all_out_err, exit_status
       # rubocop:enable Style/RedundantReturn
     end

data/lib/tfwrapper/raketasks.rb

@@ -6,6 +6,15 @@ require 'rake'
 require 'rubygems'
 require 'tfwrapper/version'
 
+# :nocov:
+begin
+  require 'terraform_landscape'
+  HAVE_LANDSCAPE = true
+rescue LoadError
+  HAVE_LANDSCAPE = false
+end
+# :nocov:
+
 module TFWrapper
   # Generates Rake tasks for working with Terraform.
   class RakeTasks
@@ -63,6 +72,22 @@ module TFWrapper
     #   the body of each task. Called with two arguments, the String full
     #   (namespaced) name of the task being executed, and ``tf_dir``. This will
     #   not execute if the body of the task fails.
+    # @option opts [Bool] :disable_landscape By default, if the
+    #  ``terraform_landscape`` gem can be loaded, it will be used to reformat
+    #  the output of ``terraform plan``. If this is not desired, set to
+    #  ``true`` to disable landscale. Default: ``false``.
+    # @option opts [Symbol, nil] :landscape_progress The ``terraform_landscape``
+    #  code used to reformat plan output requires the full output of the
+    #  complete ``plan`` execution. By default, this means that when landscape
+    #  is used, no output will appear from the time ``terraform plan`` begins
+    #  until the command is complete. If progress output is desired, this option
+    #  can be set to one of the following: ``:dots`` to print a dot to STDOUT
+    #  for every line of ``terraform plan`` output, ``:lines`` to print a dot
+    #  followed by a newline (e.g. for systems like Jenkins that line buffer)
+    #  for every line of ``terraform plan`` output, or ``:stream`` to stream
+    #  the raw ``terraform plan`` output (which will then be followed by the
+    #  reformatted landscape output). Default is ``nil`` to show no progress
+    #  output.
     def initialize(tf_dir, opts = {})
       # find the directory that contains the Rakefile
       rakedir = File.realpath(Rake.application.rakefile)
@@ -74,6 +99,15 @@ module TFWrapper
       @tf_extra_vars = opts.fetch(:tf_extra_vars, {})
       @backend_config = opts.fetch(:backend_config, {})
       @consul_url = opts.fetch(:consul_url, nil)
+      @disable_landscape = opts.fetch(:disable_landscape, false)
+      @landscape_progress = opts.fetch(:landscape_progress, nil)
+      unless [:dots, :lines, :stream, nil].include?(@landscape_progress)
+        raise(
+          ArgumentError,
+          'landscape_progress option must be one of: ' \
+          '[:dots, :lines, :stream, nil]'
+        )
+      end
       @before_proc = opts.fetch(:before_proc, nil)
       if !@before_proc.nil? && !@before_proc.is_a?(Proc)
         raise(
@@ -160,7 +194,13 @@ module TFWrapper
             args.extras
           )
 
-          terraform_runner(cmd)
+          stream_type = if HAVE_LANDSCAPE && !@disable_landscape
+                          @landscape_progress
+                        else
+                          :stream
+                        end
+          outerr = terraform_runner(cmd, progress: stream_type)
+          landscape_format(outerr) if HAVE_LANDSCAPE && !@disable_landscape
           @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
         end
       end
@@ -296,13 +336,39 @@ module TFWrapper
       res
     end
 
+    # Given a string of terraform plan output, format it with
+    # terraform_landscape and print the result to STDOUT.
+    def landscape_format(output)
+      p = TerraformLandscape::Printer.new(
+        TerraformLandscape::Output.new(STDOUT)
+      )
+      p.process_string(output)
+    rescue StandardError, ScriptError => ex
+      STDERR.puts 'Exception calling terraform_landscape to reformat ' \
+                  "output: #{ex.class.name}: #{ex}"
+      puts output unless @landscape_progress == :stream
+    end
+
     # Run a Terraform command, providing some useful output and handling AWS
     # API rate limiting gracefully. Raises StandardError on failure. The command
     # is run in @tf_dir.
     #
     # @param cmd [String] Terraform command to run
+    # @option opts [Hash] :progress How to handle streaming output. Possible
+    #  values are ``:stream`` (default) to stream each line in STDOUT/STDERR
+    #  to STDOUT, ``:dots`` to print a dot for each line, ``:lines`` to print
+    #  a dot followed by a newline for each line, or ``nil`` to not stream any
+    #  output at all.
+    # @return [String] combined STDOUT and STDERR
     # rubocop:disable Metrics/PerceivedComplexity
-    def terraform_runner(cmd)
+    def terraform_runner(cmd, opts = {})
+      stream_type = opts.fetch(:progress, :stream)
+      unless [:dots, :lines, :stream, nil].include?(stream_type)
+        raise(
+          ArgumentError,
+          'progress option must be one of: [:dots, :lines, :stream, nil]'
+        )
+      end
       require 'retries'
       STDERR.puts "terraform_runner command: '#{cmd}' (in #{@tf_dir})"
       out_err = nil
@@ -321,7 +387,9 @@ module TFWrapper
       ) do
         # this streams STDOUT and STDERR as a combined stream,
         # and also captures them as a combined string
-        out_err, status = TFWrapper::Helpers.run_cmd_stream_output(cmd, @tf_dir)
+        out_err, status = TFWrapper::Helpers.run_cmd_stream_output(
+          cmd, @tf_dir, progress: stream_type
+        )
         if status != 0 && out_err.include?('hrottling')
           raise StandardError, 'Terraform hit AWS API rate limiting'
         end
@@ -340,6 +408,7 @@ module TFWrapper
           "(exited #{status})"
       end
       STDERR.puts "terraform_runner command '#{cmd}' finished and exited 0"
+      out_err
     end
     # rubocop:enable Metrics/PerceivedComplexity
 

data/lib/tfwrapper/version.rb

@@ -4,5 +4,5 @@ module TFWrapper
   # version of the Gem/module; used in the gemspec and in messages.
   # NOTE: When updating this, also update the version in the "Installation"
   # section of README.md
-  VERSION = '0.4.1'
+  VERSION = '0.5.0'
 end

data/spec/acceptance/acceptance_helpers.rb

@@ -24,12 +24,24 @@ def desired_tf_version
     return ENV['TF_VERSION']
   end
   # else get the latest release from GitHub
+  latest_tf_version
+end
+
+def latest_tf_version
   resp = Faraday.get('https://api.github.com/repos/hashicorp/terraform/releases/latest')
   rel = JSON.parse(resp.body)['tag_name'].sub(/^v/, '')
   puts "Found latest terraform release on GitHub: #{rel}"
   rel
 end
 
+# Given the example terraform plan output with placeholders for the
+# latest terraform version and fixtures path, return the interpolated string.
+def clean_tf_plan_output(raw_out, latest_ver, fixture_path)
+  raw_out
+    .gsub('%%TF_LATEST_VER%%', latest_ver)
+    .gsub('%%FIXTUREPATH%%', fixture_path)
+end
+
 class HashicorpFetcher
   def initialize(program, version)
     @program = program

data/spec/acceptance/acceptance_spec.rb

@@ -18,6 +18,10 @@ else
   APPLY_CMD = 'terraform apply'
 end
 
+without_landscape = !HAVE_LANDSCAPE && TF_VERSION == '0.11.2'
+with_landscape = HAVE_LANDSCAPE && TF_VERSION == '0.11.2'
+latest_tf_ver = latest_tf_version
+
 Diplomat.configure do |config|
   config.url = 'http://127.0.0.1:8500'
 end
@@ -734,4 +738,190 @@ describe 'tfwrapper' do
       end
     end
   end
+  context 'landscapeTest', order: :defined do
+    before(:all) do
+      @fixturepath = File.absolute_path(
+        File.join(File.dirname(__FILE__), '..', 'fixtures', 'landscapeTest')
+      )
+    end
+    before(:each) do
+      Diplomat::Kv.put(
+        'landscapeTest/foo', '{"bar":"barval","baz":"bazval","foo":"fooval"}'
+      )
+      Diplomat::Kv.put(
+        'terraform/landscapeTest',
+        File.read(File.join(@fixturepath, 'state.json'))
+      )
+    end
+    context 'without landscape installed', if: without_landscape do
+      describe 'default_tf:plan' do
+        before(:all) do
+          @out_err, @ecode = Open3.capture2e(
+            'timeout -k 60 45 bundle exec rake default_tf:plan',
+            chdir: @fixturepath
+          )
+          @varpath = File.join(@fixturepath, 'default_build.tfvars.json')
+        end
+        after(:all) do
+          File.delete(@varpath) if File.file?(@varpath)
+        end
+        it 'does not time out' do
+          expect(@ecode.exitstatus).to_not eq(124)
+          expect(@ecode.exitstatus).to_not eq(137)
+        end
+        it 'exits zero' do
+          expect(@ecode.exitstatus).to eq(0)
+        end
+        it 'returns unmodified terraform output' do
+          expected = clean_tf_plan_output(
+            File.read(File.join(@fixturepath, 'without_landscape.out')),
+            latest_tf_ver, @fixturepath
+          )
+          expect(@out_err.strip).to eq(expected.strip)
+        end
+      end
+    end
+    context 'with landscape installed', if: with_landscape do
+      context 'and disabled' do
+        describe 'disabled_tf:plan' do
+          before(:all) do
+            @out_err, @ecode = Open3.capture2e(
+              'timeout -k 60 45 bundle exec rake disabled_tf:plan',
+              chdir: @fixturepath
+            )
+            @varpath = File.join(@fixturepath, 'disabled_build.tfvars.json')
+          end
+          after(:all) do
+            File.delete(@varpath) if File.file?(@varpath)
+          end
+          it 'does not time out' do
+            expect(@ecode.exitstatus).to_not eq(124)
+            expect(@ecode.exitstatus).to_not eq(137)
+          end
+          it 'exits zero' do
+            expect(@ecode.exitstatus).to eq(0)
+          end
+          it 'returns unmodified terraform output' do
+            expected = clean_tf_plan_output(
+              File.read(File.join(@fixturepath, 'without_landscape.out')),
+              latest_tf_ver, @fixturepath
+            ).gsub('default_build.tfvars.json', 'disabled_build.tfvars.json')
+            expect(@out_err.strip).to eq(expected.strip)
+          end
+        end
+      end
+      context 'and default progress' do
+        describe 'default_tf:plan' do
+          before(:all) do
+            @out_err, @ecode = Open3.capture2e(
+              'timeout -k 60 45 bundle exec rake default_tf:plan',
+              chdir: @fixturepath
+            )
+            @varpath = File.join(@fixturepath, 'default_build.tfvars.json')
+          end
+          after(:all) do
+            File.delete(@varpath) if File.file?(@varpath)
+          end
+          it 'does not time out' do
+            expect(@ecode.exitstatus).to_not eq(124)
+            expect(@ecode.exitstatus).to_not eq(137)
+          end
+          it 'exits zero' do
+            expect(@ecode.exitstatus).to eq(0)
+          end
+          it 'returns landscape output and no plan output' do
+            expected = clean_tf_plan_output(
+              File.read(File.join(@fixturepath, 'with_landscape_default.out')),
+              latest_tf_ver, @fixturepath
+            )
+            expect(@out_err.strip).to eq(expected.strip)
+          end
+        end
+      end
+      context 'and dots progress' do
+        describe 'dots_tf:plan' do
+          before(:all) do
+            @out_err, @ecode = Open3.capture2e(
+              'timeout -k 60 45 bundle exec rake dots_tf:plan',
+              chdir: @fixturepath
+            )
+            @varpath = File.join(@fixturepath, 'dots_build.tfvars.json')
+          end
+          after(:all) do
+            File.delete(@varpath) if File.file?(@varpath)
+          end
+          it 'does not time out' do
+            expect(@ecode.exitstatus).to_not eq(124)
+            expect(@ecode.exitstatus).to_not eq(137)
+          end
+          it 'exits zero' do
+            expect(@ecode.exitstatus).to eq(0)
+          end
+          it 'returns progress dots for plan output and landscape output' do
+            expected = clean_tf_plan_output(
+              File.read(File.join(@fixturepath, 'with_landscape_dots.out')),
+              latest_tf_ver, @fixturepath
+            )
+            expect(@out_err.strip).to eq(expected.strip)
+          end
+        end
+      end
+      context 'and lines progress' do
+        describe 'lines_tf:plan' do
+          before(:all) do
+            @out_err, @ecode = Open3.capture2e(
+              'timeout -k 60 45 bundle exec rake lines_tf:plan',
+              chdir: @fixturepath
+            )
+            @varpath = File.join(@fixturepath, 'lines_build.tfvars.json')
+          end
+          after(:all) do
+            File.delete(@varpath) if File.file?(@varpath)
+          end
+          it 'does not time out' do
+            expect(@ecode.exitstatus).to_not eq(124)
+            expect(@ecode.exitstatus).to_not eq(137)
+          end
+          it 'exits zero' do
+            expect(@ecode.exitstatus).to eq(0)
+          end
+          it 'returns progress lines for plan output and landscape output' do
+            expected = clean_tf_plan_output(
+              File.read(File.join(@fixturepath, 'with_landscape_lines.out')),
+              latest_tf_ver, @fixturepath
+            )
+            expect(@out_err.strip).to eq(expected.strip)
+          end
+        end
+      end
+      context 'and stream progress' do
+        describe 'stream_tf:plan' do
+          before(:all) do
+            @out_err, @ecode = Open3.capture2e(
+              'timeout -k 60 45 bundle exec rake stream_tf:plan',
+              chdir: @fixturepath
+            )
+            @varpath = File.join(@fixturepath, 'stream_build.tfvars.json')
+          end
+          after(:all) do
+            File.delete(@varpath) if File.file?(@varpath)
+          end
+          it 'does not time out' do
+            expect(@ecode.exitstatus).to_not eq(124)
+            expect(@ecode.exitstatus).to_not eq(137)
+          end
+          it 'exits zero' do
+            expect(@ecode.exitstatus).to eq(0)
+          end
+          it 'returns streaming plan output and landscape output' do
+            expected = clean_tf_plan_output(
+              File.read(File.join(@fixturepath, 'with_landscape_stream.out')),
+              latest_tf_ver, @fixturepath
+            )
+            expect(@out_err.strip).to eq(expected.strip)
+          end
+        end
+      end
+    end
+  end
 end