tfwrapper 0.2.0.beta1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0d770015f26d1cc19424567299a4cf0d300c7c9d
+  data.tar.gz: 682aecc64e4c300ff15140221f7b69b47a7042d1
+SHA512:
+  metadata.gz: 0501b6c78f2c23afc807ab5f2942798929d177fdfd07230504135c5470360655bd5705c6b4086977b763f6337d0a5dc28d997c816d78519e1b0bef8c1243db4c
+  data.tar.gz: 2361518650e15fef68dab14983345195935b8a46bdd2fd88dc1c8a70813140a5f43ba57781efed2a0ce79e79867dda1e950e340bfe8e51d47feacf1e381ef449

data/.gitignore

@@ -0,0 +1,54 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+/results/
+/vendor/
+.terraform
+build.tfvars.json
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,22 @@
+Metrics/AbcSize:
+  Max: 22
+
+Metrics/BlockLength:
+  Max: 1000
+
+Metrics/ClassLength:
+  Max: 500
+
+Metrics/MethodLength:
+  Max: 100
+
+# These are to compensate for some warnings that differ by
+# rubocop/ruby version
+Lint/UnneededDisable:
+  Exclude:
+    - 'lib/tfwrapper/raketasks.rb'
+
+Style/MutableConstant:
+  Exclude:
+    - 'lib/tfwrapper/version.rb'
+    - 'spec/acceptance/acceptance_spec.rb'

data/ChangeLog.md

@@ -0,0 +1,3 @@
+Version 0.1.0
+
+  - initial release (migrated from previous internal/private gem)

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec

data/Guardfile

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+guard :bundler do
+  watch('tfwrapper.gemspec')
+end
+
+guard :rubocop do
+  watch(/.+\.rb$/)
+  watch(%r{(?:.+\/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch('spec/spec_helper.rb') { 'spec' }
+  watch(%r{^spec/.+_spec\.rb})
+  watch(%r{^spec/(.+)/.+_spec\.rb})
+  watch(%r{^lib\/tfwrapper/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" }
+end
+
+guard 'yard', port: '8808' do
+  watch(/README\.md/)
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Manheim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,320 @@
+# tfwrapper
+
+Build of master branch: [![CircleCI](https://circleci.com/gh/manheim/tfwrapper.svg?style=svg)](https://circleci.com/gh/manheim/tfwrapper)
+
+Documentation: [http://www.rubydoc.info/gems/tfwrapper/](http://www.rubydoc.info/gems/tfwrapper/)
+
+tfwrapper provides Rake tasks for working with [Hashicorp Terraform](https://www.terraform.io/) 0.9+, ensuring proper initialization and passing in variables from the environment or Ruby, as well as optionally pushing some information to Consul. tfwrapper also attempts to detect and retry
+failed runs due to AWS throttling or access denied errors.
+
+## Overview
+
+This Gem provides the following Rake tasks:
+
+* __tf:init__ - run ``terraform init`` to pull down dependency modules and configure remote
+  state backend. This task also checks that any configured environment variables are set and
+  that the ``terraform`` version is compatible with this gem.
+* __tf:plan__ - run ``terraform plan`` with all variables and configuration, and TF variables written to disk. You can specify
+  one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
+  terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:plan[aws_instance.foo[1]]`` or
+  ``bundle exec rake tf:plan[aws_instance.foo[1],aws_instance.bar[2]]``; see the
+  [plan documentation](https://www.terraform.io/docs/commands/plan.html) for more information.
+* __tf:apply__ - run ``terraform apply`` with all variables and configuration, and TF variables written to disk. You can specify
+  one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
+  terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:apply[aws_instance.foo[1]]`` or
+  ``bundle exec rake tf:apply[aws_instance.foo[1],aws_instance.bar[2]]``; see the
+  [apply documentation](https://www.terraform.io/docs/commands/apply.html) for more information. This also runs a plan first.
+* __tf:refresh__ - run ``terraform refresh``
+* __tf:destroy__ - run ``terraform destroy`` with all variables and configuration, and TF variables written to disk. You can specify
+  one or more optional [resource address](https://www.terraform.io/docs/internals/resource-addressing.html) targets to pass to
+  terraform with the ``-target`` flag as Rake task arguments, i.e. ``bundle exec rake tf:destroy[aws_instance.foo[1]]`` or
+  ``bundle exec rake tf:destroy[aws_instance.foo[1],aws_instance.bar[2]]``; see the
+  [destroy documentation](https://www.terraform.io/docs/commands/destroy.html) for more information.
+* __tf:write_tf_vars__ - used as a prerequisite for other tasks; write Terraform variables to file on disk
+
+## Installation
+
+__Note:__ tfwrapper only supports Ruby >= 2.0.0. The effort to maintain compatibility
+with 1.9.3 is simply too high to justify.
+
+Add to your ``Gemfile``:
+
+```ruby
+gem 'tfwrapper', '~> 0.1.0'
+```
+
+### Supported Terraform Versions
+
+tfwrapper only supports terraform 0.9+. It has been tested up to 0.9.2.
+
+## Usage
+
+To use the Terraform rake tasks, require the module in your Rakefile and use the
+``install_tasks`` method to set up the tasks. ``install_tasks`` takes one mandatory parameter,
+``tf_dir`` specifying the relative path (from the Rakefile) to the Terraform configuration.
+
+For a directory layout like:
+
+```
+.
+├── bar.tf
+├── foo.tf
+├── main.tf
+└── Rakefile
+```
+
+The minimal ``Rakefile`` would be:
+
+```ruby
+require 'tfwrapper/raketasks'
+
+TFWrapper::RakeTasks.install_tasks('.')
+```
+
+``rake -T`` output:
+
+```
+rake tf:apply[target]        # Apply a terraform plan that will provision your resources; specify optional CSV targets
+rake tf:destroy[target]      # Destroy any live resources that are tracked by your state files; specify optional CSV targets
+rake tf:init                 # Run terraform init with appropriate arguments
+rake tf:plan[target]         # Output the set plan to be executed by apply; specify optional CSV targets
+rake tf:write_tf_vars        # Write PWD/build.tfvars.json
+```
+
+You can also point ``tf_dir`` to an arbitrary directory relative to the Rakefile, such as when your terraform
+configurations are nested below the Rakefile:
+
+```
+.
+├── infrastructure
+│   └── terraform
+│       ├── bar.tf
+│       ├── foo.tf
+│       └── main.tf
+├── lib
+├── Rakefile
+└── spec
+```
+
+Rakefile:
+
+```ruby
+require 'tfwrapper/raketasks'
+
+TFWrapper::RakeTasks.install_tasks('infrastructure/terraform')
+```
+
+### Environment Variables to Terraform Variables
+
+If you wish to bind the values of environment variables to Terraform variables, you can specify a mapping
+of Terraform variable name to environment variable name in the ``tf_vars_from_env`` option; these variables
+will be automatically read from the environment and passed into Terraform with the appropriate names. The following
+example sets the ``consul_address`` terraform variable to the value of the ``CONSUL_HOST`` environment variable
+(defaulting it to ``consul.example.com:8500`` if it is not already set in the environment),
+and likewise for the ``environment`` terraform variable from the ``ENVIRONMENT`` env var.
+
+```ruby
+require 'tfwrapper/raketasks'
+ENV['CONSUL_HOST'] ||= 'consul.example.com:8500'
+
+TFWrapper::RakeTasks.install_tasks(
+  '.',
+  tf_vars_from_env: {
+    'consul_address'           => 'CONSUL_HOST',
+    'environment'              => 'ENVIRONMENT',
+  }
+)
+```
+
+### Ruby Variables to Terraform Variables
+
+If you wish to explicitly bind values from your Ruby code to terraform variables, you can do this with
+the ``tf_extra_vars`` option. Variables specified in this way will override same-named variables populated
+via ``tf_vars_from_env``. In the following example, the ``foobar`` terraform variable will have a value
+of ``baz``, regardless of what the ``FOOBAR`` environment variable is set to, and the ``hostname``
+terraform variable will be set to the hostname (``Socket.gethostname``) of the system Rake is running on:
+
+```ruby
+require 'socket'
+require 'tfwrapper/raketasks'
+
+ENV['FOOBAR'] ||= 'not_baz'
+
+TFWrapper::RakeTasks.install_tasks(
+  '.',
+  tf_vars_from_env: {
+    'foobar' => 'FOOBAR'
+  },
+  tf_extra_vars: {
+    'foobar'   => 'baz',
+    'hostname' => Socket.gethostname
+  }
+)
+```
+
+### Namespace Prefixes for Multiple Configurations
+
+If you need to work with multiple different Terraform configurations, this is possible
+by adding a namespace prefix and calling ``install_tasks`` multiple times. The following example
+will produce two sets of terraform Rake tasks; one with the default ``tf:`` namespace
+that acts on the configurations under ``tf/foo``, and one with a ``bar_tf:`` namespace
+that acts on the configurations under ``tf/bar``. You can use as many namespaces as
+you want.
+
+Directory tree:
+
+```
+.
+├── Rakefile
+└── tf
+    ├── bar
+    │   └── bar.tf
+    └── foo
+        └── foo.tf
+```
+
+Rakefile:
+
+```ruby
+require 'tfwrapper/raketasks'
+
+# foo/ (default) terraform tasks
+TFWrapper::RakeTasks.install_tasks('tf/foo')
+
+# bar/ terraform tasks
+TFWrapper::RakeTasks.install_tasks('tf/bar', namespace_prefix: 'bar')
+```
+
+``rake -T`` output:
+
+```
+rake bar_tf:apply[target]    # Apply a terraform plan that will provision your resources; specify optional CSV targets
+rake bar_tf:destroy[target]  # Destroy any live resources that are tracked by your state files; specify optional CSV targets
+rake bar_tf:init             # Run terraform init with appropriate arguments
+rake bar_tf:plan[target]     # Output the set plan to be executed by apply; specify optional CSV targets
+rake bar_tf:write_tf_vars    # Write PWD/bar_build.tfvars.json
+rake tf:apply[target]        # Apply a terraform plan that will provision your resources; specify optional CSV targets
+rake tf:destroy[target]      # Destroy any live resources that are tracked by your state files; specify optional CSV targets
+rake tf:init                 # Run terraform init with appropriate arguments
+rake tf:plan[target]         # Output the set plan to be executed by apply; specify optional CSV targets
+rake tf:write_tf_vars        # Write PWD/build.tfvars.json
+```
+
+### Backend Configuration Options
+
+``install_tasks`` accepts a ``backend_config`` hash of options to pass as backend configuration
+to ``terraform init`` via the ``-backend-config='key=value'`` command line argument. This can
+be used when you need to pass some backend configuration in from the environment, such as a
+specific remote state storage path, credentials, etc.
+
+For a simple example, assume we aren't using [state environments](https://www.terraform.io/docs/state/environments.html)
+but instead opt to use specific paths based on a ``ENVIRONMENT`` environment variable.
+
+Our terraform configuration might include something like:
+
+```
+terraform {
+  required_version = "> 0.9.0"
+  backend "consul" {
+    address = "consul.example.com:8500"
+  }
+}
+
+variable "environment" {}
+```
+
+And the Rakefile would pass in the path to store state in Consul, as well as
+passing the ``ENVIRONMENT`` env var into Terraform for use:
+
+```ruby
+require 'tfwrapper/raketasks'
+
+TFWrapper::RakeTasks.install_tasks(
+  '.',
+  tf_vars_from_env: {'environment' => 'ENVIRONMENT'},
+  backend_config: {'path' => "terraform/foo/#{ENVIRONMENT}"}
+)
+```
+
+### Environment Variables to Consul
+
+tfwrapper also includes functionality to push environment variables to Consul
+(as a JSON object) after a successful apply. This is mainly useful when running
+tfwrapper from within Jenkins or another job runner, where they can be used to
+pre-populate user input fields on subsequent runs. This is configured via the
+``consul_url`` and ``consul_env_vars_prefix`` options:
+
+Example Terraform snippet:
+
+```
+variable "foo" {}
+variable "bar" {}
+```
+
+Rakefile:
+
+```ruby
+require 'tfwrapper/raketasks'
+
+TFWrapper::RakeTasks.install_tasks(
+  '.',
+  tf_vars_from_env: {'foo' => 'FOO', 'bar' => 'BAR'},
+  consul_url: 'http://consul.example.com:8500',
+  consul_env_vars_prefix: 'terraform/inputs/foo'
+)
+```
+
+After a successful terraform apply, e.g.:
+
+```
+FOO=one BAR=two bundle exec rake tf:apply
+```
+
+The key in Consul at ``terraform/inputs/foo`` will be set to a JSON hash of the
+environment variables used via ``tf_vars_from_env`` and their values:
+
+```shell
+$ consul kv get terraform/inputs/foo
+{"FOO":"one", "BAR":"two"}
+```
+
+## Development
+
+1. ``bundle install --path vendor``
+2. ``bundle exec rake pre_commit`` to ensure unit tests are passing and style is valid before making your changes.
+3. ``bundle exec rake spec:acceptance`` to ensure acceptance tests are passing before making your changes.
+4. make your changes, and write unit tests for them. If you introduced user-visible (public API) changes, write acceptance tests for them. You can run ``bundle exec guard`` to continually run unit tests and rubocop when files change.
+5. ``bundle exec rake pre_commit`` to confirm your unit tests pass and your style is valid. You should confirm 100% coverage. If you wish, you can run ``bundle exec guard`` to dynamically run rspec, rubocop and YARD when relevant files change.
+6. ``bundle exec rake spec:acceptance`` to ensure acceptance tests are passing.
+7. Update ``ChangeLog.md`` for your changes.
+8. Run ``bundle exec rake yard:serve`` to generate documentation for your Gem and serve it live at [http://localhost:8808](http://localhost:8808), and ensure it looks correct.
+9. Open a pull request for your changes.
+10. When shipped, wait for CircleCI to test. Once shipped and tests pass, merge the PR.
+
+When running inside CircleCI, rspec will place reports and artifacts under the right locations for CircleCI to archive them. When running outside of CircleCI, coverage reports will be written to ``coverage/`` and test reports (HTML and JUnit XML) will be written to ``results/``.
+
+### Acceptance Tests
+
+This gem includes some rspec-based acceptance tests, runnable via ``bundle exec rake spec:acceptance``. These tests download
+a specific version of Terraform and Consul, run a local Consul server (in ``-dev`` mode), and actually run ``terraform`` via
+``rake`` and confirm that Terraform both runs correctly and correctly updates state in Consul. The terraform configurations
+and rakefiles used can be found in ``spec/acceptance``. The terraform configurations use only the
+[consul](https://www.terraform.io/docs/providers/consul/index.html) provider, to remove any external dependencies other than
+Consul (which is already used to test remote state).
+
+Note that the acceptance tests depend on the GNU coreutils ``timeout`` command.
+
+## Release Checklist
+
+1. Ensure Circle tests are passing.
+2. Build docs locally (``bundle exec rake yard:serve``) and ensure they look correct.
+3. Ensure changelog entries exist for all changes since the last release.
+4. Bump the version in ``lib/tfwrapper/version.rb``
+5. Change the version specifier in the "Installation" section of this README, above, as appropriate.
+6. Commit those changes, open a PR for the release. Once shipped and Circle passes, merge and pull down locally.
+7. Deployment is done locally, with ``bundle exec rake release``.
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'bundler'
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+require 'rake/clean'
+require 'yard'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+CLOBBER.include 'pkg'
+
+desc 'Run RuboCop on the lib directory'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.fail_on_error = true
+end
+
+namespace :spec do
+  desc 'run ALL spec tests'
+  task all: %i[unit acceptance]
+
+  desc 'run unit tests'
+  RSpec::Core::RakeTask.new(:unit) do |t|
+    t.rspec_opts = ['--require', 'spec_helper']
+    t.pattern = 'spec/unit/*_spec.rb'
+  end
+
+  desc 'run acceptance tests'
+  RSpec::Core::RakeTask.new(:acceptance) do |t|
+    t.rspec_opts = ['--require', 'spec_helper']
+    t.pattern = 'spec/acceptance/*_spec.rb'
+  end
+end
+
+namespace :yard do
+  YARD::Rake::YardocTask.new do |t|
+    t.name = 'generate'
+    t.files   = ['lib/**/*.rb'] # optional
+    t.options = ['--private', '--protected'] # optional
+    t.stats_options = ['--list-undoc'] # optional
+  end
+
+  desc 'serve YARD documentation on port 8808 (restart to regenerate)'
+  task serve: [:generate] do
+    puts 'Running YARD server on port 8808'
+    puts 'Use Ctrl+C to exit server.'
+    YARD::CLI::Server.run
+  end
+end
+
+desc 'Run specs and rubocop before pushing'
+task pre_commit: %i[spec:unit rubocop]
+
+desc 'Display the list of available rake tasks'
+task :help do
+  system('rake -T')
+end
+
+task default: [:help]

data/circle.yml

@@ -0,0 +1,24 @@
+
+dependencies:
+  cache_directories:
+    - "~/.rvm/gems"
+  pre:
+    - sudo add-apt-repository -y ppa:git-core/ppa && sudo apt-get update && sudo apt-get install git
+  override:
+    - 'bundle install'
+    - 'rvm 2.0.0-p598 exec bundle install'
+    - 'rvm 2.1.8 exec bundle install'
+    - 'rvm 2.2.5 exec bundle install'
+    - 'rvm 2.3.1 exec bundle install'
+
+test:
+  override:
+    - 'bundle exec ruby --version'
+    - 'bundle exec rake spec:unit'
+    - 'bundle exec rake rubocop'
+    - 'rvm 2.0.0-p598 exec bundle exec rake spec:unit'
+    - 'rvm 2.1.8 exec bundle exec rake spec:unit'
+    - 'rvm 2.2.5 exec bundle exec rake spec:unit'
+    - 'rvm 2.3.1 exec bundle exec rake spec:unit'
+    - 'rvm 2.3.1 exec bundle exec rake spec:acceptance'
+    - 'bundle exec rake yard:generate'

data/lib/tfwrapper/helpers.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'English'
+
+# TFWrapper
+module TFWrapper
+  # generic helper functions for TFWrapper
+  module Helpers
+    # Run a system command, print the command before running it. If it exits
+    # with a non-zero status, print the exit status and output and then
+    # `fail`.
+    #
+    # @param cmd [String] the command to run
+    def self.run_cmd(cmd)
+      puts "Running command: #{cmd}"
+      out = `#{cmd}`
+      status = $CHILD_STATUS.exitstatus
+      return if status.zero?
+      puts "Command exited #{status}:"
+      puts out
+      raise StandardError, "ERROR: Command failed: #{cmd}"
+    end
+
+    # popen2e wrapper to simultaneously stream command output and capture it.
+    #
+    # STDOUT and STDERR will be combined to the same stream, and returned as one
+    # string. This is because there doesn't seem to be a safe, cross-platform
+    # way to both capture and stream STDOUT and STDERR separately that isn't
+    # prone to deadlocking if large chunks of data are written to the pipes.
+    #
+    # @param cmd [String] command to run
+    # @param pwd [String] directory/path to run command in
+    # @return [Array] - out_err [String], exit code [Fixnum]
+    def self.run_cmd_stream_output(cmd, pwd)
+      old_sync = $stdout.sync
+      $stdout.sync = true
+      all_out_err = ''.dup
+      exit_status = nil
+      Open3.popen2e(cmd, chdir: pwd) do |stdin, stdout_and_err, wait_thread|
+        stdin.close_write
+        begin
+          while (line = stdout_and_err.gets)
+            puts line
+            all_out_err << line
+          end
+        rescue IOError => e
+          STDERR.puts "IOError: #{e}"
+        end
+        exit_status = wait_thread.value.exitstatus
+      end
+      # rubocop:disable Style/RedundantReturn
+      $stdout.sync = old_sync
+      return all_out_err, exit_status
+      # rubocop:enable Style/RedundantReturn
+    end
+
+    # Ensure that a given list of environment variables are present and
+    # non-empty. Raise StandardError if any aren't.
+    #
+    # @param required [Array] list of required environment variables
+    def self.check_env_vars(required)
+      missing = []
+      required.each do |name|
+        if !ENV.include?(name)
+          puts "ERROR: Environment variable '#{name}' must be set."
+          missing << name
+        elsif ENV[name].to_s.strip.empty?
+          puts "ERROR: Environment variable '#{name}' must not be empty."
+          missing << name
+        end
+      end
+      # rubocop:disable Style/GuardClause
+      unless missing.empty?
+        raise StandardError, 'Missing or empty environment variables: ' \
+          "#{missing}"
+      end
+      # rubocop:enable Style/GuardClause
+    end
+  end
+end