covalence 0.8.4 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22fc1079779bb74dfe1aba91513a57884fb10bae4ef263a044bda6396ca0edde
-  data.tar.gz: 4e49429ce0b27d3217b865ac65459e49cb598e9b6810bda2031708a741d01bc4
+  metadata.gz: 01a6df525586ec948c89317d69451a4f8acc3e46bc80fb09b55833f6b91e9ba2
+  data.tar.gz: b20fd2249962484525305bc11f0fdf796276126c26a074e97322358b6da76622
 SHA512:
-  metadata.gz: d771183f719a23e05b0f3419bfc3ca6f3b3b102596bbe96406fb1024bf38801b2cf0b40a4a3c14ca00c751cb099f4c82e4276da2b695b9ba5fded504b888a3f1
-  data.tar.gz: ccb613d38810d5fbbcde0ac12cdbef1bc55adbf7ca0da7b1be44d00c4cc2269837795be14af7e59472383a4d298c7713bc7821ef7755110acc74ccfdcddf3d51
+  metadata.gz: 57f06cc073babb01bda236615fec3ef042b9f20165870b6468a2b9c1d9c9f25dcd1288ccbc21e78c38ed03bd703af79205fef1f83ed8fa03907516e184fda83c
+  data.tar.gz: 98ebbed71d2a26b88cc97784bc0006270ad211ef6e5398ac506cd78119e8d46f90067a415e6fe287ec5fcc38bc99596a5601bbe6d25e27276e051208461f7b0b

data/CHANGELOG.md

@@ -1,7 +1,15 @@
-## Unreleased
-* Add ability to toggle primary state store
-* Add environment variable interpolation for input variables
-* Add support for sops encryption / decryption of hierarchy data
+## 0.9.1 (Aug 11, 2019)
+
+BACKWARDS INCOMPATIBILITIES:
+- The legacy Atlas Artifact backend is no longer supported.
+
+IMPROVEMENTS:
+* Ability to toggle primary state store
+* Environment variable interpolation for input variables
+* Support for sops encryption / decryption of hierarchy data
+* Support for Terraform 0.12
+* Support for parallel execution of process.
+* A tool [prefixout](https://github.com/unifio/prefixout/releases/tag/v0.1.0) was added for additional logging.
 
 ## 0.7.8 (May 9, 2018)
 IMPROVEMENTS:

data/README.md

@@ -406,16 +406,6 @@ Module for interacting with the Terraform Enterprise backend.
 
 ### Usage
 
-Artifacts:
-
-```yaml
-ami:
-  type: 'atlas.artifact'
-  slug: 'unifio/app/amazon.ami'
-  version: 'latest'
-  key: 'region.us-east-1'
-```
-
 State Outputs:
 
 ```yaml
@@ -508,6 +498,7 @@ You will probably need the following packages installed locally
 - Terraform
 - Packer
 - Sops
+- [prefixout](https://github.com/unifio/prefixout/releases/tag/v0.1.0)
 
 Execute the following to build the gem:
 

data/lib/covalence.rb

@@ -1,6 +1,7 @@
 require "covalence/version"
 require "logger"
 require 'active_support/core_ext/object/blank'
+require 'etc'
 
 if %w(development test).include?(ENV['RAKE_ENV'])
   require 'byebug'
@@ -43,4 +44,7 @@ module Covalence
   LOGGER = Logger.new(STDOUT)
   LOG_LEVEL = String(ENV['COVALENCE_LOG'] || "warn").upcase
   LOGGER.level = Logger.const_get(LOG_LEVEL)
+
+  # worker count
+  WORKER_COUNT = ENV.has_key?('WORKER_COUNT') ? ENV['WORKER_COUNT'].to_i : Etc.nprocessors
 end

data/lib/covalence/core/cli_wrappers/popen_wrapper.rb

@@ -13,13 +13,14 @@ module Covalence
               stderr_io: STDERR,
               debug: Covalence::DEBUG_CLI,
               dry_run: false,
-              ignore_exitcode: false)
+              ignore_exitcode: false,
+              workdir: nil)
 
         # TODO: implement path prefix for the docker runs, see @tf_cmd
         cmd_string = [*cmds]
         # TODO: cmd escape issues with -var.
         cmd_string += [*args] unless args.blank?
-        cmd_string << path unless path.blank?
+        cmd_string << path unless workdir
 
         #TODO debug command string maybe
         #TODO debug command args maybe
@@ -34,12 +35,27 @@ module Covalence
           return 0 unless HighLine.new.agree('Execute? [y/n]')
         end
 
-        spawn_subprocess(ENV, run_cmd, {
-          stdin_io: stdin_io,
-          stdout_io: stdout_io,
-          stderr_io: stderr_io,
-          ignore_exitcode: ignore_exitcode
-        })
+        if workdir
+          spawn_subprocess(ENV, run_cmd,
+                           stdin_io: stdin_io,
+                           stdout_io: stdout_io,
+                           stderr_io: stderr_io,
+                           ignore_exitcode: ignore_exitcode,
+                           path: path,
+                           workdir: workdir)
+        else
+          spawn_subprocess(ENV, run_cmd,
+                           stdin_io: stdin_io,
+                           stdout_io: stdout_io,
+                           stderr_io: stderr_io,
+                           ignore_exitcode: ignore_exitcode,
+                           path: path)
+        end
+
+      end
+
+      def logger
+        Covalence::LOGGER
       end
 
       private
@@ -49,21 +65,27 @@ module Covalence
       end
 
       def spawn_subprocess(env, run_cmd,
-                           stdin_io: STDIN,
-                           stdout_io: STDOUT,
-                           stderr_io: STDERR,
-                           ignore_exitcode: false)
-        Signal.trap("INT") {} #Disable parent process from exiting, orphaning the child fork below
+                       stdin_io: STDIN,
+                       stdout_io: STDOUT,
+                       stderr_io: STDERR,
+                       ignore_exitcode: false,
+                       path: Dir.pwd,
+                       workdir: Dir.pwd)
+        logger.info "path: #{path} workdir: #{workdir} run_cmd: #{run_cmd}"
+        ## TODO one thing we can try is to use
+        # Prctl.call(Prctl::PR_SET_PDEATHSIG, Signal.list['TERM'], 0, 0, 0)
+        # so when the parent dies, child will know to terminate itself.
+        Signal.trap("INT") { logger.info "Trapped Ctrl-c. Disable parent process from exiting, orphaning the child fork below which may or may not work" }
         wait_thread = nil
-
-        Open3.popen3(env, run_cmd) do |stdin, stdout, stderr, wait_thr|
+        prefix=path.gsub(/^\/workspace*/,'')
+        whole_cmd=['prefixout', '-p', "#{prefix} ", '--'].concat(run_cmd.split)
+        Open3.popen3(env, *whole_cmd, :chdir=>workdir) do |stdin, stdout, stderr, wait_thr|
           mappings = { stdin_io => stdin, stdout => stdout_io, stderr => stderr_io }
           wait_thread = wait_thr
 
           Signal.trap("INT") {
             Process.kill("INT", wait_thr.pid)
             Process.wait(wait_thr.pid, Process::WNOHANG)
-
             exit(wait_thr.value.exitstatus)
           } # let SIGINT drop into the child process
 

data/lib/covalence/core/cli_wrappers/terraform_cli.rb

@@ -34,24 +34,68 @@ module Covalence
     end
 
     def self.terraform_check_style(path)
-      output, status = Open3.capture2e(ENV, Covalence::TERRAFORM_CMD, "fmt", "-write=false", path)
-      return false unless status.success?
-      output = output.split("\n")
-      (output.size == 0)
+      cmd = [Covalence::TERRAFORM_CMD, "fmt", "-check"]
+
+      output = PopenWrapper.run(
+          cmd,
+          path,
+          '',
+          ignore_exitcode: false)
+      (output == 0)
     end
 
-    def self.terraform_init(path='', args: '', ignore_exitcode: false)
+    def self.terraform_init(path: '', workdir: Dir.pwd, args: '', ignore_exitcode: false)
       if ENV['TF_PLUGIN_LOCAL'] == 'true'
-        cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"]
+        cmd = [Covalence::TERRAFORM_CMD, "init", "-get-plugins=false", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"]
       else
-        cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false"]
+        cmd = [Covalence::TERRAFORM_CMD, "init", "-get-plugins=false", "-get=false", "-input=false"]
       end
 
       output = PopenWrapper.run(
         cmd,
         path,
         args,
-        ignore_exitcode: ignore_exitcode)
+        ignore_exitcode: ignore_exitcode,
+        workdir: workdir)
+      (output == 0)
+    end
+
+    def self.terraform_get(path=Dir.pwd, workdir=Dir.pwd, args: '', ignore_exitcode: false)
+      cmd = [Covalence::TERRAFORM_CMD, "get", path]
+
+      output = PopenWrapper.run(
+          cmd,
+          path,
+          args,
+          ignore_exitcode: ignore_exitcode,
+          workdir: workdir)
+
+      (output == 0)
+    end
+
+    def self.terraform_plan(path: '', workdir: Dir.pwd, args: '', ignore_exitcode: false)
+      cmd = [Covalence::TERRAFORM_CMD, "plan"]
+
+      output = PopenWrapper.run(
+          cmd,
+          path,
+          args,
+          ignore_exitcode: ignore_exitcode,
+          workdir: workdir)
+
+      (output == 0)
+    end
+
+    def self.terraform_validate(path, workdir, args: '', ignore_exitcode: false)
+      cmd = [Covalence::TERRAFORM_CMD, "validate"]
+
+      output = PopenWrapper.run(
+          cmd,
+          path,
+          args,
+          ignore_exitcode: ignore_exitcode,
+          workdir: workdir)
+
       (output == 0)
     end
 
@@ -79,6 +123,9 @@ module Covalence
       raise "TODO: implement me"
     end
 
+    def self.logger
+      Covalence::LOGGER
+    end
 
     class << self
       private

data/lib/covalence/core/entities/input.rb

@@ -83,13 +83,27 @@ module Covalence
 
     # :reek:FeatureEnvy
     def parse_type(input)
+      Covalence::LOGGER.debug("parse_type input=#{input.inspect}")
+
       if input.stringify_keys.has_key?('type')
         type = input.stringify_keys.fetch('type')
 
+        # HACK This only pays attention to the first element in:
+        # - map(string)     -> ["map", "string"]
+        # - list(string)    -> ["list", "string"]
+        # But then again, this is exactly how Terraform 0.11.x forced
+        # us to handle it, so eh, yolo.
+        if type.respond_to?('each')
+          type = type[0]
+        end
+
         local_types = %w(
           list
+          set
           map
           string
+          number
+          tuple
         )
 
         if local_types.any? {|local_type| type == local_type }

data/lib/covalence/core/entities/stack.rb

@@ -41,14 +41,14 @@ module Covalence
       "#{environment_name}-#{name}"
     end
 
-    def materialize_cmd_inputs
+    def materialize_cmd_inputs(path)
       if type == "terraform"
         config = ""
         inputs.values.map(&:to_command_option).each do |input|
           config += input + "\n"
         end
-        logger.info "\nStack inputs:\n\n#{config}"
-        File.open('covalence-inputs.tfvars','w') {|f| f.write(config)}
+        logger.info "#{module_path} \nStack inputs:\n\n#{config}"
+        File.open("#{path}/covalence-inputs.tfvars",'w') {|f| f.write(config)}
       elsif type == "packer"
         config = Hash.new
         inputs.each do |name, input|

data/lib/covalence/core/services/packer_stack_tasks.rb

@@ -25,7 +25,7 @@ module Covalence
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
-          stack.materialize_cmd_inputs
+          stack.materialize_cmd_inputs(tmpdir)
           args = collect_args(stack.args,
                               additional_args,
                               "-var-file=covalence-inputs.json")
@@ -52,7 +52,7 @@ module Covalence
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
-          stack.materialize_cmd_inputs
+          stack.materialize_cmd_inputs(tmpdir)
           args = collect_args(stack.args,
                               additional_args,
                               "-var-file=covalence-inputs.json")

data/lib/covalence/core/services/terraform_stack_tasks.rb

@@ -26,27 +26,50 @@ module Covalence
     end
 
     # :reek:TooManyStatements
-    def stack_verify
+    def stack_shell
       Dir.mktmpdir do |tmpdir|
         populate_workspace(tmpdir)
         Dir.chdir(tmpdir) do
           logger.info "In #{tmpdir}:"
 
+          stack.materialize_state_inputs
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
 
-          stack.materialize_cmd_inputs
-          args = collect_args("-input=false",
-                              stack.args,
-                              "-var-file=covalence-inputs.tfvars")
-
-          TerraformCli.terraform_validate(args: args)
+          stack.materialize_cmd_inputs(tmpdir)
 
-          TerraformCli.terraform_plan(args: args)
+          shell = ENV.fetch('SHELL', 'sh')
+          system(shell)
         end
       end
     end
 
+    # :reek:TooManyStatements
+    def stack_verify(tmpdir)
+      return false if  !TerraformCli.terraform_check_style(@path)
+
+      return false if !TerraformCli.terraform_get(path=@path, workdir=tmpdir)
+      populate_workspace(tmpdir)
+
+      return false if !TerraformCli.terraform_init(path: @path, workdir: tmpdir)
+      stack.materialize_cmd_inputs(tmpdir)
+      args = collect_args("-input=false",
+                          stack.args,
+                          "-var-file=#{tmpdir}/covalence-inputs.tfvars")
+
+      tf_vers = Gem::Version.new(Covalence::TERRAFORM_VERSION)
+
+      if tf_vers >= Gem::Version.new('0.12.0')
+        # >= 0.12 does *not* support validating input vars
+        return false if !TerraformCli.terraform_validate(path=tmpdir, workdir=tmpdir, args: stack.args)
+      else
+        # < 0.12 supports validating input vars
+        return false if !TerraformCli.terraform_validate(path=tmpdir, workdir=tmpdir, args: args)
+      end
+
+      TerraformCli.terraform_plan(path: @path , workdir: tmpdir, args: args)
+    end
+
     # :reek:TooManyStatements
     def stack_refresh
       Dir.mktmpdir do |tmpdir|
@@ -60,7 +83,12 @@ module Covalence
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
 
-          TerraformCli.terraform_refresh
+          stack.materialize_cmd_inputs(tmpdir)
+          args = collect_args("-input=false",
+                              stack.args,
+                              "-var-file=covalence-inputs.tfvars")
+
+          TerraformCli.terraform_refresh(args: args)
         end
       end
     end
@@ -79,6 +107,8 @@ module Covalence
           TerraformCli.terraform_init
 
           stack.state_stores.drop(1).each do |store|
+            Covalence::LOGGER.debug("Stack: #{store.inspect}")
+
             stack.materialize_state_inputs(store: store)
             TerraformCli.terraform_init("-force-copy")
           end
@@ -99,13 +129,13 @@ module Covalence
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
 
-          stack.materialize_cmd_inputs
+          stack.materialize_cmd_inputs(tmpdir)
           args = collect_args("-input=false",
                               stack.args,
                               additional_args,
                               "-var-file=covalence-inputs.tfvars")
 
-          TerraformCli.terraform_plan(args: args)
+          TerraformCli.terraform_plan(path: @path, workdir: tmpdir, args: args)
         end
       end
     end
@@ -123,7 +153,7 @@ module Covalence
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
 
-          stack.materialize_cmd_inputs
+          stack.materialize_cmd_inputs(tmpdir)
           args = collect_args("-destroy",
                               "-input=false",
                               stack.args,
@@ -148,7 +178,7 @@ module Covalence
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
 
-          stack.materialize_cmd_inputs
+          stack.materialize_cmd_inputs(tmpdir)
           args = collect_args("-input=false",
                               "-auto-approve=true",
                               stack.args,
@@ -173,7 +203,7 @@ module Covalence
           TerraformCli.terraform_get(@path)
           TerraformCli.terraform_init
 
-          stack.materialize_cmd_inputs
+          stack.materialize_cmd_inputs(tmpdir)
           args = collect_args("-input=false",
                               "-auto-approve=true",
                               stack.args,
@@ -194,7 +224,6 @@ module Covalence
 
       # Copy any dependencies to the workspace
       @stack.dependencies.each do |dep|
-        logger.info "Copying '#{dep}' dependency to #{workspace}"
         dep_path = File.expand_path(File.join(Covalence::TERRAFORM, dep))
         FileUtils.cp_r dep_path, workspace
       end

data/lib/covalence/core/state_stores/s3.rb

@@ -53,11 +53,19 @@ module Covalence
         end
 
         # Determine whether the document is a Terraform state file
-        tf_state = true if parsed.has_key?('modules')
+        tf_state = true if parsed.has_key?('terraform_version')
 
         # Return ID for the key specified
         if tf_state
-          outputs = parsed.fetch('modules')[0].fetch('outputs')
+          tf_vers = Gem::Version.new(parsed.fetch('terraform_version'))
+
+          if tf_vers >= Gem::Version.new('0.12.0')
+            root = parsed
+          else
+            root = parsed.fetch('modules')[0]
+          end
+
+          outputs = root.fetch('outputs')
           return outputs.fetch(name)
         end
         return parsed.fetch(name) if parsed.has_key?(name)
@@ -98,9 +106,9 @@ CONF
         end
       end
 
-      params.delete('name')
-
       params.each do |k,v|
+        next if k == 'name'
+
         v = Covalence::Helpers::ShellInterpolation.parse_shell(v) if v.to_s.include?("$(")
         config += "    #{k} = \"#{v}\"\n"
       end

data/lib/covalence/environment_tasks.rb

@@ -156,7 +156,13 @@ module Covalence
       desc "Verify the #{stack_name} stack of the #{environment_name} environment"
       # Maybe verify_local to highlight that it skips pulling in remote state
       task generate_rake_taskname(environment_name, stack_name, "verify") do
-        tf_tasks.stack_verify
+        _tmp_dir = Dir.mktmpdir
+        tf_tasks.stack_verify(_tmp_dir)
+      end
+
+      desc "Shell into the #{stack_name} stack of the #{environment_name} environment"
+      task generate_rake_taskname(environment_name, stack_name, "shell") do
+        tf_tasks.stack_shell
       end
     end
 

data/lib/covalence/rake/rspec/envs_spec.rb

@@ -5,71 +5,35 @@ module Covalence
     TEST_ENVS.include?(environ.name.to_s)
   end
 
+  jobs = Queue.new
+  # populate the job queue
   environments.each do |environment|
     environment.stacks.each do |stack|
       EnvironmentRepository.populate_stack(stack)
-      case stack.type
-      when 'terraform'
-        tf_tasks = TerraformStackTasks.new(stack)
-      when 'packer'
-        packer_tasks = PackerStackTasks.new(stack)
-      end
-
-      path = File.expand_path(File.join(TERRAFORM, stack.module_path))
-
-      describe "Verify #{environment.name}:#{stack.name}" do
-
-        if stack.type == 'terraform'
-
-          it 'passes style check' do
-            expect {
-              expect(TerraformCli.terraform_check_style(path)).to be true
-            }.to_not raise_error
-          end
-
-          it 'passes validation' do
-            tmp_dir = Dir.mktmpdir
-            # Copy module to the workspace
-            FileUtils.copy_entry path, tmp_dir
-
-            # Copy any dependencies to the workspace
-            stack.dependencies.each do |dep|
-              dep_path = File.expand_path(File.join(Covalence::TERRAFORM, dep))
-              FileUtils.cp_r dep_path, tmp_dir
-            end
-
-            Dir.chdir(tmp_dir) do
-              expect {
-                expect(TerraformCli.terraform_get(path)).to be true
-              }.to_not raise_error
-
-              expect {
-                expect(TerraformCli.terraform_init).to be true
-              }.to_not raise_error
-
-              stack.materialize_cmd_inputs
-
-              expect {
-                expect(TerraformCli.terraform_validate("-input=false -var-file=covalence-inputs.tfvars")).to be true
-              }.to_not raise_error
-            end
-          end
-
-          it 'passes execution' do
-            expect {
-              expect(tf_tasks.stack_verify).to be true
-            }.to_not raise_error
-          end
-
-        elsif stack.type == 'packer'
-
-          it 'passes validation' do
-            expect {
-              expect(packer_tasks.context_validate(stack.contexts.first.to_packer_command_options)).to be true
-            }.to_not raise_error
+      jobs.push(stack)
+    end
+  end
+  Covalence::LOGGER.info "jobs.length: #{jobs.length}"
+
+  # start the workers
+  myworkers = (Covalence::WORKER_COUNT).times.map do
+    Thread.new do
+      begin
+        while stack = jobs.pop(true)
+          case stack.type
+          when 'terraform'
+            _tmp_dir = Dir.mktmpdir
+            tf_tasks = TerraformStackTasks.new(stack)
+            exit false if !tf_tasks.stack_verify(_tmp_dir)
+          when 'packer'
+            packer_tasks = PackerStackTasks.new(stack)
+            exit false if !packer_tasks.context_validate(stack.contexts.first.to_packer_command_options)
           end
         end
+      rescue ThreadError => e
       end
     end
   end
+  myworkers.map(&:join)
 end
+

data/lib/covalence/version.rb

@@ -1,3 +1,3 @@
 module Covalence
-  VERSION = "0.8.4"
+  VERSION = "0.9.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: covalence
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.9.1
 platform: ruby
 authors:
 - Unif.io
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-01 00:00:00.000000000 Z
+date: 2019-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: deep_merge
@@ -318,6 +318,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.16.1
+- !ruby/object:Gem::Dependency
+  name: solargraph
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - support@unif.io
@@ -383,7 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: A tool for the management and orchestration of data used by HashiCorp infrastructure