covalence 0.0.1 → 0.7.9.rc1

data/TODO.md

@@ -0,0 +1,14 @@
+- individual state-stores: would be nice if it had something to describe the backing input types
+- consider httparty vs rest-client
+- Use the invalid yaml syntax to figure out more things in rake tasks that needs to be lazy loaded
+- Like the idea of maybe splitting out the syntax elements to a different gem. Gem installation & management would become easier
+- missing spec around reports?
+- Look into muting the reporter as dev default: https://github.com/ci-reporter/ci_reporter
+- Look into getting HieraDB wrapper to read .yml files
+- Terraform CLI in docker probably isn't going anywhere, might need to figure out a nicer way of handling this
+- Use covalence to do verification on the open source stacks:
+  - terraform-aws-openvpn
+- Plugin architecture
+- rollup tasks in terraform need to check terraform only tasks
+- stack_repository: backfill packer related tests
+- packer: utils to gen uuids as inputs?

data/lib/covalence.rb

@@ -0,0 +1,41 @@
+require "covalence/version"
+require "logger"
+require 'active_support/core_ext/object/blank'
+
+if %w(development test).include?(ENV['RAKE_ENV'])
+  require 'byebug'
+  require 'awesome_print'
+end
+
+# :reek:TooManyConstants
+module Covalence
+  # Configurable constants
+  #TODO: look into how WORKSPACE is being used, maybe this can just be an internal ROOT and make CONFIG not depend on WORKSPACE
+  WORKSPACE = File.absolute_path(ENV['COVALENCE_WORKSPACE'] || '.')
+  CONFIG = File.join(WORKSPACE, (ENV['COVALENCE_CONFIG'] || 'covalence.yaml'))
+  # TODO: could use better naming
+  PACKER = File.absolute_path(File.join(WORKSPACE, (ENV['COVALENCE_PACKER_DIR'] || '.')))
+  TERRAFORM =  File.absolute_path(File.join(WORKSPACE, (ENV['COVALENCE_TERRAFORM_DIR'] || '.')))
+  TEST_ENVS = (ENV['COVALENCE_TEST_ENVS'] || "").split(',')
+  # Reserved namespace including default ci and spec
+  RESERVED_NS = [(ENV['COVALENCE_RESERVED_NAMESPACE'] || "").split(','), 'ci', 'spec']
+
+  TERRAFORM_CMD = ENV['TERRAFORM_CMD'] || "terraform"
+  TERRAFORM_VERSION = ENV['TERRAFORM_VERSION'] || `#{TERRAFORM_CMD} version`.split("\n", 2)[0].gsub('Terraform v','')
+  TERRAFORM_PLUGIN_CACHE = File.absolute_path("#{ENV['TF_PLUGIN_CACHE_DIR']}/linux_amd64" || "#{ENV['HOME']}/.terraform.d/plugin-cache/linus_amd64")
+
+  PACKER_CMD = ENV['PACKER_CMD'] || "packer"
+
+  # No-op shell command. Should not need to modify for most unix shells.
+  DRY_RUN_CMD = (ENV['COVALENCE_DRY_RUN_CMD'] || ":")
+  DEBUG_CLI = (ENV['COVALENCE_DEBUG'] || 'false') =~ (/(true|t|yes|y|1)$/i)
+
+  #DOCKER_ENV_FILE
+
+  # Internal constants
+  GEM_ROOT = File.expand_path('covalence', File.dirname(__FILE__)).freeze
+  # look into logger-colors
+  LOGGER = Logger.new(STDOUT)
+  LOG_LEVEL = String(ENV['COVALENCE_LOG'] || "warn").upcase
+  LOGGER.level = Logger.const_get(LOG_LEVEL)
+end

data/lib/covalence/core/bootstrap.rb

@@ -0,0 +1,8 @@
+require 'yaml'
+
+require_relative '../../covalence'
+
+Dir[File.expand_path("cli_wrappers/**/*.rb", File.dirname(__FILE__))].sort.each { |file| require file }
+Dir[File.expand_path("entities/**/*.rb", File.dirname(__FILE__))].sort.each { |file| require file }
+Dir[File.expand_path("repositories/**/*.rb", File.dirname(__FILE__))].sort.each { |file| require file }
+Dir[File.expand_path("services/**/*.rb", File.dirname(__FILE__))].sort.each { |file| require file }

data/lib/covalence/core/cli_wrappers/packer.yml

@@ -0,0 +1,9 @@
+---
+version: Packer v1.0.0
+commands:
+  build:
+  fix:
+  inspect:
+  push:
+  validate:
+  version:

data/lib/covalence/core/cli_wrappers/packer_cli.rb

@@ -0,0 +1,27 @@
+module Covalence
+  class PackerCli
+    class << self
+      def require_init()
+        cmds_yml = File.expand_path("packer.yml", __dir__)
+        init_packer_cmds(cmds_yml)
+      end
+
+      private
+      def init_packer_cmds(file)
+        definition = YAML.load_file(file)
+
+        definition['commands'].each do |cmd, _|
+          packer_cmd = "packer_#{cmd}"
+
+          next if respond_to?(packer_cmd.to_sym)
+          define_singleton_method(packer_cmd) do |template, args: ''|
+            output = PopenWrapper.run([Covalence::PACKER_CMD, cmd], template, args)
+            (output == 0)
+          end #define_singleton_method
+        end # definition
+      end
+    end # class << self
+  end #PackerCli
+end
+
+Covalence::PackerCli.require_init

data/lib/covalence/core/cli_wrappers/popen_wrapper.rb

@@ -0,0 +1,123 @@
+require 'open3'
+require 'highline'
+
+require_relative '../../../covalence'
+
+module Covalence
+  class PopenWrapper
+    class << self
+
+      def run(cmds, path, args,
+              stdin_io: STDIN,
+              stdout_io: STDOUT,
+              stderr_io: STDERR,
+              debug: Covalence::DEBUG_CLI,
+              dry_run: false,
+              ignore_exitcode: false)
+
+        # TODO: implement path prefix for the docker runs, see @tf_cmd
+        cmd_string = [*cmds]
+        # TODO: cmd escape issues with -var.
+        cmd_string += [*args] unless args.blank?
+        cmd_string << path unless path.blank?
+
+        #TODO debug command string maybe
+        #TODO debug command args maybe
+
+        run_cmd = cmd_string.join(' ')
+        print_cmd_string(run_cmd)
+        if dry_run
+          run_cmd = Covalence::DRY_RUN_CMD
+        end
+
+        if debug
+          return 0 unless HighLine.new.agree('Execute? [y/n]')
+        end
+
+        spawn_subprocess(ENV, run_cmd, {
+          stdin_io: stdin_io,
+          stdout_io: stdout_io,
+          stderr_io: stderr_io,
+          ignore_exitcode: ignore_exitcode
+        })
+      end
+
+      private
+      def print_cmd_string(cmd_string)
+        Covalence::LOGGER.warn "---"
+        Covalence::LOGGER.warn cmd_string
+      end
+
+      def spawn_subprocess(env, run_cmd,
+                           stdin_io: STDIN,
+                           stdout_io: STDOUT,
+                           stderr_io: STDERR,
+                           ignore_exitcode: false)
+        Signal.trap("INT") {} #Disable parent process from exiting, orphaning the child fork below
+        wait_thread = nil
+
+        Open3.popen3(env, run_cmd) do |stdin, stdout, stderr, wait_thr|
+          mappings = { stdin_io => stdin, stdout => stdout_io, stderr => stderr_io }
+          wait_thread = wait_thr
+
+          Signal.trap("INT") {
+            Process.kill("INT", wait_thr.pid)
+            Process.wait(wait_thr.pid, Process::WNOHANG)
+
+            exit(wait_thr.value.exitstatus)
+          } # let SIGINT drop into the child process
+
+          handle_io_streams(mappings, stdin_io)
+        end
+
+        Signal.trap("INT") { exit } #Restore parent SIGINT
+
+        return 0 if ignore_exitcode
+        exit(wait_thread.value.exitstatus) unless wait_thread.value.success?
+        return wait_thread.value.exitstatus
+      end
+
+      def handle_io_streams(mappings, stdin_io)
+        inputs = mappings.keys
+        streams_ready_for_eof_check = []
+
+        until inputs.empty? || (inputs.size == 1 && inputs.first == stdin_io) do
+
+          readable_inputs, _ = IO.select(inputs, [], [])
+          streams_ready_for_eof_check = readable_inputs
+
+          streams_ready_for_eof_check.select(&:eof).each do |src|
+            Covalence::LOGGER.debug "Stopping redirection from an IO in EOF: " + src.inspect
+            # `select`ing an IO which has reached EOF blocks forever.
+            # So you have to delete such IO from the array of IOs to `select`.
+            inputs.delete src
+
+            # You must close the child process' STDIN immeditely after the parent's STDIN reached EOF,
+            # or some kinds of child processes never exit.
+            # e.g.) echo foobar | joumae run -- cat
+            # After the `echo` finished outputting `foobar`, you have to tell `cat` about that or `cat` will wait for more inputs forever.
+            mappings[src].close if src == stdin_io
+          end
+
+          break if inputs.empty? || (inputs.size == 1 && inputs.first == stdin_io)
+
+          readable_inputs.each do |input|
+            begin
+              data = input.read_nonblock(1024)
+              output = mappings[input]
+              output.write(data)
+              output.flush
+            rescue EOFError => e
+              Covalence::LOGGER.debug "Reached EOF: #{e}"
+              inputs.delete input
+            rescue Errno::EPIPE => e
+              Covalence::LOGGER.debug "Handled error: #{e}: io: #{input.inspect}"
+              inputs.delete input
+            end
+          end #readable_inputs
+        end #until inputs.empty?
+      end #handle_io_streams
+
+    end
+  end
+end

data/lib/covalence/core/cli_wrappers/terraform.yml

@@ -0,0 +1,39 @@
+---
+version: Terraform v0.10.6
+commands:
+  apply:
+  #console:
+  destroy:
+  #env:
+    #list:
+    #select:
+    #new:
+    #delete:
+  fmt:
+  #force-unlock
+  get:
+  graph:
+  #import
+  #init:
+  #output:
+  plan:
+  providers:
+  push:
+  refresh:
+  show:
+  state:
+    #list:
+    #mv:
+    #pull:
+    #push:
+    #rm:
+    #show:
+  #taint:
+  validate:
+  #untaint:
+  workspace:
+    list:
+    select:
+    new:
+    delete:
+  version:

data/lib/covalence/core/cli_wrappers/terraform_cli.rb

@@ -0,0 +1,119 @@
+require 'yaml'
+require 'fileutils'
+require 'tmpdir'
+require 'active_support/core_ext/object/blank'
+require 'open3'
+require 'semantic'
+
+require_relative '../../../covalence'
+require_relative 'popen_wrapper'
+
+module Covalence
+  class TerraformCli
+    def self.require_init()
+      if Semantic::Version.new(Covalence::TERRAFORM_VERSION) < Semantic::Version.new("0.9.0")
+        raise "Terraform v0.9.0 or newer required"
+      else
+        cmds_yml = File.expand_path("terraform.yml", __dir__)
+      end
+      init_terraform_cmds(cmds_yml)
+    end
+
+    # :reek:BooleanParameter
+    def self.terraform_clean(path, dry_run: false, verbose: true)
+      # standard run shouldn't need this since it does a chdir on a temp dir anyway
+      # something about cln_cmd when working with docker images
+      targets = [ File.join(File.expand_path(path), ".terraform") ] +
+        Dir.glob(File.join(File.expand_path(path), "*.tfstate*"))
+
+      FileUtils.rm_rf(targets, {
+        noop: dry_run,
+        verbose: verbose,
+        secure: true,
+      })
+    end
+
+    def self.terraform_check_style(path)
+      output, status = Open3.capture2e(ENV, Covalence::TERRAFORM_CMD, "fmt", "-write=false", path)
+      return false unless status.success?
+      output = output.split("\n")
+      (output.size == 0)
+    end
+
+    def self.terraform_init(path='', args: '', ignore_exitcode: false)
+      if ENV['TF_PLUGIN_LOCAL'] == 'true'
+        cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false", "-plugin-dir=#{Covalence::TERRAFORM_PLUGIN_CACHE}"]
+      else
+        cmd = [Covalence::TERRAFORM_CMD, "init", "-get=false", "-input=false"]
+      end
+
+      output = PopenWrapper.run(
+        cmd,
+        path,
+        args,
+        ignore_exitcode: ignore_exitcode)
+      (output == 0)
+    end
+
+    def self.terraform_workspace(workspace, path='', args: '', ignore_exitcode: false)
+      cmd = [Covalence::TERRAFORM_CMD, "workspace", "new", workspace]
+
+      output = PopenWrapper.run(
+        cmd,
+        path,
+        args,
+        ignore_exitcode: ignore_exitcode)
+
+      (output == 0)
+    end
+
+    def self.terraform_output(output_var, args: '')
+      raise "TODO: implement me"
+    end
+
+    def self.terraform_taint(resource_name, args: '')
+      raise "TODO: implement me"
+    end
+
+    def self.terraform_untaint(resource_name, args: '')
+      raise "TODO: implement me"
+    end
+
+
+    class << self
+      private
+
+      # The only args that should be automated are ones that only expect some DIR/PATH as it's only
+      # required arg, most other things need a little bit more manual definition.
+      def init_terraform_cmds(file)
+        definition = YAML.load_file(file)
+
+        definition['commands'].each do |cmd, sub_hash|
+          if sub_hash.blank?
+            terraform_cmd = "terraform_#{cmd}"
+
+            next if respond_to?(terraform_cmd.to_sym)
+            define_singleton_method(terraform_cmd) do |path=Dir.pwd(), args: ''|
+              output = PopenWrapper.run([Covalence::TERRAFORM_CMD, cmd], path, args)
+              (output == 0)
+            end
+          elsif sub_hash.is_a?(Hash)
+            sub_hash.keys.each do |sub_command|
+              terraform_cmd = "terraform_#{cmd}_#{sub_command}"
+
+              next if respond_to?(terraform_cmd.to_sym)
+              define_singleton_method(terraform_cmd) do |path=Dir.pwd(), args: ''|
+                output = PopenWrapper.run([Covalence::TERRAFORM_CMD, cmd, sub_command], path, args)
+                (output == 0)
+              end
+            end
+          else
+            raise "Invalid yml context"
+          end
+        end
+      end #init_terraform_cmds
+    end #private
+  end #TerraformCli
+end
+
+Covalence::TerraformCli.require_init

data/lib/covalence/core/data_stores/hiera.rb

@@ -0,0 +1,50 @@
+require_relative '../../../covalence'
+require 'yaml'
+require 'hiera'
+
+module Covalence
+  module HieraDB
+    # TODO: maybe HieraWrapper
+    # :reek:DataClump
+    class Client
+      attr_reader :scope
+
+      def initialize(config, scope = {})
+        @config = config
+        @scope = scope
+
+        begin
+          @client = Hiera.new(:config => config)
+        rescue RuntimeError => e
+          Covalence::LOGGER.error e.message
+          exit 1
+        end
+      end
+
+      def initialize_scope(scope)
+        self.class.new(@config, scope)
+      end
+
+      def set_scope(env, stack)
+        @scope = {
+          "environment" => env,
+          "stack" => stack
+        }
+      end
+
+      def lookup(key, default = nil, scope = @scope)
+        @client.lookup(key, default, scope)
+      end
+
+      def hash_lookup(key, default = nil, scope = @scope)
+        # https://github.com/puppetlabs/hiera/blob/d7ed74f4eec8f4fb1aa84cd0e158a595f86debd4/lib/hiera/backend.rb#L241
+        # def lookup(key, default, scope, order_override, resolution_type, context = {:recurse_guard => nil})
+        @client.lookup(key, default, scope, nil, :hash)
+      end
+
+      def array_lookup(key, default = nil, scope = @scope)
+        @client.lookup(key, default, scope, nil, :array)
+      end
+    end
+  end
+end

data/lib/covalence/core/entities/context.rb

@@ -0,0 +1,38 @@
+require 'active_support/core_ext/object/blank'
+require 'virtus'
+require 'active_model'
+
+module Covalence
+  # Maybe just call this targets
+  class Context
+    include Virtus.model
+    include ActiveModel::Validations
+
+    attribute :name, String, default: ''
+    attribute :values, Array, default: []
+
+    validates! :name, format: {
+      without: /(\s+|,)/,
+      message: "Context %{attribute}: \"%{value}\" cannot contain spaces or commas"
+    }
+
+    def initialize(attributes = {}, *args)
+      super
+      self.valid?
+    end
+
+    def namespace
+      return "" if name.blank?
+      "#{name}:"
+    end
+
+    def to_command_options
+      values.map { |value| "-target=\"#{value}\"" }
+    end
+
+    def to_packer_command_options
+      return "" if values.blank?
+      "-only=#{values.join(',')}"
+    end
+  end
+end