auster 0.2.2

data/lib/cfer/auster/repo.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+module Cfer
+  module Auster
+    class Repo
+      include Cfer::Auster::Logging::Mixin
+
+      REPO_FILE = ".auster.yaml"
+      STEP_REGEX = /(?<count>[0-9]{2})\.(?<tag>[a-z0-9][a-z0-9\-]*[a-z0-9])/
+      INT_REGEX = /[0-9]+/
+
+      attr_reader :root
+      attr_reader :options
+
+      attr_reader :steps
+      attr_reader :config_sets
+
+      attr_reader :param_validator
+
+      def initialize(root)
+        raise "root must be a String." unless root.is_a?(String)
+        @root = File.expand_path(root).freeze
+
+        logger.debug "Repo location: #{@root}"
+
+        options_file = File.expand_path(REPO_FILE, @root)
+        logger.debug "Loading options file..."
+        raise "#{options_file} does not exist." unless File.file?(options_file)
+        @options = YAML.load_file(options_file)
+
+        @cfg_root = File.join(@root, "config").freeze
+        raise "#{@cfg_root} does not exist." unless File.directory?(@cfg_root)
+        @step_root = File.join(@root, "steps").freeze
+        raise "#{@step_root} does not exist." unless File.directory?(@step_root)
+
+        logger.debug "Enumerating steps..."
+        steps = build_steps
+        step_tags = steps.map(&:tag)
+        step_counts = steps.map(&:count)
+
+        raise "Multiple steps with the same tag found. Can't continue." \
+          if step_tags.uniq.length != steps.length
+        raise "Multiple steps with the same count found. Can't continue." \
+          if step_counts.uniq.length != steps.length
+
+        @steps = steps.map { |step| [step.count, step] }.to_h.freeze
+        @steps_by_tag = steps.map { |step| [step.tag, step.count] }.to_h.freeze
+
+        logger.debug "Enumerating config sets..."
+        @config_sets = find_config_sets
+
+        logger.debug "Loading param validator..."
+        @param_validator = load_param_validator
+      end
+
+      def step_by_tag(tag)
+        raise "Couldn't find a step with tag '#{tag}'." unless @steps_by_tag.key?(tag)
+        @steps[@steps_by_tag[tag]]
+      end
+
+      def step_by_count(count)
+        @steps[count.to_i]
+      end
+
+      def step_by_count_or_tag(value)
+        if INT_REGEX.match(value)
+          step_by_count(value)
+        else
+          step_by_tag(value)
+        end
+      end
+
+      def ordered_steps
+        @steps.sort.map(&:last)
+      end
+
+      def config_set(id)
+        raise "Config set not found with id '#{id}'." unless @config_sets.include?(id)
+
+        tokens = id.split("/")
+
+        name = tokens[1]
+        region = tokens[0]
+
+        filename = File.join(@cfg_root, "#{id}.yaml")
+
+        schema_file = File.join(@cfg_root, "schema.yaml")
+        schema_file = nil unless File.file?(schema_file)
+
+        Cfer::Auster::Config.from_file(name: name, aws_region: region,
+                                       schema_file: schema_file, data_file: filename)
+      end
+
+      def nuke(config_set)
+        raise "config_set must be a Cfer::Auster::Config." unless config_set.is_a?(Cfer::Auster::Config)
+
+        logger.info "Nuking '#{config_set.full_name}'."
+
+        ordered_steps.reverse.each { |step| step.destroy(config_set) }
+      end
+
+      private
+
+      def build_steps
+        Dir[File.join(@step_root, "*")].map do |step_dir|
+          step_name = File.basename(step_dir)
+          match = STEP_REGEX.match(step_name)
+
+          if match.nil?
+            logger.warn "Unrecognized directory '#{step_name}' in steps, skipping."
+            nil
+          else
+            logger.debug "Step found: #{match[:count]}.#{match[:tag]}"
+            Cfer::Auster::Step.new(repo: self, count: match[:count], tag: match[:tag], directory: step_dir)
+          end
+        end.reject(&:nil?)
+      end
+
+      def find_config_sets
+        Dir[File.join(@cfg_root, "**/*.yaml")].map do |cfgset_file|
+          if File.dirname(cfgset_file) == @cfg_root
+            nil # because it's not inside an AWS region
+          else
+            # this could probably be cleaner.
+            cfgset_file.gsub(@cfg_root, "").gsub(".yaml", "")[1..-1]
+          end
+        end.reject(&:nil?)
+      end
+
+      def load_param_validator
+        validator_path = File.join(@cfg_root, "validator.rb")
+
+        if File.file?(validator_path)
+          validator_rb = IO.read(validator_path)
+          validator = instance_eval(validator_rb, validator_path)
+
+          raise "#{validator_path} error: must return a Cfer::Auster::ParamValidator." \
+            unless validator.is_a?(Cfer::Auster::ParamValidator)
+
+          validator
+        else
+          Cfer::Auster::ParamValidator.new { |_, _| }
+        end
+      end
+
+      class << self
+        def discover_from_cwd
+          require "search_up"
+
+          repo_file = SearchUp.search(Dir.pwd, REPO_FILE) { |f| File.file?(f) }.first
+
+          raise "No repo found (signaled by #{REPO_FILE}) from pwd to root." if repo_file.nil?
+
+          Cfer::Auster::Repo.new File.dirname(repo_file)
+        end
+      end
+    end
+  end
+end

data/lib/cfer/auster/script_executor.rb

@@ -0,0 +1,57 @@
+module Cfer
+  module Auster
+    class ScriptExecutor
+      attr_reader :logger
+
+      def initialize(consts)
+        raise "consts must be a Hash" unless consts.is_a?(Hash)
+
+        vars = []
+        consts.each_pair do |k, v|
+          key_name = k.downcase.to_sym
+          vars << key_name
+          define_singleton_method(key_name) { v }
+        end
+
+        define_singleton_method(:vars) { vars }
+      end
+
+      def run(filename)
+        @logger = Logger.new(Cfer::Auster::Logging.logdev)
+        @logger.level = Cfer::Auster::Logging.logger.level
+
+        basename = File.basename(filename)
+        standard_formatter = Logger::Formatter.new
+
+        logger.formatter = proc do |severity, datetime, progname, msg|
+          standard_formatter.call(severity, datetime, progname, "> #{basename}: #{msg}")
+        end
+
+        instance_eval IO.read(filename), filename
+      end
+
+      private
+
+      def exports(export_plan_id = nil)
+        export_plan_id ||= plan_id
+
+        cfn_client = Aws::CloudFormation::Client.new(region: config_set.aws_region)
+
+        # this call isn't super fast, so we cache a little bit.
+        @exports ||= {}
+        @exports[export_plan_id] ||=
+          cfn_client.list_exports.exports.map do |cfn_export|
+            tokens = cfn_export.name.split("--", 2)
+
+            if tokens.length != 2 || tokens[0] != export_plan_id
+              nil
+            else
+              [tokens[1].to_sym, cfn_export.value]
+            end
+          end.reject(&:nil?).to_h
+
+        @exports[export_plan_id]
+      end
+    end
+  end
+end

data/lib/cfer/auster/step.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "semantic"
+
+module Cfer
+  module Auster
+    class Step
+      include Cfer::Auster::Logging::Mixin
+
+      attr_reader :repo
+
+      attr_reader :count
+      attr_reader :tag
+      attr_reader :directory
+
+      def initialize(repo:, count:, tag:, directory:)
+        raise "directory '#{directory}' does not exist." unless File.directory?(directory)
+        @repo = repo
+
+        @count = count.to_i
+        @tag = tag.dup.freeze
+
+        @directory = directory.dup.freeze
+
+        @cfer_directory = File.join(@directory, "cfer")
+        raise "directory '#{@cfer_directory}' does not exist." unless File.directory?(directory)
+      end
+
+      def json(config_set)
+        raise "config_set must be a Cfer::Auster::Config." unless config_set.is_a?(Cfer::Auster::Config)
+
+        with_cfer(config_set) do |cfer|
+          cfer.generate_json
+        end
+      end
+
+      def cfn_stack_name(config_set)
+        "#{config_set.name}-step#{count.to_s.rjust(2, '0')}"
+      end
+
+      def cfn_stack(config_set)
+        cfn_client = Aws::CloudFormation::Resource.new(region: config_set.aws_region)
+        cfn_client.stack(cfn_stack_name(config_set))
+      end
+
+      def cfn_data(config_set)
+        stack = cfn_stack(config_set)
+
+        if !stack.exists?
+          nil
+        else
+          cfn_parameters = stack.parameters.map { |p| [p.parameter_key, p.parameter_value] }.to_h
+          cfn_outputs = stack.outputs.map { |o| [o.output_key, o.output_value] }.to_h
+
+          cfn_combined = cfn_parameters.merge(cfn_outputs)
+
+          {
+            inputs: cfn_parameters.symbolize_keys,
+            outputs: cfn_outputs.symbolize_keys,
+            combined: cfn_combined.symbolize_keys
+          }
+        end
+      end
+
+      def run(config_set)
+        raise "config_set must be a Cfer::Auster::Config." unless config_set.is_a?(Cfer::Auster::Config)
+
+        with_cfer(config_set) do |cfer|
+          do_on_create(config_set) unless cfn_stack(config_set).exists?
+          do_pre_converge(config_set)
+
+          logger.info "Converging Cfer stack as '#{cfn_stack_name(config_set)}'."
+          cfer.converge!(block: true)
+
+          do_post_converge(config_set, cfn_data(config_set))
+        end
+      end
+
+      def destroy(config_set)
+        raise "config_set must be a Cfer::Auster::Config." unless config_set.is_a?(Cfer::Auster::Config)
+
+        stack_name = cfn_stack_name(config_set)
+        stack = cfn_stack(config_set)
+
+        if !stack.exists?
+          logger.warn "No underlying CloudFormation stack '#{stack_name}' exists."
+        else
+          logger.info "Deleting CloudFormation stack '#{stack_name}'."
+          with_cfer(config_set) do |cfer|
+            cfer.destroy!(block: true)
+          end
+
+          do_on_destroy(config_set)
+        end
+      end
+
+      private
+
+      def do_on_create(config_set)
+        run_scripts("on-create", config_set)
+      end
+
+      def do_on_destroy(config_set)
+        run_scripts("on-destroy", config_set)
+      end
+
+      def do_pre_converge(config_set)
+        run_scripts("pre-converge", config_set)
+      end
+
+      def do_post_converge(config_set, cfn_data)
+        run_scripts("post-converge", config_set, cfn_data: cfn_data)
+      end
+
+      def with_cfer(config_set, &block)
+        parameters = config_set.data
+        validation_errors = repo.param_validator.validate(parameters)
+
+        unless validation_errors.empty?
+          logger.error "Config set '#{config_set.full_name}' failed to validate with the following errors:"
+          validation_errors.each do |e|
+            logger.error " - #{e}"
+          end
+
+          raise "Validation of config set '#{config_set.full_name}' failed during Cfer build."
+        end
+
+        cfer = CferEvaluator.new(path: @cfer_directory, stack_name: cfn_stack_name(config_set),
+                                 parameters: parameters, metadata: stack_metadata)
+
+        block.call(cfer)
+      end
+
+      def stack_metadata
+        {
+          Version: Semantic::Version.new(Cfer::Auster::VERSION).to_h.reject { |k, v| v.nil? }
+        }
+      end
+
+      def converge_cfn(config_set)
+      end
+
+      def run_scripts(event_type, config_set, rb_vars = {})
+        logger.info "Running scripts for #{event_type}"
+        env_vars = config_set.env_vars_for_shell
+
+        injected_vars = env_vars.merge(rb_vars).merge(
+          repo: repo,
+          step: self,
+          config_set: config_set
+        )
+
+        scripts_for_event(event_type).each do |script_name|
+          case File.extname(script_name)
+          when ".rb"
+            logger.debug "Executing #{script_name} in-process."
+            Cfer::Auster::ScriptExecutor.new(injected_vars).run(script_name)
+          else
+            logger.debug "Executing #{script_name} through system()."
+            system(env_vars, script_name)
+
+            raise "#{script_name} failed with exit code #{$?.exitstatus}." unless $?.success?
+          end
+        end
+      end
+
+      def scripts_for_event(event_type)
+        dir = File.join(directory, "#{event_type}.d")
+
+        raise "#{dir} exists, but it isn't a directory.'" \
+          if File.exist?(dir) && !File.directory?(dir)
+
+        if !File.directory?(dir)
+          []
+        else
+          Dir[File.join(dir, "*")].select { |f| File.executable?(f) }
+        end
+      end
+    end
+  end
+end

data/lib/cfer/auster/version.rb

@@ -0,0 +1,5 @@
+module Cfer
+  module Auster
+    VERSION = "0.2.2".freeze
+  end
+end

data/lib/cfer/auster.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+require "cfer/auster/version"
+
+require "yaml"
+
+require "active_support/all"
+require "aws-sdk"
+require "ice_nine"
+
+module Cfer
+  module Auster
+    autoload :CferEvaluator,  "cfer/auster/cfer_evaluator"
+    autoload :CferHelpers,    "cfer/auster/cfer_helpers"
+    autoload :CLI,            "cfer/auster/cli"
+    autoload :Config,         "cfer/auster/config"
+    autoload :Logging,        "cfer/auster/logging"
+    autoload :ParamValidator, "cfer/auster/param_validator"
+    autoload :Repo,           "cfer/auster/repo"
+    autoload :ScriptExecutor, "cfer/auster/script_executor"
+    autoload :Step,           "cfer/auster/step"
+  end
+end

metadata

@@ -0,0 +1,270 @@
+--- !ruby/object:Gem::Specification
+name: auster
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Ed Ropple
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-04-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: search_up
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
+- !ruby/object:Gem::Dependency
+  name: ice_nine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.2
+- !ruby/object:Gem::Dependency
+  name: kwalify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+- !ruby/object:Gem::Dependency
+  name: cfer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: semantic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.11
+description: 
+email:
+- ed+auster@edropple.com
+executables:
+- auster
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- auster.gemspec
+- bin/auster
+- bin/console
+- example-repo/.auster.yaml
+- example-repo/cfer-helpers/.keep
+- example-repo/config/.keep
+- example-repo/config/schema.yaml
+- example-repo/config/us-west-2/dev-ed1.yaml
+- example-repo/config/validator.rb
+- example-repo/steps/.keep
+- example-repo/steps/00.bootstrap/.keep
+- example-repo/steps/00.bootstrap/cfer/defs/s3.rb
+- example-repo/steps/00.bootstrap/cfer/outputs.rb
+- example-repo/steps/00.bootstrap/cfer/parameters.rb
+- example-repo/steps/00.bootstrap/cfer/require.rb
+- example-repo/steps/00.bootstrap/on-create.d/00-debug.rb
+- example-repo/steps/00.bootstrap/on-destroy.d/00-debug.rb
+- example-repo/steps/00.bootstrap/post-converge.d/00-debug.rb
+- example-repo/steps/00.bootstrap/pre-converge.d/00-debug.rb
+- example-repo/steps/01.dependent/.keep
+- example-repo/steps/01.dependent/cfer/defs/s3.rb
+- example-repo/steps/01.dependent/cfer/outputs.rb
+- example-repo/steps/01.dependent/cfer/parameters.rb
+- example-repo/steps/01.dependent/cfer/require.rb
+- example-repo/steps/01.dependent/on-create.d/00-debug.rb
+- example-repo/steps/01.dependent/on-destroy.d/00-debug.rb
+- example-repo/steps/01.dependent/post-converge.d/00-debug.rb
+- example-repo/steps/01.dependent/pre-converge.d/00-debug.rb
+- lib/cfer/auster.rb
+- lib/cfer/auster/cfer_evaluator.rb
+- lib/cfer/auster/cfer_helpers.rb
+- lib/cfer/auster/cli.rb
+- lib/cfer/auster/cli/_shared.rb
+- lib/cfer/auster/cli/destroy.rb
+- lib/cfer/auster/cli/generate.rb
+- lib/cfer/auster/cli/generate/repo.rb
+- lib/cfer/auster/cli/generate/step.rb
+- lib/cfer/auster/cli/json.rb
+- lib/cfer/auster/cli/nuke.rb
+- lib/cfer/auster/cli/run.rb
+- lib/cfer/auster/config.rb
+- lib/cfer/auster/logging.rb
+- lib/cfer/auster/param_validator.rb
+- lib/cfer/auster/repo.rb
+- lib/cfer/auster/script_executor.rb
+- lib/cfer/auster/step.rb
+- lib/cfer/auster/version.rb
+homepage: https://github.com/eropple/auster
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Best-practices tooling for CloudFormation and AWS.
+test_files: []