simplygenius-atmos 0.11.9 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 685baebb62604a3094114ff1616e1e044c0560ed9285c27686a4b8c3e635f964
-  data.tar.gz: 2eadc8adc7ca8574de3d9f44d1af4d7668c17de8d9b6bee7f38c5f3234f47f63
+  metadata.gz: e57a5d360cd9c6016f83847333f3bf1519e70a8b73b0203c969d79fdf5a5e3f0
+  data.tar.gz: 4307c006f083fb94fdf5155df08a18a43d2ad6493070d279502b9a2a87a73c34
 SHA512:
-  metadata.gz: 96249401ac017e427d0a5131f8ee441d51da30b91ae5da13a42bbddc258b1f7402c30411d5d304f2680c79e88d6e62eed8bfa26f0f70bd1b85eaf5ced69fd985
-  data.tar.gz: 2025652bff07e78886a228f75e87183eb8b70ec0ec294223008f6abc4203ad9bf3cd617a39c06a521320812454c61e28d2a8c541eb9d5beb7c35b86bad4a3e3f
+  metadata.gz: d084865f58d1dfe8081debef0ce69c7fd2ab1068d813d8d704a4dc40962337c883be4763a2b161d5b8df4d0f6edd34a28209f95abb780b964d3abeb6dd7a94ab
+  data.tar.gz: 9f5f91afd5545dc14a073c4dfea67b86b699ed4add68077896cb151c990d50d32089246fe03b08b372eb98bb2694e98166933ac072fd22578e654a48771bc5b2

data/CHANGELOG.md

@@ -1,3 +1,63 @@
+0.13.0 (01/20/2021)
+-------------------
+
+#### Notes on major changes
+
+* Now testing the atmos runtime against terraform 0.14 (and 0.11, 0.13)
+* The ability to link files into the terraform execution directory was enhanced, but still doesn't work for the .terraform.lock.hcl as it is overwritten by terraform and doesn't follow the link  
+
+#### Full changelog
+
+* exclude some new taskdef attributes for creation of new taskdef during deploy [a01b997](https://github.com/simplygenius/atmos/commit/a01b997)
+* add debug logging for aws sdk logger [55dd303](https://github.com/simplygenius/atmos/commit/55dd303)
+* directory sorting made more consistent across platforms [13622ff](https://github.com/simplygenius/atmos/commit/13622ff)
+* switch ci/cd from travis to github actions [13622ff](https://github.com/simplygenius/atmos/commit/13622ff)
+* fix test for tf 0.14 [122f861](https://github.com/simplygenius/atmos/commit/122f861)
+* replace plugin sharing with built in terraform variant [eb75266](https://github.com/simplygenius/atmos/commit/eb75266)
+* allow more options for linking in non-tf files into working dir [9878336](https://github.com/simplygenius/atmos/commit/9878336)
+* sort secret list [6b2dbfe](https://github.com/simplygenius/atmos/commit/6b2dbfe)
+* Only run terraform init when needed [d77186b](https://github.com/simplygenius/atmos/commit/d77186b)
+
+
+0.12.1 (10/14/2020)
+-------------------
+
+* fix bug with deeply nested overrides [60cb84a](https://github.com/simplygenius/atmos/commit/60cb84a)
+
+0.12.0 (10/12/2020)
+-------------------
+
+#### Notes on major changes
+
+* Now testing the atmos runtime against terraform 0.13 (and 0.11, 0.12), but no changes were necessary, so it should work if you upgrade aside from recipe changes
+* The fix to to make overrides (keys like ^foo to replace foo rather than add to it) work no matter where they appear in the config load chain could break compatibility if you were depending on the old behavior, i.e. you have multiple overrides for the same key with only the last one winning, or you have an override earlier in the change that was previously not taking effect
+
+#### Full changelog
+
+* Make overrides work no matter where they appear in the config load chain [47d73ba](https://github.com/simplygenius/atmos/commit/47d73ba)
+* allow interpolation in (remote_)config_source references, e.g. for supplying an access key from ~/.atmos.yml in the url [43f0c4d](https://github.com/simplygenius/atmos/commit/43f0c4d)
+* fix docker build [f4b3e2e](https://github.com/simplygenius/atmos/commit/f4b3e2e)
+* fix stdin test [5bd3f95](https://github.com/simplygenius/atmos/commit/5bd3f95)
+* exit with code on terraform error [2d2af2a](https://github.com/simplygenius/atmos/commit/2d2af2a)
+* test against tf 0.1[123] [1bbfb3b](https://github.com/simplygenius/atmos/commit/1bbfb3b)
+* add 'none' provider for tests (or if one wants to only run against terraform without aws) [991cefe](https://github.com/simplygenius/atmos/commit/991cefe)
+* fix activesupport cve [3d8b30c](https://github.com/simplygenius/atmos/commit/3d8b30c)
+* Add the ability to fetch atmos config from remote sources [b4b97fb](https://github.com/simplygenius/atmos/commit/b4b97fb)
+* Add convenience (`atmos account setup_credentials`)for populating aws local credential stores with the aws accounts managed by atmos [05466bd](https://github.com/simplygenius/atmos/commit/05466bd)
+* Add error msg for when task definition does't exist [0f8291e](https://github.com/simplygenius/atmos/commit/0f8291e)
+* Fix `jsonify` to convert the correct value to JSON. (#4) [1bda229](https://github.com/simplygenius/atmos/commit/1bda229)
+* Use ruby 2.4 for legacy testing since 2.3 is eol [78d2862](https://github.com/simplygenius/atmos/commit/78d2862)
+
+0.11.11 (02/20/2020)
+--------------------
+
+* Add the ability to setup a version requirement for forcing teams to use a consistent version of atmos [5c44ef3](https://github.com/simplygenius/atmos/commit/5c44ef3)
+
+0.11.10 (02/19/2020)
+--------------------
+
+* allow overriding atmos config from cli [f83902b](https://github.com/simplygenius/atmos/commit/f83902b)
+
 0.11.9 (02/13/2020)
 -------------------
 

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/simplygenius/atmos.svg?branch=master)](https://travis-ci.org/simplygenius/atmos)
+[![Build Status](https://github.com/simplygenius/atmos/workflows/CD/badge.svg)](https://github.com/simplygenius/atmos/actions)
 [![Coverage Status](https://coveralls.io/repos/github/simplygenius/atmos/badge.svg?branch=master)](https://coveralls.io/github/simplygenius/atmos?branch=master)
 [![Gitter](https://badges.gitter.im/simplygenius/atmos.svg)](https://gitter.im/simplygenius/atmos?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 

data/lib/simplygenius/atmos/cli.rb

@@ -2,6 +2,7 @@ require_relative '../atmos'
 require_relative '../atmos/ui'
 require 'clamp'
 require 'sigdump/setup'
+require 'open-uri'
 
 Dir.glob(File.join(File.join(__dir__, 'commands'), '*.rb')) do |f|
   require_relative "commands/#{File.basename(f).sub(/\.rb$/, "")}"
@@ -51,6 +52,10 @@ module SimplyGenius
              "PATH", "adds additional paths to ruby load path",
              multivalued: true
 
+      option ["-o", "--override"],
+             "KEYVALUE", "overrides atmos configuration\nin the form 'some.config=value' where value can\nbe expressed in yaml form for complex types\ne.g. foo=1 foo=abc, foo=[x, y], foo={x: y}",
+             multivalued: true
+
       option ["-v", "--version"],
              :flag, "Shows the atmos version"
 
@@ -115,6 +120,41 @@ module SimplyGenius
         end
       end
 
+      def fetch_latest_version
+        begin
+          latest_ver = JSON.parse(URI.open("https://rubygems.org/api/v1/versions/simplygenius-atmos/latest.json").read)['version']
+        rescue => e
+          latest_ver = "[Version Fetch Failed]"
+          logger.log_exception(e, "Couldn't check latest atmos gem version", level: :debug)
+        end
+        latest_ver
+      end
+
+      def version_check(atmos_version)
+
+        required_ver = Atmos.config["atmos.version_requirement"]
+        if required_ver.present?
+          case required_ver
+
+          when "latest"
+            latest_ver = fetch_latest_version
+
+            if latest_ver != atmos_version
+              raise "The atmos version (#{atmos_version}) does not match the given requirement (latest: #{latest_ver})"
+            end
+
+          when /[~<>=]*\s*[\d\.]*/
+            if ! Gem::Dependency.new('', required_ver).match?('', atmos_version)
+              raise "The atmos version (#{atmos_version}) does not match the given requirement (#{required_ver})"
+            end
+
+          else
+            raise "Invalid atmos.version_requirement, should be 'latest' or in a gem dependency form"
+          end
+        end
+
+      end
+
       # hook into clamp lifecycle to force logging setup even when we are calling
       # a subcommand
       def parse(arguments)
@@ -128,6 +168,13 @@ module SimplyGenius
 
           Logging.setup_logging(level, color?, log)
 
+          override_list.each do |o|
+            k, v = o.split("=")
+            v = YAML.load(v)
+            logger.debug("Overriding config '#{k}' = #{v.inspect}")
+            Atmos.config.[]=(k, v, additive: false)
+          end
+
           UI.color_enabled = color?
 
           Atmos.config.add_user_load_path(*load_path_list)
@@ -138,6 +185,8 @@ module SimplyGenius
             logger.info "Atmos Version #{VERSION}"
             exit(0)
           end
+
+          version_check(VERSION)
         end
       end
 

data/lib/simplygenius/atmos/commands/account.rb

@@ -62,6 +62,42 @@ module SimplyGenius
           end
         end
 
+        subcommand "setup_credentials", "Convenience that adds accounts to the local aws\ncredentials store" do
+
+          option ["-u", "--user"],
+                 "USERNAME", "The username in the cloud provider\n", required: true
+
+          option ["-k", "--key"],
+                 "KEY", "The access key in the cloud provider\n"
+
+          option ["-s", "--secret"],
+                 "SECRET", "The access secret in the cloud provider\n"
+
+          option ["-d", "--default"],
+                 :flag, "Sets as default credentials\n"
+
+          option ["-f", "--force"],
+                 :flag, "Forces overwrites of existing\n"
+
+          option ["-n", "--nowrite"],
+                 :flag, "Trial run without writing results to files\n"
+
+          def execute
+
+            if ! key.present?
+              key = ask("Input your access key: ") { |q| q.echo = "*" }
+              raise ArgumentError.new("An access key is required") if key.blank?
+            end
+            if ! secret.present?
+              secret = ask("Input your access secret: ") { |q| q.echo = "*" }
+              raise ArgumentError.new("An access secret is required") if secret.blank?
+            end
+
+            Atmos.config.provider.account_manager.setup_credentials(username: user, access_key: key, access_secret: secret,
+                                                                    become_default: default?, force: force?, nowrite: nowrite?)
+          end
+        end
+
       end
 
     end

data/lib/simplygenius/atmos/commands/bootstrap.rb

@@ -68,6 +68,7 @@ module SimplyGenius
             rescue TerraformExecutor::ProcessFailed => e
               logger.error(e.message)
               logger.error(rebootstrap_msg)
+              exit(1)
             end
           end
         end

data/lib/simplygenius/atmos/commands/init.rb

@@ -15,8 +15,6 @@ module SimplyGenius
         def execute
           @terraform_arguments.insert(0, "init")
           super
-
-          init_shared_plugins
         end
 
       end

data/lib/simplygenius/atmos/commands/secret.rb

@@ -60,7 +60,7 @@ module SimplyGenius
             Atmos.config.provider.auth_manager.authenticate(ENV) do |auth_env|
               ClimateControl.modify(auth_env) do
                 logger.info "Secret keys are:"
-                Atmos.config.provider.secret_manager.to_h.keys.each {|k| logger.info k}
+                Atmos.config.provider.secret_manager.to_h.keys.sort.each {|k| logger.info k}
               end
             end
 

data/lib/simplygenius/atmos/commands/terraform.rb

@@ -21,27 +21,23 @@ module SimplyGenius
         end
 
         def init_automatically(auth_env, get_modules)
-          tf_init_dir = File.join(Atmos.config.tf_working_dir, '.terraform')
+          tf_init_dir = File.join(Atmos.config.tf_working_dir, 'recipes', '.terraform')
           backend_initialized = File.exist?(File.join(tf_init_dir, 'terraform.tfstate'))
           auto_init_enabled = Atmos.config["atmos.terraform.auto_init"].to_s == "true"
 
           if auto_init && auto_init_enabled && ! backend_initialized
             exe = TerraformExecutor.new(process_env: auth_env)
             exe.run("init", get_modules: get_modules.present?)
-            init_shared_plugins
           end
         end
 
-        def init_shared_plugins
+        def enable_shared_plugins(env)
           if ! Atmos.config["atmos.terraform.disable_shared_plugins"]
             home_dir = OS.windows? ? File.join("~", "Application Data") : "~"
-            shared_plugins_dir = File.expand_path(File.join(home_dir,".terraform.d", "plugins"))
-            logger.debug("Updating shared terraform plugins dir: #{shared_plugins_dir}")
-            mkdir_p(shared_plugins_dir)
-            terraform_plugins_dir = File.join(Atmos.config.tf_working_dir,'recipes', '.terraform', 'plugins')
-            if File.exist?(terraform_plugins_dir)
-              cp_r("#{terraform_plugins_dir}/.", shared_plugins_dir)
-            end
+            plugin_cache_dir = File.expand_path(File.join(home_dir,".terraform.d", "plugin-cache"))
+            logger.debug("Plugin cache dir: #{plugin_cache_dir}")
+            mkdir_p(plugin_cache_dir)
+            env["TF_PLUGIN_CACHE_DIR"] = plugin_cache_dir
           end
         end
 
@@ -58,12 +54,15 @@ module SimplyGenius
             begin
               get_modules = @terraform_arguments.delete("--get-modules")
 
+              enable_shared_plugins(auth_env)
+
               init_automatically(auth_env, get_modules)
 
               exe = TerraformExecutor.new(process_env: auth_env)
               exe.run(*@terraform_arguments, get_modules: get_modules.present?)
             rescue TerraformExecutor::ProcessFailed => e
               logger.error(e.message)
+              exit(1)
             end
           end
         end

data/lib/simplygenius/atmos/commands/tfutil.rb

@@ -61,7 +61,7 @@ module SimplyGenius
                 result[k] = ev
               end
             else
-              result["data"] = JSON.generate(result)
+              result["data"] = JSON.generate(obj)
             end
 
             return result
@@ -73,9 +73,9 @@ module SimplyGenius
               chunk = $stdin.read_nonblock(1)
               data = chunk + $stdin.read
               logger.debug("Received stdin: " + data)
-            rescue Errno::EAGAIN
+            rescue EOFError, SystemCallError => e # All Errno exceptions
               data = nil
-              logger.debug("No stdin")
+              logger.debug("No stdin due to: #{e}")
             end
             return data
           end

data/lib/simplygenius/atmos/config.rb

@@ -5,6 +5,7 @@ require_relative '../atmos/plugin_manager'
 require 'yaml'
 require 'fileutils'
 require 'find'
+require 'open-uri'
 
 module SimplyGenius
   module Atmos
@@ -26,7 +27,7 @@ module SimplyGenius
         @config_file = config ? File.expand_path(config, root_dir) : File.join(root_dir, "config", "atmos.yml")
         @user_config_file = "~/.atmos.yml"
         @tmp_root = File.join(root_dir, "tmp")
-        @included_configs = []
+        @included_configs = {}
       end
 
       def is_atmos_repo?
@@ -39,9 +40,9 @@ module SimplyGenius
         return result
       end
 
-      def []=(key, value)
+      def []=(key, value, additive: true)
         load
-        result = @config.notation_put(key, value)
+        result = @config.notation_put(key, value, additive: additive)
         return result
       end
 
@@ -114,6 +115,7 @@ module SimplyGenius
         if merge_to_existing
           existing = load_file(user_config_file, SettingsHash.new)
           data = config_merge(existing, data, ["saving #{user_config_file}"])
+          data = finalize_merge(data)
         end
         File.write(user_config_file, YAML.dump(data.to_hash))
         File.chmod(0600, user_config_file)
@@ -132,21 +134,27 @@ module SimplyGenius
           when Hash
             result = lhs.deep_dup
 
-            lhs.each do |k, v|
-              if k =~ /^\^(.*)/
-                key = k.is_a?(Symbol) ? $1.to_sym : $1
-                result[key] = result.delete(k)
-              end
-            end
-
             rhs.each do |k, v|
-              if k =~ /^\^(.*)/
-                key = k.is_a?(Symbol) ? $1.to_sym : $1
-                result[key] = v
-              else
-                result[k] = config_merge(result[k], v, debug_state + [k])
+              new_state = debug_state + [k]
+
+              # The load sequence could have multiple overrides and its not
+              # obvious to the user which are lhs vs rhs, so rather than having
+              # one seem to win arbitrarily, we collect them all additively
+              # under their key, then at the end of the load process we push the
+              # override back as a replacement for its base key in the
+              # finalize_merge method. This means that if one has multiple
+              # overrides for the same key (not usually what one wants), those
+              # overrides get merged together additively before replacing the
+              # base.  Thus the need for the log messages below.
+              #
+              if k =~ /^\^/
+                logger.debug { "Override seen at #{new_state.join(" -> ")}" }
+                logger.warn { "Multiple overrides on a single key seen at #{new_state.join(" -> ")}" } if result.has_key?(k)
               end
+
+              result[k] = config_merge(result[k], v, new_state)
             end
+
           when Enumerable
             result = lhs + rhs
           else
@@ -168,6 +176,37 @@ module SimplyGenius
 
       private
 
+      def finalize_merge(config)
+        result = config.deep_dup
+
+        config.each do |k, v|
+
+          key = k
+
+          if k =~ /^\^(.*)/
+            key = k.is_a?(Symbol) ? $1.to_sym : $1
+            result[key] = result.delete(k)
+          end
+
+          if v.is_a?(Hash)
+            result[key] = finalize_merge(v)
+          end
+
+        end
+
+        return result
+      end
+
+      def load_remote_config_sources(config, *remote_sources)
+        remote_sources.each do |remote_source|
+          logger.debug("Loading remote atmos config file: #{remote_source}")
+          contents = URI.open(remote_source).read
+          config = load_config(remote_source, contents, config)
+        end
+
+        config
+      end
+
       def load_config_sources(relative_root, config, *patterns)
         patterns.each do |pattern|
           logger.debug("Loading atmos config files using pattern: #{pattern}")
@@ -180,7 +219,7 @@ module SimplyGenius
           pattern = File.expand_path(pattern)
           logger.debug("Expanded pattern: #{pattern}")
 
-          Dir[pattern].each do |f|
+          Dir[pattern].sort.each do |f|
             config = load_file(f, config)
           end
         end
@@ -217,26 +256,26 @@ module SimplyGenius
             @full_config = load_file(config_file, SettingsHash.new)
           end
 
-          @full_config = load_config_sources(File.dirname(config_file), @full_config, *Array(@full_config.notation_get("atmos.config_sources")))
+          @user_config_file = @full_config.notation_get("atmos.user_config") || @user_config_file
+          @user_config_file = File.expand_path(@user_config_file)
+          user_config_file_data = load_file(@user_config_file)
+          temp_settings = create_settings(config_merge(@full_config, user_config_file_data, [@user_config_file]), finalize: true)
+
+          @full_config = load_config_sources(File.dirname(config_file), @full_config, *Array(temp_settings.notation_get("atmos.config_sources")))
+          @full_config = load_remote_config_sources(@full_config, *Array(temp_settings.notation_get("atmos.remote_config_sources")))
 
           @full_config['provider'] = provider_name = @full_config['provider'] || 'aws'
 
           @full_config = load_submap(File.dirname(config_file), 'providers', provider_name, @full_config)
           @full_config = load_submap(File.dirname(config_file), 'environments', atmos_env, @full_config)
 
-          @user_config_file = @full_config.notation_get("atmos.user_config") || @user_config_file
-          @user_config_file = File.expand_path(@user_config_file)
-          @full_config = load_file(user_config_file, @full_config)
+          @full_config = config_merge(@full_config, user_config_file_data, [@user_config_file])
 
-          global = SettingsHash.new(@full_config.reject {|k, v| ['providers', 'environments'].include?(k) })
-          conf = config_merge(global, {
-              atmos_env: atmos_env,
-              atmos_working_group: working_group,
-              atmos_version: VERSION
-          }, ["builtins"])
+          conf = create_settings(@full_config, finalize: true)
+
+          # hash emptied out to allow GC of all loaded file contents
+          @included_configs = {}
 
-          conf.error_resolver = ->(statement) { find_config_error(statement) }
-          conf.enable_expansion = true
           conf
 
         end
@@ -245,15 +284,8 @@ module SimplyGenius
       def load_file(file, config=SettingsHash.new, &block)
         if File.exist?(file)
           logger.debug("Loading atmos config file #{file}")
-          data = YAML.load_file(file)
-          if ! data.is_a?(Hash)
-            logger.debug("Skipping invalid atmos config file (not hash-like): #{file}")
-          else
-            data = SettingsHash.new(data)
-            data = block.call(data) if block
-            config = config_merge(config, data, [file])
-            @included_configs << file
-          end
+          contents = File.read(file)
+          config = load_config(file, contents, config, &block)
         else
           logger.debug   "Could not find an atmos config file at: #{file}"
         end
@@ -261,16 +293,49 @@ module SimplyGenius
         config
       end
 
+      def load_config(location, yml_string, config=SettingsHash.new, &block)
+        data = YAML.load(yml_string)
+
+        if ! data.is_a?(Hash)
+          logger.debug("Skipping invalid atmos config (not hash-like): #{location}")
+        else
+          data = SettingsHash.new(data)
+          data = block.call(data) if block
+          # if lhs has a override ^, then it loses it when rhs gets merged in, which breaks things for subsequent merges
+          config = config_merge(config, data, [location])
+          @included_configs[location] = yml_string
+        end
+
+        config
+      end
+
+      def create_settings(config, finalize: true)
+        builtins = {
+            atmos_env: atmos_env,
+            atmos_working_group: working_group,
+            atmos_version: VERSION
+        }
+
+        global = SettingsHash.new(config.reject {|k, v| ['providers', 'environments'].include?(k) })
+        conf = config_merge(global, builtins, ["builtins"])
+        conf = finalize_merge(conf) if finalize
+
+        conf.error_resolver = ->(statement) { find_config_error(statement) }
+        conf.enable_expansion = true
+
+        conf
+      end
+
       def find_config_error(statement)
         filename = nil
         line = 0
 
-        @included_configs.each do |c|
+        @included_configs.each do |location, contents|
           current_line = 0
-          File.foreach(c) do |f|
+          contents.each_line do |line|
             current_line += 1
-            if f.include?(statement)
-              filename = c
+            if line.include?(statement)
+              filename = location
               line = current_line
               break
             end

data/lib/simplygenius/atmos/provider_factory.rb

@@ -7,14 +7,15 @@ module SimplyGenius
       include GemLogger::LoggerSupport
 
       def self.get(name)
-        @provider ||= begin
+        @providers ||= {}
+        provider = @providers[name] ||= begin
           logger.debug("Loading provider: #{name}")
           require "simplygenius/atmos/providers/#{name}/provider"
-          provider = "SimplyGenius::Atmos::Providers::#{name.camelize}::Provider".constantize
-          logger.debug("Loaded provider #{provider}")
-          provider.new(name)
+          provider_class = "SimplyGenius::Atmos::Providers::#{name.camelize}::Provider".constantize
+          logger.debug("Loaded provider #{provider_class}")
+          provider_class.new(name)
         end
-        return @provider
+        return provider
       end
 
     end

data/lib/simplygenius/atmos/providers/aws/account_manager.rb

@@ -1,5 +1,6 @@
 require_relative '../../../atmos'
 require 'aws-sdk-organizations'
+require 'inifile'
 
 module SimplyGenius
   module Atmos
@@ -75,6 +76,76 @@ module SimplyGenius
             return result
           end
 
+          def setup_credentials(username:, access_key:, access_secret:, become_default: false, force: false, nowrite: false)
+            creds = File.expand_path("~/.aws/credentials")
+            config = File.expand_path("~/.aws/config")
+
+            creds_data = IniFile.load(creds) || IniFile.new
+            config_data = IniFile.load(config) || IniFile.new
+
+            org = Atmos.config["org"]
+            accounts = Atmos.config.account_hash
+
+            # Our entrypoint account (the ops env/account) is the account we
+            # have credentials for, and typically one authenticate using the
+            # credentials for that account, then assume role (<env>-admin) to
+            # actually operate in each env, even for the ops account
+            ops_config = Atmos::Config.new("ops")
+            entrypoint_section = become_default ? "default" : org
+            entrypoint_config_section = become_default ? "default" : "profile #{org}"
+            write_entrypoint = true
+            if config_data.has_section?(entrypoint_config_section)  || creds_data.has_section?(entrypoint_section)
+              if force
+                logger.info "Overwriting pre-existing sections: '#{entrypoint_config_section}'/'#{entrypoint_section}'"
+              else
+                logger.info "Skipping pre-existing sections (use force to overwrite): '#{entrypoint_config_section}'/'#{entrypoint_section}'"
+                write_entrypoint = false
+              end
+            end
+            if write_entrypoint
+              config_data[entrypoint_config_section]["region"] = ops_config["region"]
+              creds_data[entrypoint_section]["aws_access_key_id"] = access_key
+              creds_data[entrypoint_section]["aws_secret_access_key"] = access_secret
+              creds_data[entrypoint_section]["mfa_serial"] = "arn:aws:iam::#{accounts["ops"]}:mfa/#{username}"
+            end
+
+            accounts.each do |env, account_id|
+              env_config = Atmos::Config.new(env)
+
+              section = "#{org}-#{env}"
+              config_section = "profile #{section}"
+
+              if config_data.has_section?(config_section)  || creds_data.has_section?(section)
+                if force
+                  logger.info "Overwriting pre-existing sections: '#{config_section}'/'#{section}'"
+                else
+                  logger.info "Skipping pre-existing sections (use force to overwrite): '#{config_section}'/'#{section}'"
+                  next
+                end
+              end
+
+              config_data[config_section]["source_profile"] = entrypoint_section
+              role_name = env_config["auth.assume_role_name"]
+              config_data[config_section]["role_arn"] = "arn:aws:iam::#{account_id}:role/#{role_name}"
+            end
+
+            if nowrite
+              logger.info "Trial run only, would write the following:\n"
+              puts "*** #{config}:\n\n"
+              puts config_data.to_s
+
+              puts "\n\n*** #{creds}:\n\n"
+              puts creds_data.to_s
+            else
+              logger.info "Writing credentials to disk"
+              mkdir_p(File.dirname(config))
+              mv config, "#{config}.bak" if File.exist?(config)
+              mv creds, "#{creds}.bak" if File.exist?(creds)
+              config_data.write(filename: config)
+              creds_data.write(filename: creds)
+            end
+
+          end
         end
 
       end

data/lib/simplygenius/atmos/providers/aws/container_manager.rb

@@ -64,7 +64,8 @@ module SimplyGenius
 
             new_defn = latest_defn.to_h
             [:revision, :status, :task_definition_arn,
-             :requires_attributes, :compatibilities].each do |attr|
+             :requires_attributes, :compatibilities,
+             :registered_at, :registered_by].each do |attr|
               new_defn.delete(attr)
             end
             new_defn[:container_definitions].each {|c| c[:image] = remote_image}
@@ -198,6 +199,8 @@ module SimplyGenius
               logger.info "Running task as '#{task_opts[:launch_type]}'"
             end
 
+            raise "Could not find a task definition in AWS for '#{name}'" if defn_arn.blank?
+
             resp = ecs.describe_task_definition(task_definition: defn_arn)
             defn = resp.task_definition
             raise "Invalid Launch type '#{launch_type}'" unless (defn.requires_compatibilities + defn.compatibilities).include?(launch_type)

data/lib/simplygenius/atmos/providers/aws/provider.rb

@@ -1,5 +1,7 @@
 require_relative '../../../atmos'
 
+require 'aws-sdk-core'
+
 Dir.glob(File.join(__dir__, '*.rb')) do |f|
   require_relative "#{File.basename(f).sub(/\.rb$/, "")}"
 end
@@ -11,6 +13,7 @@ module SimplyGenius
 
         class Provider
           include GemLogger::LoggerSupport
+          ::Aws.config.update(logger: logger, log_level: :debug)
 
           def initialize(name)
             @name = name

data/lib/simplygenius/atmos/providers/none/auth_manager.rb

@@ -0,0 +1,24 @@
+require_relative '../../../atmos'
+
+module SimplyGenius
+  module Atmos
+    module Providers
+      module None
+
+        class AuthManager
+          include GemLogger::LoggerSupport
+
+          def initialize(provider)
+            @provider = provider
+          end
+
+          def authenticate(system_env, **opts, &block)
+            logger.debug("Calling none authentication target")
+            block.call(system_env)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/simplygenius/atmos/providers/none/provider.rb

@@ -0,0 +1,47 @@
+require_relative '../../../atmos'
+
+Dir.glob(File.join(__dir__, '*.rb')) do |f|
+  require_relative "#{File.basename(f).sub(/\.rb$/, "")}"
+end
+
+module SimplyGenius
+  module Atmos
+    module Providers
+      module None
+
+        class Provider
+          include GemLogger::LoggerSupport
+
+          def initialize(name)
+            @name = name
+          end
+
+          def auth_manager
+            @auth_manager ||= begin
+              AuthManager.new(self)
+            end
+          end
+
+          def user_manager
+            raise NotImplementedError.new("No user manager in none provider")
+          end
+
+          def account_manager
+            raise NotImplementedError.new("No account manager in none provider")
+          end
+
+          def secret_manager
+            @secret_manager ||= begin
+              SecretManager.new(self)
+            end
+          end
+
+          def container_manager
+            raise NotImplementedError.new("No container manager in none provider")
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/simplygenius/atmos/providers/none/secret_manager.rb

@@ -0,0 +1,40 @@
+require_relative '../../../atmos'
+
+module SimplyGenius
+  module Atmos
+    module Providers
+      module None
+
+        class SecretManager
+          include GemLogger::LoggerSupport
+
+          def initialize(provider)
+            @provider = provider
+            @secrets = {}
+          end
+
+          def set(key, value, force: false)
+            if @secrets.has_key?(key) && ! force
+              raise "A value already exists for the given key, force overwrite or delete first"
+            end
+            @secrets[key] = value
+          end
+
+          def get(key)
+            @secrets[key]
+          end
+
+          def delete(key)
+            @secrets.delete(key)
+          end
+
+          def to_h
+            @secrets.to_h
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/simplygenius/atmos/source_path.rb

@@ -113,7 +113,7 @@ module SimplyGenius
             begin
               logger.debug("Cloning zip archive to tmpdir")
 
-              open(sourcepath, 'rb') do |io|
+              URI.open(sourcepath, 'rb') do |io|
                 Zip::File.open_buffer(io) do |zip_file|
                   zip_file.each do |f|
                     fpath = File.join(tmpdir, f.name)

data/lib/simplygenius/atmos/terraform_executor.rb

@@ -308,7 +308,7 @@ module SimplyGenius
 
       def clean_links
         Find.find(Atmos.config.tf_working_dir) do |f|
-          Find.prune if f =~ /\/.terraform\/modules\//
+          Find.prune if f =~ /\/.terraform\//
           File.delete(f) if File.symlink?(f)
         end
       end
@@ -318,13 +318,30 @@ module SimplyGenius
         working_dir_links ||= ['modules', 'templates']
         working_dir_links.each do |subdir|
           source = File.join(Atmos.config.root_dir, subdir)
-          ln_sf(source, Atmos.config.tf_working_dir) if File.exist?(source)
+          parts = File.split(subdir)
+          target_dir = Atmos.config.tf_working_dir
+          if parts[0] != "."
+            target_dir = File.join(Atmos.config.tf_working_dir, parts[0])
+            mkdir_p(target_dir)
+          end
+          ln_sf(source, target_dir) if File.exist?(source)
         end
       end
 
       def link_recipes
         @recipes.each do |recipe|
-          ln_sf(File.join(Atmos.config.root_dir, 'recipes', "#{recipe}.tf"), tf_recipes_dir)
+          recipe_satisfied = false
+          ["#{recipe}.tf", recipe].each do |recipe_variant|
+            fqrecipe = File.join(Atmos.config.root_dir, 'recipes', recipe_variant)
+            if File.exist?(fqrecipe)
+              ln_sf(fqrecipe, tf_recipes_dir)
+              recipe_satisfied = true
+              break
+            else
+              logger.debug("Recipe target does not exist: #{fqrecipe}")
+            end
+          end
+          logger.error("Recipe '#{recipe}' is not present") unless recipe_satisfied
         end
       end
 

data/lib/simplygenius/atmos/version.rb

@@ -1,5 +1,5 @@
 module SimplyGenius
   module Atmos
-    VERSION = "0.11.9"
+    VERSION = "0.13.0"
   end
 end

data/templates/new/config/atmos.yml

@@ -67,4 +67,9 @@ atmos:
   # to the directory containing atmos.yml
   #
   config_sources:
-  - atmos/*.y{,a}ml
+    - atmos/*.y{,a}ml
+  # A list of remote atmos configuration yml files.
+  # Takes any URI supported by ruby's open-uri
+  #
+  remote_config_sources:
+    # - https://some.url/to/config.yml

data/templates/new/config/atmos/runtime.yml

@@ -35,6 +35,12 @@ atmos:
   # gets expanded at runtime
   load_path:
 
+  # Forces one to use the version of atmos that is given.  This version
+  # specifier can be 'latest' or in the form of a gem version dependency
+  # specifier, e.g. '~> 0.11.10'.  If unset, then no version check will be
+  # performed
+  version_requirement:
+
   # Configure the mechanism that allows terraform to callback into atmos
   ipc:
     # Disables all IPC callbacks into atmos from terraform
@@ -113,12 +119,12 @@ atmos:
   terraform:
     # Disable module fetch from convenience plan/apply commands
     disable_auto_modules: false
-    # By default (value=false), `atmos init` will update the terraform user
-    # plugin directory (~/.terraform.d/plugins) with the plugins for the current
-    # env/group so that they can be reused across all env/group combinations.
-    # Otherwise, disabling this functionality (value=true) means that each
-    # env/group combination will be independent and download all plugins for
-    # itself only
+    # By default (value=false), `atmos init` (and auto init) will set
+    # TF_PLUGIN_CACHE_DIR to ~/.terraform.d/plugin-cache so that when terraform
+    # installs providers/plugins they can be reused across all env/group
+    # combinations. Otherwise, disabling this functionality (value=true) means
+    # that each env/group combination will be independent and download all
+    # plugins for itself only
     disable_shared_plugins: false
     # Customize what gets linked into the working directory that terraform gets
     # executed in

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simplygenius-atmos
 version: !ruby/object:Gem::Version
-  version: 0.11.9
+  version: 0.13.0
 platform: ruby
 authors:
 - Matt Conway
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-02-13 00:00:00.000000000 Z
+date: 2021-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,142 +28,156 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.8.23
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.8.23
 - !ruby/object:Gem::Dependency
   name: test_construct
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.11.3
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.11.3
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.1
+        version: 5.2.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.1
+        version: 5.2.4.3
 - !ruby/object:Gem::Dependency
   name: gem_logger
   requirement: !ruby/object:Gem::Requirement
@@ -472,6 +486,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: inifile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: deepsort
   requirement: !ruby/object:Gem::Requirement
@@ -558,6 +586,9 @@ files:
 - lib/simplygenius/atmos/providers/aws/s3_secret_manager.rb
 - lib/simplygenius/atmos/providers/aws/ssm_secret_manager.rb
 - lib/simplygenius/atmos/providers/aws/user_manager.rb
+- lib/simplygenius/atmos/providers/none/auth_manager.rb
+- lib/simplygenius/atmos/providers/none/provider.rb
+- lib/simplygenius/atmos/providers/none/secret_manager.rb
 - lib/simplygenius/atmos/settings_hash.rb
 - lib/simplygenius/atmos/source_path.rb
 - lib/simplygenius/atmos/template.rb
@@ -574,7 +605,7 @@ homepage: https://github.com/simplygenius/atmos
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -589,8 +620,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Atmos provides a terraform scaffold for creating cloud system architectures
 test_files: []