simplygenius-atmos 0.12.0 → 0.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a43cd732a3f97e0fc2884f1b284b8f389db9b1845879a1e8cd5b23a657b9dab6
-  data.tar.gz: 0b84e5d5026b849bac355875bc4759b8cfc21abc4ad51627d2e35626f928f2e4
+  metadata.gz: b3da6d623f0f47dd62493eaf985c527b153698844d26027c2838660f4fdfe93f
+  data.tar.gz: 8f7deb9ff4397607aad846869898c9fd28df2c484b588175e9fb19bb614860d4
 SHA512:
-  metadata.gz: 84bfe472a2bef731c2cd3dc715ca2b0d214a343cef6491f09bc4762445d7faf340b704366a5b0e099c3e9a42cb6f4e23fecf918671cdeb6a51b4a1a7e15a993e
-  data.tar.gz: 2a3f7167cd23068e9e8ea810810ed3e896b36d6c803b8f844b38bfa11e8127dd37505fcf91995321d9a53af4d851236eebebfc8d6421a6c03109aa8e90267596
+  metadata.gz: da42675bc9bb7d9b5923fb1f9a98b043cd16acb1fe70332e290d6398d7138db7c9c7d0b02c4c85200634c7de5379810655394eae283e0c04e5a4a3515018519e
+  data.tar.gz: b946205f7c8893d4e8e54af2cd0a1309d407f830287a27cf597daeefc2b38e032bb9e3ea1e75ff38f738f635b8b8ebd6c91f90a749b97707fd1cd20081c05127

data/CHANGELOG.md

@@ -1,3 +1,43 @@
+0.13.2 (01/04/2022)
+-------------------
+
+* require top level active_support to fix issue with 7.0.0 [2b2f584](../../commit/2b2f584)
+
+0.13.1 (05/04/2021)
+-------------------
+
+* fix console exception, extend console startup wait loop, and add better console logging and error reporting [2d44553](../../commit/2d44553)
+* fix logging [3cc07b9](../../commit/3cc07b9)
+* allow selection of specific items with cli config dump [675ebfe](../../commit/675ebfe)
+* cli help line formatting [13b36a3](../../commit/13b36a3)
+* add container pull helper [22518f3](../../commit/22518f3)
+
+0.13.0 (01/20/2021)
+-------------------
+
+#### Notes on major changes
+
+* Now testing the atmos runtime against terraform 0.14 (and 0.11, 0.13)
+* The ability to link files into the terraform execution directory was enhanced, but still doesn't work for the .terraform.lock.hcl as it is overwritten by terraform and doesn't follow the link  
+
+#### Full changelog
+
+* exclude some new taskdef attributes for creation of new taskdef during deploy [a01b997](https://github.com/simplygenius/atmos/commit/a01b997)
+* add debug logging for aws sdk logger [55dd303](https://github.com/simplygenius/atmos/commit/55dd303)
+* directory sorting made more consistent across platforms [13622ff](https://github.com/simplygenius/atmos/commit/13622ff)
+* switch ci/cd from travis to github actions [13622ff](https://github.com/simplygenius/atmos/commit/13622ff)
+* fix test for tf 0.14 [122f861](https://github.com/simplygenius/atmos/commit/122f861)
+* replace plugin sharing with built in terraform variant [eb75266](https://github.com/simplygenius/atmos/commit/eb75266)
+* allow more options for linking in non-tf files into working dir [9878336](https://github.com/simplygenius/atmos/commit/9878336)
+* sort secret list [6b2dbfe](https://github.com/simplygenius/atmos/commit/6b2dbfe)
+* Only run terraform init when needed [d77186b](https://github.com/simplygenius/atmos/commit/d77186b)
+
+
+0.12.1 (10/14/2020)
+-------------------
+
+* fix bug with deeply nested overrides [60cb84a](https://github.com/simplygenius/atmos/commit/60cb84a)
+
 0.12.0 (10/12/2020)
 -------------------
 

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/simplygenius/atmos.svg?branch=master)](https://travis-ci.org/simplygenius/atmos)
+[![Build Status](https://github.com/simplygenius/atmos/workflows/CD/badge.svg)](https://github.com/simplygenius/atmos/actions)
 [![Coverage Status](https://coveralls.io/repos/github/simplygenius/atmos/badge.svg?branch=master)](https://coveralls.io/github/simplygenius/atmos?branch=master)
 [![Gitter](https://badges.gitter.im/simplygenius/atmos.svg)](https://gitter.im/simplygenius/atmos?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 

data/lib/simplygenius/atmos/cli.rb

@@ -30,22 +30,22 @@ module SimplyGenius
       end
 
       option ["-d", "--debug"],
-             :flag, "debug output\n",
+             :flag, "debug output",
              default: false
 
       option ["-q", "--quiet"],
-             :flag, "suppress output\n",
+             :flag, "suppress output",
              default: false
 
       option ["-c", "--[no-]color"],
-             :flag, "colorize output (or not)\n (default: $stdout.tty?)"
+             :flag, "colorize output (or not) (default: $stdout.tty?)"
 
       option ["-e", "--atmos-env"],
-             'ENV', "The atmos environment\n",
+             'ENV', "The atmos environment",
              environment_variable: 'ATMOS_ENV', default: 'ops'
 
       option ["-g", "--atmos-group"],
-             'GROUP', "The atmos working group\n for selecting recipe groups\n",
+             'GROUP', "The atmos working group for selecting recipe groups",
              default: 'default'
 
       option ["-p", "--load-path"],
@@ -53,7 +53,7 @@ module SimplyGenius
              multivalued: true
 
       option ["-o", "--override"],
-             "KEYVALUE", "overrides atmos configuration\nin the form 'some.config=value' where value can\nbe expressed in yaml form for complex types\ne.g. foo=1 foo=abc, foo=[x, y], foo={x: y}",
+             "KEYVALUE", "overrides atmos configuration in the form 'some.config=value' where value can be expressed in yaml form for complex types e.g. foo=1 foo=abc, foo=[x, y], foo={x: y}",
              multivalued: true
 
       option ["-v", "--version"],
@@ -64,7 +64,7 @@ module SimplyGenius
       end
 
       option ["-l", "--[no-]log"],
-             :flag, "log to file in addition to terminal (or not)\n",
+             :flag, "log to file in addition to terminal (or not)",
              default: true
 
 
@@ -110,11 +110,29 @@ module SimplyGenius
         option ["-j", "--json"],
                :flag, "Dump config as json instead of yaml"
 
+        parameter "PATH",
+                  "The dot notation path of a specific config item to get",
+                  required: false
+
         def execute
+          if path
+            result = Atmos.config[path]
+            result = case result
+              when Hash
+                result.to_h
+              when Array
+                result.to_a
+              else
+                result
+            end
+          else
+            result = Atmos.config.to_h
+          end
+
           if json?
-            output = JSON.pretty_generate(Atmos.config.to_h)
+            output = JSON.pretty_generate(result)
           else
-            output = YAML.dump(Atmos.config.to_h)
+            output = YAML.dump(result).sub(/^\s*---\s*/, '')
           end
           logger.info output
         end
@@ -213,3 +231,37 @@ module SimplyGenius
 
   end
 end
+
+# Hack to make clamp usage less of a pain to get long lines to fit within a
+# standard terminal width
+class Clamp::Help::Builder
+
+  def word_wrap(text, line_width:)
+    text.split("\n").collect do |line|
+      line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip.split("\n") : line
+    end.flatten
+  end
+
+  def string
+    line_width = 79
+    indent_size = 4
+    indent = " " * indent_size
+    StringIO.new.tap do |out|
+      lines.each do |line|
+        case line
+        when Array
+          out << indent
+          out.puts(line[0])
+          formatted_line = line[1].gsub(/\((default|required)/, "\n\\0")
+          word_wrap(formatted_line, line_width: (line_width - indent_size * 2)).each do |l|
+            out << (indent * 2)
+            out.puts(l)
+          end
+        else
+          out.puts(line)
+        end
+      end
+    end.string
+  end
+
+end

data/lib/simplygenius/atmos/commands/account.rb

@@ -15,13 +15,13 @@ module SimplyGenius
         subcommand "create", "Create a new account" do
 
           option ["-s", "--source-env"],
-                 "SOURCE_ENV", "Base the new env on a clone of the given one\n"
+                 "SOURCE_ENV", "Base the new env on a clone of the given one"
 
           option ["-e", "--email"],
-                 "EMAIL", "override default email used for new account\n"
+                 "EMAIL", "override default email used for new account"
 
           option ["-n", "--name"],
-                 "NAME", "override default name used for new account\n"
+                 "NAME", "override default name used for new account"
 
           parameter "ENV",
                     "The name of the new env to create"
@@ -62,25 +62,25 @@ module SimplyGenius
           end
         end
 
-        subcommand "setup_credentials", "Convenience that adds accounts to the local aws\ncredentials store" do
+        subcommand "setup_credentials", "Convenience that adds accounts to the local aws credentials store" do
 
           option ["-u", "--user"],
-                 "USERNAME", "The username in the cloud provider\n", required: true
+                 "USERNAME", "The username in the cloud provider", required: true
 
           option ["-k", "--key"],
-                 "KEY", "The access key in the cloud provider\n"
+                 "KEY", "The access key in the cloud provider"
 
           option ["-s", "--secret"],
-                 "SECRET", "The access secret in the cloud provider\n"
+                 "SECRET", "The access secret in the cloud provider"
 
           option ["-d", "--default"],
-                 :flag, "Sets as default credentials\n"
+                 :flag, "Sets as default credentials"
 
           option ["-f", "--force"],
-                 :flag, "Forces overwrites of existing\n"
+                 :flag, "Forces overwrites of existing"
 
           option ["-n", "--nowrite"],
-                 :flag, "Trial run without writing results to files\n"
+                 :flag, "Trial run without writing results to files"
 
           def execute
 

data/lib/simplygenius/atmos/commands/auth_exec.rb

@@ -12,7 +12,7 @@ module SimplyGenius
         end
 
         option ["-r", "--role"],
-               'ROLE', "overrides assume role name\n"
+               'ROLE', "overrides assume role name"
 
         parameter "COMMAND ...", "command to exec", :attribute_name => :command
 

data/lib/simplygenius/atmos/commands/bootstrap.rb

@@ -11,7 +11,7 @@ module SimplyGenius
         end
 
         option ["-f", "--force"],
-               :flag, "forces bootstrap\n"
+               :flag, "forces bootstrap"
 
         def execute
           orig_config = Atmos.config

data/lib/simplygenius/atmos/commands/container.rb

@@ -15,22 +15,44 @@ module SimplyGenius
         end
 
         option ["-c", "--cluster"],
-               "CLUSTER", "The cluster name\n",
+               "CLUSTER", "The cluster name",
                required: true
 
         option ["-r", "--role"],
-               "ROLE", "The role to assume when deploying\n"
+               "ROLE", "The role to assume when deploying"
+
+        subcommand "pull", "Pulls a container image from repository" do
+
+          option ["-v", "--revision"],
+                 "REVISION", "Use as the remote image revision"
+
+          parameter "NAME",
+                    "The name of the service (or task) to pull the image for"
+
+          def execute
+            Atmos.config.provider.auth_manager.authenticate(ENV, role: role) do |auth_env|
+              ClimateControl.modify(auth_env) do
+                mgr = Atmos.config.provider.container_manager
+
+                result = mgr.pull(name, revision: revision)
+
+                logger.info "Container pulled:\n #{display result}"
+              end
+            end
+          end
+
+        end
 
         subcommand "push", "Only push a container image without activating it" do
 
           option ["-i", "--image"],
-                 "IMAGE", "The local container image to deploy\nDefaults to service/task name"
+                 "IMAGE", "The local container image to deploy Defaults to service/task name"
 
           option ["-v", "--revision"],
-                 "REVISION", "Use as the remote image revision\n"
+                 "REVISION", "Use as the remote image revision"
 
           parameter "NAME ...",
-                    "The name of the service (or task) to deploy\nWhen multiple, the first is the primary, and\nthe rest get deployed with its image"
+                    "The name of the service (or task) to deploy. When multiple, the first is the primary, and the rest get deployed with its image"
 
           def default_image
             name_list.first
@@ -55,22 +77,22 @@ module SimplyGenius
         subcommand "activate", "Activate a container that has already been pushed" do
 
           option ["-v", "--revision"],
-                 "REVISION", "Use the given revision of the pushed image\nto activate\n"
+                 "REVISION", "Use the given revision of the pushed image to activate"
 
           option ["-w", "--wait"],
-                 :flag, "Wait for service to become stable after deploy\nnon-zero exit on fail"
+                 :flag, "Wait for service to become stable after deploy non-zero exit on fail"
 
           option ["-l", "--list"],
-                 :flag, "List the most recent pushed images\n"
+                 :flag, "List the most recent pushed images"
 
           option ["-t", "--tagcount"],
-                 "N", "Only show the last N items when listing\n",
+                 "N", "Only show the last N items when listing",
                  default: 10 do |s|
             Integer(s)
           end
 
           parameter "NAME ...",
-                    "The name of the service (or task) to activate\nWhen multiple, the first is the primary, and\nthe rest get activated with its image"
+                    "The name of the service (or task) to activate. When multiple, the first is the primary, and the rest get activated with its image"
 
           def execute
             Atmos.config.provider.auth_manager.authenticate(ENV, role: role) do |auth_env|
@@ -125,16 +147,16 @@ module SimplyGenius
         subcommand "deploy", "Push and activate a container" do
 
           option ["-i", "--image"],
-                 "IMAGE", "The local container image to deploy\nDefaults to service/task name"
+                 "IMAGE", "The local container image to deploy. Defaults to service/task name"
 
           option ["-w", "--wait"],
-                 :flag, "Wait for service to become stable after deploy\nnon-zero exit on fail"
+                 :flag, "Wait for service to become stable after deploy non-zero exit on fail"
 
           option ["-v", "--revision"],
-                 "REVISION", "Use as the remote image revision\n"
+                 "REVISION", "Use as the remote image revision"
 
           parameter "NAME ...",
-                    "The name of the service (or task) to deploy\nWhen multiple, the first is the primary, and\nthe rest get deployed with its image"
+                    "The name of the service (or task) to deploy. When multiple, the first is the primary, and the rest get deployed with its image"
 
           def default_image
             name_list.first
@@ -171,10 +193,10 @@ module SimplyGenius
         subcommand "console", "Spawn a console and attach to it" do
 
           option ["-p", "--persist"],
-                 :flag, "Leave the task running after disconnect\n"
+                 :flag, "Leave the task running after disconnect"
 
           parameter "NAME",
-                    "The name of the service (or task) to attach\nthe console to"
+                    "The name of the service (or task) to attach the console to"
 
           def execute
             Atmos.config.provider.auth_manager.authenticate(ENV, role: role) do |auth_env|
@@ -191,8 +213,12 @@ module SimplyGenius
                 logger.debug "Run task result: #{result}"
                 begin
                   match = result[:log_match]
-                  local_command = local_command.collect {|c| match.names.each {|n| c = c.gsub("<#{n}>", match[n]) }; c }
-                  system(*local_command)
+                  if match.blank?
+                    logger.error("Aborting, the console task failed to produce the expected output: #{log_pattern}")
+                  else
+                    local_command = local_command.collect {|c| match.names.each {|n| c = c.gsub("<#{n}>", match[n]) }; c }
+                    system(*local_command)
+                  end
                 ensure
                   if persist?
                     logger.info "Console disconnected, you can reconnect with: #{local_command.join(" ")}"

data/lib/simplygenius/atmos/commands/generate.rb

@@ -34,12 +34,12 @@ module SimplyGenius
         option ["-l", "--list"],
                :flag, "list available templates"
         option ["-u", "--update"],
-               :flag, "update all installed templates\n"
+               :flag, "update all installed templates"
         option ["-p", "--sourcepath"],
                "PATH", "search for templates using given sourcepath",
                multivalued: true
         option ["-r", "--[no-]sourcepaths"],
-               :flag, "clear sourcepaths from template search\n", default: true
+               :flag, "clear sourcepaths from template search", default: true
         option ["-c", "--context"],
                "CONTEXT", "provide context variables (dot notation)",
                multivalued: true

data/lib/simplygenius/atmos/commands/init.rb

@@ -15,8 +15,6 @@ module SimplyGenius
         def execute
           @terraform_arguments.insert(0, "init")
           super
-
-          init_shared_plugins
         end
 
       end

data/lib/simplygenius/atmos/commands/otp.rb

@@ -13,7 +13,7 @@ module SimplyGenius
         end
 
         option ["-s", "--secret"],
-               'SECRET', "The otp secret\nWill save for future use"
+               'SECRET', "The otp secret. Will save for future use"
 
         option ["-c", "--clipboard"],
                :flag,

data/lib/simplygenius/atmos/commands/secret.rb

@@ -31,7 +31,7 @@ module SimplyGenius
         subcommand "set", "Sets the secret value" do
 
           option ["-f", "--force"],
-                 :flag, "forces updates for pre-existing secret\n",
+                 :flag, "forces updates for pre-existing secret",
                  default: false
 
           parameter "KEY",
@@ -60,7 +60,7 @@ module SimplyGenius
             Atmos.config.provider.auth_manager.authenticate(ENV) do |auth_env|
               ClimateControl.modify(auth_env) do
                 logger.info "Secret keys are:"
-                Atmos.config.provider.secret_manager.to_h.keys.each {|k| logger.info k}
+                Atmos.config.provider.secret_manager.to_h.keys.sort.each {|k| logger.info k}
               end
             end
 

data/lib/simplygenius/atmos/commands/terraform.rb

@@ -21,27 +21,23 @@ module SimplyGenius
         end
 
         def init_automatically(auth_env, get_modules)
-          tf_init_dir = File.join(Atmos.config.tf_working_dir, '.terraform')
+          tf_init_dir = File.join(Atmos.config.tf_working_dir, 'recipes', '.terraform')
           backend_initialized = File.exist?(File.join(tf_init_dir, 'terraform.tfstate'))
           auto_init_enabled = Atmos.config["atmos.terraform.auto_init"].to_s == "true"
 
           if auto_init && auto_init_enabled && ! backend_initialized
             exe = TerraformExecutor.new(process_env: auth_env)
             exe.run("init", get_modules: get_modules.present?)
-            init_shared_plugins
           end
         end
 
-        def init_shared_plugins
+        def enable_shared_plugins(env)
           if ! Atmos.config["atmos.terraform.disable_shared_plugins"]
             home_dir = OS.windows? ? File.join("~", "Application Data") : "~"
-            shared_plugins_dir = File.expand_path(File.join(home_dir,".terraform.d", "plugins"))
-            logger.debug("Updating shared terraform plugins dir: #{shared_plugins_dir}")
-            mkdir_p(shared_plugins_dir)
-            terraform_plugins_dir = File.join(Atmos.config.tf_working_dir,'recipes', '.terraform', 'plugins')
-            if File.exist?(terraform_plugins_dir)
-              cp_r("#{terraform_plugins_dir}/.", shared_plugins_dir)
-            end
+            plugin_cache_dir = File.expand_path(File.join(home_dir,".terraform.d", "plugin-cache"))
+            logger.debug("Plugin cache dir: #{plugin_cache_dir}")
+            mkdir_p(plugin_cache_dir)
+            env["TF_PLUGIN_CACHE_DIR"] = plugin_cache_dir
           end
         end
 
@@ -58,6 +54,8 @@ module SimplyGenius
             begin
               get_modules = @terraform_arguments.delete("--get-modules")
 
+              enable_shared_plugins(auth_env)
+
               init_automatically(auth_env, get_modules)
 
               exe = TerraformExecutor.new(process_env: auth_env)

data/lib/simplygenius/atmos/commands/tfutil.rb

@@ -13,21 +13,21 @@ module SimplyGenius
           "Useful utilities when calling out from terraform with data.external"
         end
 
-        subcommand "jsonify", "Manages json on stdin/out to conform\nto use in terraform data.external" do
+        subcommand "jsonify", "Manages json on stdin/out to conform to use in terraform data.external" do
 
           banner "Ensures json output only contains a single level Hash with string values (e.g. when execing curl returns a deep json hash of mixed values)"
 
           option ["-a", "--atmos_config"],
-                 :flag, "Includes the atmos config in the\nhash from parsing json on stdin"
+                 :flag, "Includes the atmos config in the hash from parsing json on stdin"
 
           option ["-c", "--clipboard"],
-                 :flag, "Copies the actual command used\nto the clipboard to allow external debugging"
+                 :flag, "Copies the actual command used to the clipboard to allow external debugging"
 
           option ["-j", "--json"],
                  :flag, "The command output is parsed as json"
 
           option ["-x", "--[no-]exit"],
-                 :flag, "Exit with the command's exit code\non failure (or not)", default: true
+                 :flag, "Exit with the command's exit code on failure (or not)", default: true
 
           parameter "COMMAND ...",
                     "The command to call", :attribute_name => :command
@@ -73,9 +73,9 @@ module SimplyGenius
               chunk = $stdin.read_nonblock(1)
               data = chunk + $stdin.read
               logger.debug("Received stdin: " + data)
-            rescue Errno::EAGAIN
+            rescue EOFError, SystemCallError => e # All Errno exceptions
               data = nil
-              logger.debug("No stdin")
+              logger.debug("No stdin due to: #{e}")
             end
             return data
           end

data/lib/simplygenius/atmos/commands/user.rb

@@ -14,31 +14,31 @@ module SimplyGenius
         subcommand "create", "Create a new user" do
 
           option ["-f", "--force"],
-                 :flag, "forces deletion/updates for pre-existing\nresources",
+                 :flag, "forces deletion/updates for pre-existing resources",
                  default: false
 
           option ["-l", "--login"],
-                 :flag, "generate a login password\n",
+                 :flag, "generate a login password",
                  default: false
 
           option ["-m", "--mfa"],
-                 :flag, "setup a mfa device\n",
+                 :flag, "setup a mfa device",
                  default: false
 
           option ["-k", "--key"],
-                 :flag, "create access keys\n",
+                 :flag, "create access keys",
                  default: false
 
           option ["-p", "--public-key"],
-                 "PUBLIC_KEY", "add ssh public key\n"
+                 "PUBLIC_KEY", "add ssh public key"
 
           option ["-g", "--group"],
                  "GROUP",
-                 "associate the given groups to new user\n",
+                 "associate the given groups to new user",
                  multivalued: true
 
           parameter "USERNAME",
-                    "The username of the user to add\nShould be an email address" do |u|
+                    "The username of the user to add. Should be an email address" do |u|
             raise ArgumentError.new("Not an email") if u !~ URI::MailTo::EMAIL_REGEXP
             u
           end

data/lib/simplygenius/atmos/config.rb

@@ -114,13 +114,14 @@ module SimplyGenius
 
         if merge_to_existing
           existing = load_file(user_config_file, SettingsHash.new)
-          data = config_merge(existing, data, ["saving #{user_config_file}"], finalize: true)
+          data = config_merge(existing, data, ["saving #{user_config_file}"])
+          data = finalize_merge(data)
         end
         File.write(user_config_file, YAML.dump(data.to_hash))
         File.chmod(0600, user_config_file)
       end
 
-      def config_merge(lhs, rhs, debug_state=[], finalize: true)
+      def config_merge(lhs, rhs, debug_state=[])
         result = nil
 
         return rhs if lhs.nil?
@@ -136,30 +137,22 @@ module SimplyGenius
             rhs.each do |k, v|
               new_state = debug_state + [k]
 
+              # The load sequence could have multiple overrides and its not
+              # obvious to the user which are lhs vs rhs, so rather than having
+              # one seem to win arbitrarily, we collect them all additively
+              # under their key, then at the end of the load process we push the
+              # override back as a replacement for its base key in the
+              # finalize_merge method. This means that if one has multiple
+              # overrides for the same key (not usually what one wants), those
+              # overrides get merged together additively before replacing the
+              # base.  Thus the need for the log messages below.
+              #
               if k =~ /^\^/
                 logger.debug { "Override seen at #{new_state.join(" -> ")}" }
                 logger.warn { "Multiple overrides on a single key seen at #{new_state.join(" -> ")}" } if result.has_key?(k)
               end
 
-              result[k] = config_merge(result[k], v, new_state, finalize: finalize)
-            end
-
-            # The load sequence could have multiple overrides and its not
-            # obvious to the user which are lhs vs rhs, so rather than having
-            # one seem to win arbitrarily, we collect them all additively under
-            # their key, then at the end of the load process we push the
-            # override back as a replacement for its base key.
-            # This means that if one has multiple overrides for the same key
-            # (not usually what one wants), those overrides get merged together
-            # additively before replacing the base.  Thus the need for the debug
-            # log message above.
-            if finalize
-              result.each do |k, v|
-                if k =~ /^\^(.*)/
-                  key = k.is_a?(Symbol) ? $1.to_sym : $1
-                  result[key] = result.delete(k)
-                end
-              end
+              result[k] = config_merge(result[k], v, new_state)
             end
 
           when Enumerable
@@ -183,6 +176,27 @@ module SimplyGenius
 
       private
 
+      def finalize_merge(config)
+        result = config.deep_dup
+
+        config.each do |k, v|
+
+          key = k
+
+          if k =~ /^\^(.*)/
+            key = k.is_a?(Symbol) ? $1.to_sym : $1
+            result[key] = result.delete(k)
+          end
+
+          if v.is_a?(Hash)
+            result[key] = finalize_merge(v)
+          end
+
+        end
+
+        return result
+      end
+
       def load_remote_config_sources(config, *remote_sources)
         remote_sources.each do |remote_source|
           logger.debug("Loading remote atmos config file: #{remote_source}")
@@ -205,7 +219,7 @@ module SimplyGenius
           pattern = File.expand_path(pattern)
           logger.debug("Expanded pattern: #{pattern}")
 
-          Dir[pattern].each do |f|
+          Dir[pattern].sort.each do |f|
             config = load_file(f, config)
           end
         end
@@ -222,7 +236,7 @@ module SimplyGenius
 
         begin
           submap = config.deep_fetch(group, name)
-          config = config_merge(config, submap, ["#{submap_file} submap(#{group}.#{name})"], finalize: false)
+          config = config_merge(config, submap, ["#{submap_file} submap(#{group}.#{name})"])
         rescue
           logger.debug("No #{group} config found for '#{name}'")
         end
@@ -245,7 +259,7 @@ module SimplyGenius
           @user_config_file = @full_config.notation_get("atmos.user_config") || @user_config_file
           @user_config_file = File.expand_path(@user_config_file)
           user_config_file_data = load_file(@user_config_file)
-          temp_settings = create_settings(config_merge(@full_config, user_config_file_data, [@user_config_file], finalize: false), finalize: false)
+          temp_settings = create_settings(config_merge(@full_config, user_config_file_data, [@user_config_file]), finalize: true)
 
           @full_config = load_config_sources(File.dirname(config_file), @full_config, *Array(temp_settings.notation_get("atmos.config_sources")))
           @full_config = load_remote_config_sources(@full_config, *Array(temp_settings.notation_get("atmos.remote_config_sources")))
@@ -255,7 +269,7 @@ module SimplyGenius
           @full_config = load_submap(File.dirname(config_file), 'providers', provider_name, @full_config)
           @full_config = load_submap(File.dirname(config_file), 'environments', atmos_env, @full_config)
 
-          @full_config = config_merge(@full_config, user_config_file_data, [@user_config_file], finalize: false)
+          @full_config = config_merge(@full_config, user_config_file_data, [@user_config_file])
 
           conf = create_settings(@full_config, finalize: true)
 
@@ -288,7 +302,7 @@ module SimplyGenius
           data = SettingsHash.new(data)
           data = block.call(data) if block
           # if lhs has a override ^, then it loses it when rhs gets merged in, which breaks things for subsequent merges
-          config = config_merge(config, data, [location], finalize: false)
+          config = config_merge(config, data, [location])
           @included_configs[location] = yml_string
         end
 
@@ -303,7 +317,8 @@ module SimplyGenius
         }
 
         global = SettingsHash.new(config.reject {|k, v| ['providers', 'environments'].include?(k) })
-        conf = config_merge(global, builtins, ["builtins"], finalize: finalize)
+        conf = config_merge(global, builtins, ["builtins"])
+        conf = finalize_merge(conf) if finalize
 
         conf.error_resolver = ->(statement) { find_config_error(statement) }
         conf.enable_expansion = true

data/lib/simplygenius/atmos/providers/aws/container_manager.rb

@@ -16,6 +16,33 @@ module SimplyGenius
             @provider = provider
           end
 
+          def pull(ecr_repo, revision: nil)
+
+            revision ||= 'latest'
+            result = {}
+
+            ecr = ::Aws::ECR::Client.new
+            resp = nil
+
+            resp = ecr.get_authorization_token
+            auth_data = resp.authorization_data.first
+            token = auth_data.authorization_token
+            endpoint = auth_data.proxy_endpoint
+            user, password = Base64.decode64(token).split(':')
+
+            # docker login into the ECR repo for the current account so that we can pull/push to it
+            run("docker", "login", "-u", user, "-p", password, endpoint)#, stdin_data: token)
+
+            image="#{ecr_repo}:#{revision}"
+            ecs_image="#{endpoint.sub(/https?:\/\//, '')}/#{image}"
+
+            logger.info "Pulling image from ECR repo #{ecs_image}"
+            run("docker", "pull", "#{ecs_image}")
+
+            result[:remote_image] = "#{ecs_image}"
+            return result
+          end
+
           def push(ecs_name, local_image,
                    ecr_repo: ecs_name, revision: nil)
 
@@ -64,7 +91,8 @@ module SimplyGenius
 
             new_defn = latest_defn.to_h
             [:revision, :status, :task_definition_arn,
-             :requires_attributes, :compatibilities].each do |attr|
+             :requires_attributes, :compatibilities,
+             :registered_at, :registered_by, :deregistered_at].each do |attr|
               new_defn.delete(attr)
             end
             new_defn[:container_definitions].each {|c| c[:image] = remote_image}
@@ -228,9 +256,12 @@ module SimplyGenius
 
               waiter_regexp = Regexp.new(waiter_log_pattern)
               log_stream = "#{log_stream_prefix}/#{name}/#{task_id}"
-              logger.info "Task started, looking for log pattern in group=#{log_group} stream=#{log_stream}"
+              logger.info "Task started, waiting for remote command"
+              logger.debug "Looking for log_pattern='#{waiter_log_pattern}' in group=#{log_group} stream=#{log_stream}"
               log_token = nil
-              10.times do
+              count = (Atmos.config['atmos.container.console.retry_count'] || 30).to_i
+              interval = (Atmos.config['atmos.container.console.retry_interval'] || 2).to_i
+              count.times do
                 resp = cwl.get_log_events(log_group_name: log_group, log_stream_name: log_stream, start_from_head: true, next_token: log_token)
                 resp.events.each do |e|
                   logger.debug("Task log #{e.timestamp}: #{e.message}")
@@ -239,8 +270,8 @@ module SimplyGenius
                     return result # return, not break due to doubly nested iterator
                   end
                 end
-                log_token = resp.next_forward_token
-                sleep 1
+                log_token = resp.next_forward_token if resp.events.length > 0
+                sleep interval
               end
             end
 

data/lib/simplygenius/atmos/providers/aws/provider.rb

@@ -1,5 +1,7 @@
 require_relative '../../../atmos'
 
+require 'aws-sdk-core'
+
 Dir.glob(File.join(__dir__, '*.rb')) do |f|
   require_relative "#{File.basename(f).sub(/\.rb$/, "")}"
 end
@@ -11,6 +13,7 @@ module SimplyGenius
 
         class Provider
           include GemLogger::LoggerSupport
+          ::Aws.config.update(logger: logger, log_level: :debug) if logger.debug?
 
           def initialize(name)
             @name = name

data/lib/simplygenius/atmos/terraform_executor.rb

@@ -308,7 +308,7 @@ module SimplyGenius
 
       def clean_links
         Find.find(Atmos.config.tf_working_dir) do |f|
-          Find.prune if f =~ /\/.terraform\/modules\//
+          Find.prune if f =~ /\/.terraform\//
           File.delete(f) if File.symlink?(f)
         end
       end
@@ -318,13 +318,30 @@ module SimplyGenius
         working_dir_links ||= ['modules', 'templates']
         working_dir_links.each do |subdir|
           source = File.join(Atmos.config.root_dir, subdir)
-          ln_sf(source, Atmos.config.tf_working_dir) if File.exist?(source)
+          parts = File.split(subdir)
+          target_dir = Atmos.config.tf_working_dir
+          if parts[0] != "."
+            target_dir = File.join(Atmos.config.tf_working_dir, parts[0])
+            mkdir_p(target_dir)
+          end
+          ln_sf(source, target_dir) if File.exist?(source)
         end
       end
 
       def link_recipes
         @recipes.each do |recipe|
-          ln_sf(File.join(Atmos.config.root_dir, 'recipes', "#{recipe}.tf"), tf_recipes_dir)
+          recipe_satisfied = false
+          ["#{recipe}.tf", recipe].each do |recipe_variant|
+            fqrecipe = File.join(Atmos.config.root_dir, 'recipes', recipe_variant)
+            if File.exist?(fqrecipe)
+              ln_sf(fqrecipe, tf_recipes_dir)
+              recipe_satisfied = true
+              break
+            else
+              logger.debug("Recipe target does not exist: #{fqrecipe}")
+            end
+          end
+          logger.error("Recipe '#{recipe}' is not present") unless recipe_satisfied
         end
       end
 

data/lib/simplygenius/atmos/version.rb

@@ -1,5 +1,5 @@
 module SimplyGenius
   module Atmos
-    VERSION = "0.12.0"
+    VERSION = "0.13.2"
   end
 end

data/lib/simplygenius/atmos.rb

@@ -5,6 +5,7 @@ require_relative 'atmos/version'
 # ui.rb when running its spec:
 # NameError: uninitialized constant ActiveSupport::JSON Did you mean?  JSON
 #
+require 'active_support'
 require 'active_support/json'
 require 'active_support/core_ext/object/json'
 
@@ -18,4 +19,7 @@ module SimplyGenius
 end
 
 require_relative 'atmos/logging'
+# Need to setup logging before loading any other files
+SimplyGenius::Atmos::Logging.setup_logging(:info, false, nil)
+
 require_relative 'atmos/config'

data/templates/new/config/atmos/runtime.yml

@@ -119,12 +119,12 @@ atmos:
   terraform:
     # Disable module fetch from convenience plan/apply commands
     disable_auto_modules: false
-    # By default (value=false), `atmos init` will update the terraform user
-    # plugin directory (~/.terraform.d/plugins) with the plugins for the current
-    # env/group so that they can be reused across all env/group combinations.
-    # Otherwise, disabling this functionality (value=true) means that each
-    # env/group combination will be independent and download all plugins for
-    # itself only
+    # By default (value=false), `atmos init` (and auto init) will set
+    # TF_PLUGIN_CACHE_DIR to ~/.terraform.d/plugin-cache so that when terraform
+    # installs providers/plugins they can be reused across all env/group
+    # combinations. Otherwise, disabling this functionality (value=true) means
+    # that each env/group combination will be independent and download all
+    # plugins for itself only
     disable_shared_plugins: false
     # Customize what gets linked into the working directory that terraform gets
     # executed in

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simplygenius-atmos
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.2
 platform: ruby
 authors:
 - Matt Conway
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-13 00:00:00.000000000 Z
+date: 2022-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -620,7 +620,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Atmos provides a terraform scaffold for creating cloud system architectures