simplygenius-atmos 0.13.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e57a5d360cd9c6016f83847333f3bf1519e70a8b73b0203c969d79fdf5a5e3f0
-  data.tar.gz: 4307c006f083fb94fdf5155df08a18a43d2ad6493070d279502b9a2a87a73c34
+  metadata.gz: e6e7037a18301c9dc8f8908d72b83d2390481145dcd1061d2c2d678e922bf658
+  data.tar.gz: '08858cc4a83f9af8cca448dc64c9a094ece7216c43b872c0c07bc9f8dcd32b0b'
 SHA512:
-  metadata.gz: d084865f58d1dfe8081debef0ce69c7fd2ab1068d813d8d704a4dc40962337c883be4763a2b161d5b8df4d0f6edd34a28209f95abb780b964d3abeb6dd7a94ab
-  data.tar.gz: 9f5f91afd5545dc14a073c4dfea67b86b699ed4add68077896cb151c990d50d32089246fe03b08b372eb98bb2694e98166933ac072fd22578e654a48771bc5b2
+  metadata.gz: 75e6f5eac9061b38e12f162a31e721d74f97d532d576b142b5a4be5b39b952a91ffc013fc2cce175bd6b8fd0ee2fe8da34bbe73790a8686b930c2a6fb8368556
+  data.tar.gz: 9ebfb5234e13d3972c507c1366c1785ddebea03e23bd6dcc33b32c3057876f0dca2ec696c77bbfcd2ef7ba274aed95f56ac73809e142b52178a4c30902a73bb5

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+0.13.1 (05/04/2021)
+-------------------
+
+* fix console exception, extend console startup wait loop, and add better console logging and error reporting [2d44553](../../commit/2d44553)
+* fix logging [3cc07b9](../../commit/3cc07b9)
+* allow selection of specific items with cli config dump [675ebfe](../../commit/675ebfe)
+* cli help line formatting [13b36a3](../../commit/13b36a3)
+* add container pull helper [22518f3](../../commit/22518f3)
+
 0.13.0 (01/20/2021)
 -------------------
 

data/lib/simplygenius/atmos.rb

@@ -18,4 +18,7 @@ module SimplyGenius
 end
 
 require_relative 'atmos/logging'
+# Need to setup logging before loading any other files
+SimplyGenius::Atmos::Logging.setup_logging(:info, false, nil)
+
 require_relative 'atmos/config'

data/lib/simplygenius/atmos/cli.rb

@@ -30,22 +30,22 @@ module SimplyGenius
       end
 
       option ["-d", "--debug"],
-             :flag, "debug output\n",
+             :flag, "debug output",
              default: false
 
       option ["-q", "--quiet"],
-             :flag, "suppress output\n",
+             :flag, "suppress output",
              default: false
 
       option ["-c", "--[no-]color"],
-             :flag, "colorize output (or not)\n (default: $stdout.tty?)"
+             :flag, "colorize output (or not) (default: $stdout.tty?)"
 
       option ["-e", "--atmos-env"],
-             'ENV', "The atmos environment\n",
+             'ENV', "The atmos environment",
              environment_variable: 'ATMOS_ENV', default: 'ops'
 
       option ["-g", "--atmos-group"],
-             'GROUP', "The atmos working group\n for selecting recipe groups\n",
+             'GROUP', "The atmos working group for selecting recipe groups",
              default: 'default'
 
       option ["-p", "--load-path"],
@@ -53,7 +53,7 @@ module SimplyGenius
              multivalued: true
 
       option ["-o", "--override"],
-             "KEYVALUE", "overrides atmos configuration\nin the form 'some.config=value' where value can\nbe expressed in yaml form for complex types\ne.g. foo=1 foo=abc, foo=[x, y], foo={x: y}",
+             "KEYVALUE", "overrides atmos configuration in the form 'some.config=value' where value can be expressed in yaml form for complex types e.g. foo=1 foo=abc, foo=[x, y], foo={x: y}",
              multivalued: true
 
       option ["-v", "--version"],
@@ -64,7 +64,7 @@ module SimplyGenius
       end
 
       option ["-l", "--[no-]log"],
-             :flag, "log to file in addition to terminal (or not)\n",
+             :flag, "log to file in addition to terminal (or not)",
              default: true
 
 
@@ -110,11 +110,29 @@ module SimplyGenius
         option ["-j", "--json"],
                :flag, "Dump config as json instead of yaml"
 
+        parameter "PATH",
+                  "The dot notation path of a specific config item to get",
+                  required: false
+
         def execute
+          if path
+            result = Atmos.config[path]
+            result = case result
+              when Hash
+                result.to_h
+              when Array
+                result.to_a
+              else
+                result
+            end
+          else
+            result = Atmos.config.to_h
+          end
+
           if json?
-            output = JSON.pretty_generate(Atmos.config.to_h)
+            output = JSON.pretty_generate(result)
           else
-            output = YAML.dump(Atmos.config.to_h)
+            output = YAML.dump(result).sub(/^\s*---\s*/, '')
           end
           logger.info output
         end
@@ -213,3 +231,37 @@ module SimplyGenius
 
   end
 end
+
+# Hack to make clamp usage less of a pain to get long lines to fit within a
+# standard terminal width
+class Clamp::Help::Builder
+
+  def word_wrap(text, line_width:)
+    text.split("\n").collect do |line|
+      line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip.split("\n") : line
+    end.flatten
+  end
+
+  def string
+    line_width = 79
+    indent_size = 4
+    indent = " " * indent_size
+    StringIO.new.tap do |out|
+      lines.each do |line|
+        case line
+        when Array
+          out << indent
+          out.puts(line[0])
+          formatted_line = line[1].gsub(/\((default|required)/, "\n\\0")
+          word_wrap(formatted_line, line_width: (line_width - indent_size * 2)).each do |l|
+            out << (indent * 2)
+            out.puts(l)
+          end
+        else
+          out.puts(line)
+        end
+      end
+    end.string
+  end
+
+end

data/lib/simplygenius/atmos/commands/account.rb

@@ -15,13 +15,13 @@ module SimplyGenius
         subcommand "create", "Create a new account" do
 
           option ["-s", "--source-env"],
-                 "SOURCE_ENV", "Base the new env on a clone of the given one\n"
+                 "SOURCE_ENV", "Base the new env on a clone of the given one"
 
           option ["-e", "--email"],
-                 "EMAIL", "override default email used for new account\n"
+                 "EMAIL", "override default email used for new account"
 
           option ["-n", "--name"],
-                 "NAME", "override default name used for new account\n"
+                 "NAME", "override default name used for new account"
 
           parameter "ENV",
                     "The name of the new env to create"
@@ -62,25 +62,25 @@ module SimplyGenius
           end
         end
 
-        subcommand "setup_credentials", "Convenience that adds accounts to the local aws\ncredentials store" do
+        subcommand "setup_credentials", "Convenience that adds accounts to the local aws credentials store" do
 
           option ["-u", "--user"],
-                 "USERNAME", "The username in the cloud provider\n", required: true
+                 "USERNAME", "The username in the cloud provider", required: true
 
           option ["-k", "--key"],
-                 "KEY", "The access key in the cloud provider\n"
+                 "KEY", "The access key in the cloud provider"
 
           option ["-s", "--secret"],
-                 "SECRET", "The access secret in the cloud provider\n"
+                 "SECRET", "The access secret in the cloud provider"
 
           option ["-d", "--default"],
-                 :flag, "Sets as default credentials\n"
+                 :flag, "Sets as default credentials"
 
           option ["-f", "--force"],
-                 :flag, "Forces overwrites of existing\n"
+                 :flag, "Forces overwrites of existing"
 
           option ["-n", "--nowrite"],
-                 :flag, "Trial run without writing results to files\n"
+                 :flag, "Trial run without writing results to files"
 
           def execute
 

data/lib/simplygenius/atmos/commands/auth_exec.rb

@@ -12,7 +12,7 @@ module SimplyGenius
         end
 
         option ["-r", "--role"],
-               'ROLE', "overrides assume role name\n"
+               'ROLE', "overrides assume role name"
 
         parameter "COMMAND ...", "command to exec", :attribute_name => :command
 

data/lib/simplygenius/atmos/commands/bootstrap.rb

@@ -11,7 +11,7 @@ module SimplyGenius
         end
 
         option ["-f", "--force"],
-               :flag, "forces bootstrap\n"
+               :flag, "forces bootstrap"
 
         def execute
           orig_config = Atmos.config

data/lib/simplygenius/atmos/commands/container.rb

@@ -15,22 +15,44 @@ module SimplyGenius
         end
 
         option ["-c", "--cluster"],
-               "CLUSTER", "The cluster name\n",
+               "CLUSTER", "The cluster name",
                required: true
 
         option ["-r", "--role"],
-               "ROLE", "The role to assume when deploying\n"
+               "ROLE", "The role to assume when deploying"
+
+        subcommand "pull", "Pulls a container image from repository" do
+
+          option ["-v", "--revision"],
+                 "REVISION", "Use as the remote image revision"
+
+          parameter "NAME",
+                    "The name of the service (or task) to pull the image for"
+
+          def execute
+            Atmos.config.provider.auth_manager.authenticate(ENV, role: role) do |auth_env|
+              ClimateControl.modify(auth_env) do
+                mgr = Atmos.config.provider.container_manager
+
+                result = mgr.pull(name, revision: revision)
+
+                logger.info "Container pulled:\n #{display result}"
+              end
+            end
+          end
+
+        end
 
         subcommand "push", "Only push a container image without activating it" do
 
           option ["-i", "--image"],
-                 "IMAGE", "The local container image to deploy\nDefaults to service/task name"
+                 "IMAGE", "The local container image to deploy Defaults to service/task name"
 
           option ["-v", "--revision"],
-                 "REVISION", "Use as the remote image revision\n"
+                 "REVISION", "Use as the remote image revision"
 
           parameter "NAME ...",
-                    "The name of the service (or task) to deploy\nWhen multiple, the first is the primary, and\nthe rest get deployed with its image"
+                    "The name of the service (or task) to deploy. When multiple, the first is the primary, and the rest get deployed with its image"
 
           def default_image
             name_list.first
@@ -55,22 +77,22 @@ module SimplyGenius
         subcommand "activate", "Activate a container that has already been pushed" do
 
           option ["-v", "--revision"],
-                 "REVISION", "Use the given revision of the pushed image\nto activate\n"
+                 "REVISION", "Use the given revision of the pushed image to activate"
 
           option ["-w", "--wait"],
-                 :flag, "Wait for service to become stable after deploy\nnon-zero exit on fail"
+                 :flag, "Wait for service to become stable after deploy non-zero exit on fail"
 
           option ["-l", "--list"],
-                 :flag, "List the most recent pushed images\n"
+                 :flag, "List the most recent pushed images"
 
           option ["-t", "--tagcount"],
-                 "N", "Only show the last N items when listing\n",
+                 "N", "Only show the last N items when listing",
                  default: 10 do |s|
             Integer(s)
           end
 
           parameter "NAME ...",
-                    "The name of the service (or task) to activate\nWhen multiple, the first is the primary, and\nthe rest get activated with its image"
+                    "The name of the service (or task) to activate. When multiple, the first is the primary, and the rest get activated with its image"
 
           def execute
             Atmos.config.provider.auth_manager.authenticate(ENV, role: role) do |auth_env|
@@ -125,16 +147,16 @@ module SimplyGenius
         subcommand "deploy", "Push and activate a container" do
 
           option ["-i", "--image"],
-                 "IMAGE", "The local container image to deploy\nDefaults to service/task name"
+                 "IMAGE", "The local container image to deploy. Defaults to service/task name"
 
           option ["-w", "--wait"],
-                 :flag, "Wait for service to become stable after deploy\nnon-zero exit on fail"
+                 :flag, "Wait for service to become stable after deploy non-zero exit on fail"
 
           option ["-v", "--revision"],
-                 "REVISION", "Use as the remote image revision\n"
+                 "REVISION", "Use as the remote image revision"
 
           parameter "NAME ...",
-                    "The name of the service (or task) to deploy\nWhen multiple, the first is the primary, and\nthe rest get deployed with its image"
+                    "The name of the service (or task) to deploy. When multiple, the first is the primary, and the rest get deployed with its image"
 
           def default_image
             name_list.first
@@ -171,10 +193,10 @@ module SimplyGenius
         subcommand "console", "Spawn a console and attach to it" do
 
           option ["-p", "--persist"],
-                 :flag, "Leave the task running after disconnect\n"
+                 :flag, "Leave the task running after disconnect"
 
           parameter "NAME",
-                    "The name of the service (or task) to attach\nthe console to"
+                    "The name of the service (or task) to attach the console to"
 
           def execute
             Atmos.config.provider.auth_manager.authenticate(ENV, role: role) do |auth_env|
@@ -191,8 +213,12 @@ module SimplyGenius
                 logger.debug "Run task result: #{result}"
                 begin
                   match = result[:log_match]
-                  local_command = local_command.collect {|c| match.names.each {|n| c = c.gsub("<#{n}>", match[n]) }; c }
-                  system(*local_command)
+                  if match.blank?
+                    logger.error("Aborting, the console task failed to produce the expected output: #{log_pattern}")
+                  else
+                    local_command = local_command.collect {|c| match.names.each {|n| c = c.gsub("<#{n}>", match[n]) }; c }
+                    system(*local_command)
+                  end
                 ensure
                   if persist?
                     logger.info "Console disconnected, you can reconnect with: #{local_command.join(" ")}"

data/lib/simplygenius/atmos/commands/generate.rb

@@ -34,12 +34,12 @@ module SimplyGenius
         option ["-l", "--list"],
                :flag, "list available templates"
         option ["-u", "--update"],
-               :flag, "update all installed templates\n"
+               :flag, "update all installed templates"
         option ["-p", "--sourcepath"],
                "PATH", "search for templates using given sourcepath",
                multivalued: true
         option ["-r", "--[no-]sourcepaths"],
-               :flag, "clear sourcepaths from template search\n", default: true
+               :flag, "clear sourcepaths from template search", default: true
         option ["-c", "--context"],
                "CONTEXT", "provide context variables (dot notation)",
                multivalued: true

data/lib/simplygenius/atmos/commands/otp.rb

@@ -13,7 +13,7 @@ module SimplyGenius
         end
 
         option ["-s", "--secret"],
-               'SECRET', "The otp secret\nWill save for future use"
+               'SECRET', "The otp secret. Will save for future use"
 
         option ["-c", "--clipboard"],
                :flag,

data/lib/simplygenius/atmos/commands/secret.rb

@@ -31,7 +31,7 @@ module SimplyGenius
         subcommand "set", "Sets the secret value" do
 
           option ["-f", "--force"],
-                 :flag, "forces updates for pre-existing secret\n",
+                 :flag, "forces updates for pre-existing secret",
                  default: false
 
           parameter "KEY",

data/lib/simplygenius/atmos/commands/tfutil.rb

@@ -13,21 +13,21 @@ module SimplyGenius
           "Useful utilities when calling out from terraform with data.external"
         end
 
-        subcommand "jsonify", "Manages json on stdin/out to conform\nto use in terraform data.external" do
+        subcommand "jsonify", "Manages json on stdin/out to conform to use in terraform data.external" do
 
           banner "Ensures json output only contains a single level Hash with string values (e.g. when execing curl returns a deep json hash of mixed values)"
 
           option ["-a", "--atmos_config"],
-                 :flag, "Includes the atmos config in the\nhash from parsing json on stdin"
+                 :flag, "Includes the atmos config in the hash from parsing json on stdin"
 
           option ["-c", "--clipboard"],
-                 :flag, "Copies the actual command used\nto the clipboard to allow external debugging"
+                 :flag, "Copies the actual command used to the clipboard to allow external debugging"
 
           option ["-j", "--json"],
                  :flag, "The command output is parsed as json"
 
           option ["-x", "--[no-]exit"],
-                 :flag, "Exit with the command's exit code\non failure (or not)", default: true
+                 :flag, "Exit with the command's exit code on failure (or not)", default: true
 
           parameter "COMMAND ...",
                     "The command to call", :attribute_name => :command

data/lib/simplygenius/atmos/commands/user.rb

@@ -14,31 +14,31 @@ module SimplyGenius
         subcommand "create", "Create a new user" do
 
           option ["-f", "--force"],
-                 :flag, "forces deletion/updates for pre-existing\nresources",
+                 :flag, "forces deletion/updates for pre-existing resources",
                  default: false
 
           option ["-l", "--login"],
-                 :flag, "generate a login password\n",
+                 :flag, "generate a login password",
                  default: false
 
           option ["-m", "--mfa"],
-                 :flag, "setup a mfa device\n",
+                 :flag, "setup a mfa device",
                  default: false
 
           option ["-k", "--key"],
-                 :flag, "create access keys\n",
+                 :flag, "create access keys",
                  default: false
 
           option ["-p", "--public-key"],
-                 "PUBLIC_KEY", "add ssh public key\n"
+                 "PUBLIC_KEY", "add ssh public key"
 
           option ["-g", "--group"],
                  "GROUP",
-                 "associate the given groups to new user\n",
+                 "associate the given groups to new user",
                  multivalued: true
 
           parameter "USERNAME",
-                    "The username of the user to add\nShould be an email address" do |u|
+                    "The username of the user to add. Should be an email address" do |u|
             raise ArgumentError.new("Not an email") if u !~ URI::MailTo::EMAIL_REGEXP
             u
           end

data/lib/simplygenius/atmos/providers/aws/container_manager.rb

@@ -16,6 +16,33 @@ module SimplyGenius
             @provider = provider
           end
 
+          def pull(ecr_repo, revision: nil)
+
+            revision ||= 'latest'
+            result = {}
+
+            ecr = ::Aws::ECR::Client.new
+            resp = nil
+
+            resp = ecr.get_authorization_token
+            auth_data = resp.authorization_data.first
+            token = auth_data.authorization_token
+            endpoint = auth_data.proxy_endpoint
+            user, password = Base64.decode64(token).split(':')
+
+            # docker login into the ECR repo for the current account so that we can pull/push to it
+            run("docker", "login", "-u", user, "-p", password, endpoint)#, stdin_data: token)
+
+            image="#{ecr_repo}:#{revision}"
+            ecs_image="#{endpoint.sub(/https?:\/\//, '')}/#{image}"
+
+            logger.info "Pulling image from ECR repo #{ecs_image}"
+            run("docker", "pull", "#{ecs_image}")
+
+            result[:remote_image] = "#{ecs_image}"
+            return result
+          end
+
           def push(ecs_name, local_image,
                    ecr_repo: ecs_name, revision: nil)
 
@@ -65,7 +92,7 @@ module SimplyGenius
             new_defn = latest_defn.to_h
             [:revision, :status, :task_definition_arn,
              :requires_attributes, :compatibilities,
-             :registered_at, :registered_by].each do |attr|
+             :registered_at, :registered_by, :deregistered_at].each do |attr|
               new_defn.delete(attr)
             end
             new_defn[:container_definitions].each {|c| c[:image] = remote_image}
@@ -229,9 +256,12 @@ module SimplyGenius
 
               waiter_regexp = Regexp.new(waiter_log_pattern)
               log_stream = "#{log_stream_prefix}/#{name}/#{task_id}"
-              logger.info "Task started, looking for log pattern in group=#{log_group} stream=#{log_stream}"
+              logger.info "Task started, waiting for remote command"
+              logger.debug "Looking for log_pattern='#{waiter_log_pattern}' in group=#{log_group} stream=#{log_stream}"
               log_token = nil
-              10.times do
+              count = (Atmos.config['atmos.container.console.retry_count'] || 30).to_i
+              interval = (Atmos.config['atmos.container.console.retry_interval'] || 2).to_i
+              count.times do
                 resp = cwl.get_log_events(log_group_name: log_group, log_stream_name: log_stream, start_from_head: true, next_token: log_token)
                 resp.events.each do |e|
                   logger.debug("Task log #{e.timestamp}: #{e.message}")
@@ -240,8 +270,8 @@ module SimplyGenius
                     return result # return, not break due to doubly nested iterator
                   end
                 end
-                log_token = resp.next_forward_token
-                sleep 1
+                log_token = resp.next_forward_token if resp.events.length > 0
+                sleep interval
               end
             end
 

data/lib/simplygenius/atmos/providers/aws/provider.rb

@@ -13,7 +13,7 @@ module SimplyGenius
 
         class Provider
           include GemLogger::LoggerSupport
-          ::Aws.config.update(logger: logger, log_level: :debug)
+          ::Aws.config.update(logger: logger, log_level: :debug) if logger.debug?
 
           def initialize(name)
             @name = name

data/lib/simplygenius/atmos/version.rb

@@ -1,5 +1,5 @@
 module SimplyGenius
   module Atmos
-    VERSION = "0.13.0"
+    VERSION = "0.13.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simplygenius-atmos
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.13.1
 platform: ruby
 authors:
 - Matt Conway
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler