simplygenius-atmos 0.7.0

data/lib/atmos/providers/aws/s3_secret_manager.rb

@@ -0,0 +1,49 @@
+require_relative '../../../atmos'
+require 'aws-sdk-s3'
+
+module Atmos
+  module Providers
+    module Aws
+
+      class S3SecretManager
+        include GemLogger::LoggerSupport
+
+        def initialize(provider)
+          @provider = provider
+          logger.debug("Secrets config is: #{Atmos.config[:secret]}")
+          @bucket_name = Atmos.config[:secret][:bucket]
+          @encrypt = Atmos.config[:secret][:encrypt]
+        end
+
+        def set(key, value)
+          opts = {}
+          opts[:server_side_encryption] = "AES256" if @encrypt
+          bucket.object(key).put(body: value, **opts)
+        end
+
+        def get(key)
+          bucket.object(key).get.body.read
+        end
+
+        def delete(key)
+          bucket.object(key).delete
+        end
+
+        def to_h
+          Hash[bucket.objects.collect {|o|
+            [o.key, o.get.body.read]
+          }]
+        end
+
+        private
+
+        def bucket
+          raise ArgumentError.new("The s3 secret bucket is not set") unless @bucket_name
+          @bucket ||= ::Aws::S3::Bucket.new(@bucket_name)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/atmos/providers/aws/user_manager.rb

@@ -0,0 +1,211 @@
+require_relative '../../../atmos'
+require_relative '../../../atmos/otp'
+require 'aws-sdk-iam'
+require 'securerandom'
+
+module Atmos
+  module Providers
+    module Aws
+
+      class UserManager
+        include GemLogger::LoggerSupport
+        include FileUtils
+
+        def initialize(provider)
+          @provider = provider
+        end
+
+        def create_user(user_name)
+          result = {}
+          client = ::Aws::IAM::Client.new
+          resource = ::Aws::IAM::Resource.new
+
+          user = resource.user(user_name)
+
+          if user.exists?
+            logger.info "User '#{user_name}' already exists"
+          else
+            logger.info "Creating new user '#{user_name}'"
+            user = resource.create_user(user_name: user_name)
+            client.wait_until(:user_exists, user_name: user_name)
+            logger.debug "User created, user_name=#{user_name}"
+          end
+
+          result[:user_name] = user_name
+
+          return result
+        end
+
+        def set_groups(user_name, groups, force: false)
+          result = {}
+          resource = ::Aws::IAM::Resource.new
+
+          user = resource.user(user_name)
+
+          existing_groups = user.groups.collect(&:name)
+          groups_to_add = groups -  existing_groups
+          groups_to_remove = existing_groups - groups
+
+          result[:groups] = existing_groups
+
+          groups_to_add.each do |group|
+            logger.debug "Adding group: #{group}"
+            user.add_group(group_name: group)
+            result[:groups] << group
+          end
+
+          if force
+            groups_to_remove.each do |group|
+              logger.debug "Removing group: #{group}"
+              user.remove_group(group_name: group)
+              result[:groups].delete(group)
+            end
+          end
+
+          logger.info "User associated with groups=#{result[:groups]}"
+
+          return result
+        end
+
+        def enable_login(user_name, force: false)
+          result = {}
+          resource = ::Aws::IAM::Resource.new
+
+          user = resource.user(user_name)
+
+          password = ""
+          classes = [/[a-z]/, /[A-Z]/, /[0-9]/, /[!@#$%^&*()_+\-=\[\]{}|']/]
+          while ! classes.all? {|c| password =~ c }
+            password = SecureRandom.base64(15)
+          end
+
+          exists = false
+          begin
+            user.login_profile.create_date
+            exists = true
+          rescue ::Aws::IAM::Errors::NoSuchEntity
+            exists = false
+          end
+
+          if exists
+            logger.info "User login already exists"
+            if force
+              user.login_profile.update(password: password, password_reset_required: true)
+              result[:password] = password
+              logger.info "Updated user login with password=#{password}"
+            end
+          else
+            user.create_login_profile(password: password, password_reset_required: true)
+            result[:password] = password
+            logger.info "User login enabled with password=#{password}"
+          end
+
+          return result
+        end
+
+        def enable_mfa(user_name, force: false)
+          result = {}
+          client = ::Aws::IAM::Client.new
+          resource = ::Aws::IAM::Resource.new
+
+          user = resource.user(user_name)
+
+          if user.mfa_devices.first
+            logger.info "User mfa devices already exist"
+            if force
+              logger.info "Deleting old mfa devices"
+              user.mfa_devices.each do |dev|
+                dev.disassociate
+                client.delete_virtual_mfa_device(serial_number: dev.serial_number)
+                Atmos::Otp.instance.remove(user_name)
+              end
+            else
+              return result
+            end
+          end
+
+          resp = client.create_virtual_mfa_device(
+            virtual_mfa_device_name: user_name
+          )
+
+          serial = resp.virtual_mfa_device.serial_number
+          seed = resp.virtual_mfa_device.base_32_string_seed
+
+          Atmos::Otp.instance.add(user_name, seed)
+          code1 = Atmos::Otp.instance.generate(user_name)
+          interval = (30 - (Time.now.to_i % 30)) + 1
+          logger.info "Waiting for #{interval}s to generate second otp key for enablement"
+          sleep interval
+          code2 = Atmos::Otp.instance.generate(user_name)
+          raise "MFA codes should not be the same" if code1 == code2
+
+          resp = client.enable_mfa_device({
+            user_name: user_name,
+            serial_number: serial,
+            authentication_code_1: code1,
+            authentication_code_2: code2,
+          })
+
+          result[:mfa_secret] = seed
+
+          return result
+        end
+
+        def enable_access_keys(user_name, force: false)
+          result = {}
+          resource = ::Aws::IAM::Resource.new
+
+          user = resource.user(user_name)
+
+          if user.access_keys.first
+            logger.info "User access keys already exist"
+            if force
+              logger.info "Deleting old access keys"
+              user.access_keys.each do |key|
+                key.delete
+              end
+            else
+              return result
+            end
+          end
+
+          # TODO: auto add to ~/.aws/credentials and config
+          key_pair = user.create_access_key_pair
+          result[:key] = key_pair.access_key_id
+          result[:secret] = key_pair.secret
+          logger.debug "User keys generated key=#{key_pair.access_key_id}, secret=#{key_pair.secret}"
+
+          return result
+        end
+
+        def set_public_key(user_name, public_key, force: false)
+          result = {}
+          client = ::Aws::IAM::Client.new
+          resource = ::Aws::IAM::Resource.new
+
+          user = resource.user(user_name)
+          keys = client.list_ssh_public_keys(user_name: user_name).ssh_public_keys
+          if keys.size > 0
+            logger.info "User ssh public keys already exist"
+            if force
+              logger.info "Deleting old ssh public keys"
+              keys.each do |key|
+                client.delete_ssh_public_key(user_name: user_name,
+                                             ssh_public_key_id: key.ssh_public_key_id)
+              end
+            else
+              return result
+            end
+          end
+
+          client.upload_ssh_public_key(user_name: user_name, ssh_public_key_body: public_key)
+          logger.debug "User public key assigned: #{public_key}"
+
+          return result
+        end
+
+      end
+
+    end
+  end
+end

data/lib/atmos/settings_hash.rb

@@ -0,0 +1,90 @@
+require 'hashie'
+
+module Atmos
+
+  class SettingsHash <  Hashie::Mash
+    include GemLogger::LoggerSupport
+    include Hashie::Extensions::DeepMerge
+    include Hashie::Extensions::DeepFetch
+    disable_warnings
+
+    PATH_PATTERN = /[\.\[\]]/
+
+    def notation_get(key)
+      path = key.to_s.split(PATH_PATTERN).compact
+      path = path.collect {|p| p =~ /^\d+$/ ? p.to_i : p }
+      result = nil
+
+      begin
+        result = deep_fetch(*path)
+      rescue Hashie::Extensions::DeepFetch::UndefinedPathError => e
+        logger.debug("Settings missing value for key='#{key}'")
+      end
+
+      return result
+    end
+
+    def notation_put(key, value, additive: true)
+      path = key.to_s.split(PATH_PATTERN).compact
+      path = path.collect {|p| p =~ /^\d+$/ ? p.to_i : p }
+      current_level = self
+      path.each_with_index do |p, i|
+
+        if i == path.size - 1
+          if additive && current_level[p].is_a?(Array)
+            current_level[p] = current_level[p] | Array(value)
+          else
+            current_level[p] = value
+          end
+        else
+          if current_level[p].nil?
+            if path[i+1].is_a?(Integer)
+              current_level[p] = []
+            else
+              current_level[p] = {}
+            end
+          end
+        end
+
+        current_level = current_level[p]
+      end
+    end
+
+    def self.add_config(yml_file, key, value, additive: true)
+      orig_config_with_comments = File.read(yml_file)
+
+      comment_places = {}
+      comment_lines = []
+      orig_config_with_comments.each_line do |line|
+        line.gsub!(/\s+$/, "\n")
+        if line =~ /^\s*(#.*)?$/
+          comment_lines << line
+        else
+          if comment_lines.present?
+            comment_places[line.chomp] = comment_lines
+            comment_lines = []
+          end
+        end
+      end
+      comment_places["<EOF>"] = comment_lines
+
+      orig_config = SettingsHash.new((YAML.load_file(yml_file) rescue {}))
+      orig_config.notation_put(key, value, additive: additive)
+      new_config_no_comments = YAML.dump(orig_config.to_hash)
+      new_config_no_comments.sub!(/\A---\n/, "")
+
+      new_yml = ""
+      new_config_no_comments.each_line do |line|
+        line.gsub!(/\s+$/, "\n")
+        cline = comment_places.keys.find {|k| line =~ /^#{k}/ }
+        comments = comment_places[cline]
+        comments.each {|comment| new_yml << comment } if comments
+        new_yml << line
+      end
+      comment_places["<EOF>"].each {|comment| new_yml << comment }
+
+      return new_yml
+    end
+
+  end
+end

data/lib/atmos/terraform_executor.rb

@@ -0,0 +1,267 @@
+require_relative '../atmos'
+require_relative '../atmos/ipc'
+require_relative '../atmos/ui'
+require 'open3'
+require 'fileutils'
+require 'find'
+require 'climate_control'
+
+module Atmos
+
+  class TerraformExecutor
+    include GemLogger::LoggerSupport
+    include FileUtils
+    include Atmos::UI
+
+    class ProcessFailed < RuntimeError; end
+
+    def initialize(process_env: ENV, working_group: 'default')
+      @process_env = process_env
+      @working_group = working_group
+      @working_dir = Atmos.config.tf_working_dir(@working_group)
+      @recipes = Atmos.config["recipes.#{@working_group}"]
+    end
+
+    def run(*terraform_args, skip_backend: false, skip_secrets: false, get_modules: false, output_io: nil)
+      setup_working_dir(skip_backend: skip_backend)
+
+      if get_modules
+        logger.debug("Getting modules")
+        get_modules_io = StringIO.new
+        begin
+          execute("get", output_io: get_modules_io)
+        rescue Atmos::TerraformExecutor::ProcessFailed => e
+          logger.info(get_modules_io.string)
+          raise
+        end
+      end
+
+      return execute(*terraform_args, skip_secrets: skip_secrets, output_io: output_io)
+    end
+
+    private
+
+    def tf_cmd(*args)
+      ['terraform'] + args
+    end
+
+    def execute(*terraform_args, skip_secrets: false, output_io: nil)
+      cmd = tf_cmd(*terraform_args)
+      logger.debug("Running terraform: #{cmd.join(' ')}")
+
+      env = Hash[@process_env]
+      if ! skip_secrets
+        begin
+          env = env.merge(secrets_env)
+        rescue => e
+          logger.debug("Secrets not available: #{e}")
+        end
+      end
+
+      # lets tempfiles create by subprocesses be easily found by users
+      env['TMPDIR'] = Atmos.config.tmp_dir
+
+      # Lets terraform communicate back to atmos, e.g. for UI notifications
+      ipc = Atmos::Ipc.new(Atmos.config.tmp_dir)
+
+      IO.pipe do |stdout, stdout_writer|
+        IO.pipe do |stderr, stderr_writer|
+
+          stdout_writer.sync = stderr_writer.sync = true
+          # TODO: more filtering on terraform output?
+          stdout_thr = pipe_stream(stdout, output_io.nil? ? $stdout : output_io) do |data|
+            if data =~ /^[\e\[\dm\s]*Enter a value:[\e\[\dm\s]*$/
+              notify(message: "Terraform is waiting for user input")
+            end
+            data
+          end
+          stderr_thr = pipe_stream(stderr, output_io.nil? ? $stderr : output_io)
+
+          ipc.listen do |sock_path|
+
+            if Atmos.config['ipc.disable']
+              # Using : as the command makes execution of ipc from the
+              # terraform side a no-op in both cases of how we call it.  This
+              # way, terraform execution continues to work when IPC is disabled
+              # command = "$ATMOS_IPC_CLIENT <json_string>"
+              # program = ["sh", "-c", "$ATMOS_IPC_CLIENT"]
+              env['ATMOS_IPC_CLIENT'] = ":"
+            else
+              env['ATMOS_IPC_SOCK'] = sock_path
+              env['ATMOS_IPC_CLIENT'] = ipc.generate_client_script
+            end
+
+            # Was unable to get piping to work with stdin for some reason.  It
+            # worked in simple case, but started to fail when terraform config
+            # got more extensive.  Thus, using spawn to redirect stdin from the
+            # terminal direct to terraform, with IO.pipe to copy the outher
+            # streams.  Maybe in the future we can completely disconnect stdin
+            # and have atmos do the output parsing and stdin prompting
+            pid = spawn(env, *cmd,
+                        chdir: tf_recipes_dir,
+                        :out=>stdout_writer, :err=> stderr_writer, :in => :in)
+
+            logger.debug("Terraform started with pid #{pid}")
+            Process.wait(pid)
+          end
+
+          stdout_writer.close
+          stderr_writer.close
+          stdout_thr.join
+          stderr_thr.join
+
+          status = $?.exitstatus
+          logger.debug("Terraform exited: #{status}")
+          if status != 0
+            raise ProcessFailed.new "Terraform exited with non-zero exit code: #{status}"
+          end
+
+        end
+      end
+
+    end
+
+    def setup_working_dir(skip_backend: false)
+      clean_links
+      link_shared_plugin_dir
+      link_support_dirs
+      link_recipes
+      write_atmos_vars
+      setup_backend(skip_backend)
+    end
+
+    def setup_backend(skip_backend=false)
+      backend_file = File.join(tf_recipes_dir, 'atmos-backend.tf.json')
+      backend_config = (Atmos.config["backend"] || {}).clone
+
+      if backend_config.present? && ! skip_backend
+        logger.debug("Writing out terraform state backend config")
+
+        # Use a different state file per group
+        if @working_group
+          backend_config['key'] = "#{@working_group}-#{backend_config['key']}"
+        end
+
+        backend_type = backend_config.delete("type")
+
+        backend = {
+            "terraform" => {
+                "backend" => {
+                    backend_type => backend_config
+                }
+            }
+        }
+
+        File.write(backend_file, JSON.pretty_generate(backend))
+      else
+        logger.debug("Clearing terraform state backend config")
+        File.delete(backend_file) if File.exist?(backend_file)
+      end
+    end
+
+    # terraform currently (v0.11.3) doesn't handle maps with nested maps or
+    # lists well, so flatten them - nested maps get expanded into the top level
+    # one, with their keys being appended with underscores, and lists get
+    # joined with "," so we end up with a single hash with homogenous types
+    def homogenize_for_terraform(h, root={}, prefix="")
+      h.each do |k, v|
+        if v.is_a? Hash
+          homogenize_for_terraform(v, root, "#{k}_")
+        else
+          v = v.join(",") if v.is_a? Array
+          root["#{prefix}#{k}"] = v
+        end
+      end
+      return root
+    end
+
+    def tf_recipes_dir
+      @tf_recipes_dir ||= begin
+        dir = File.join(@working_dir, 'recipes')
+        logger.debug("Tf recipes dir: #{dir}")
+        mkdir_p(dir)
+        dir
+      end
+    end
+
+    def write_atmos_vars
+      File.open(File.join(tf_recipes_dir, 'atmos.auto.tfvars.json'), 'w') do |f|
+        atmos_var_config = atmos_config = homogenize_for_terraform(Atmos.config.to_h)
+
+        var_prefix = Atmos.config['var_prefix']
+        if var_prefix
+          atmos_var_config = Hash[atmos_var_config.collect {|k, v| ["#{var_prefix}#{k}", v]}]
+        end
+
+        var_hash = {
+            atmos_env: Atmos.config.atmos_env,
+            all_env_names: Atmos.config.all_env_names,
+            account_ids: Atmos.config.account_hash,
+            atmos_config: atmos_config
+        }
+        var_hash = var_hash.merge(atmos_var_config)
+        f.puts(JSON.pretty_generate(var_hash))
+      end
+    end
+
+    def secrets_env
+      # NOTE use an auto-deleting temp file if passing secrets through env ends
+      # up being problematic
+      # TODO fix the need for CC - TE calls for secrets which needs auth in
+      # ENV, so kinda clunk to have to do both CC and pass the env in
+      ClimateControl.modify(@process_env) do
+        secrets = Atmos.config.provider.secret_manager.to_h
+        env_secrets = Hash[secrets.collect { |k, v| ["TF_VAR_#{k}", v] }]
+        return env_secrets
+      end
+    end
+
+    def clean_links
+      Find.find(@working_dir) do |f|
+        Find.prune if f =~ /\/.terraform\/modules\//
+        File.delete(f) if File.symlink?(f)
+      end
+    end
+
+    def link_support_dirs
+      ['modules', 'templates'].each do |subdir|
+        ln_sf(File.join(Atmos.config.root_dir, subdir), @working_dir)
+      end
+    end
+
+    def link_shared_plugin_dir
+      if ! Atmos.config["terraform.disable_shared_plugins"]
+        shared_plugins_dir = File.join(Atmos.config.tmp_root, "terraform_plugins")
+        mkdir_p(shared_plugins_dir)
+        terraform_state_dir = File.join(tf_recipes_dir, '.terraform')
+        mkdir_p(terraform_state_dir)
+        terraform_plugins_dir = File.join(terraform_state_dir, 'plugins')
+        ln_sf(shared_plugins_dir, terraform_plugins_dir)
+      end
+    end
+
+    def link_recipes
+      @recipes.each do |recipe|
+        ln_sf(File.join(Atmos.config.root_dir, 'recipes', "#{recipe}.tf"), tf_recipes_dir)
+      end
+    end
+
+    def pipe_stream(src, dest)
+       Thread.new do
+         block_size = 1024
+         begin
+           while data = src.readpartial(block_size)
+             data = yield data if block_given?
+             dest.write(data)
+           end
+         rescue EOFError
+           nil
+         rescue Exception => e
+           logger.log_exception(e, "Stream failure")
+         end
+       end
+    end
+
+  end
+
+end