simplygenius-atmos 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e52d0f9fe95219722359232e6a8895b0cdbf23f99fbeda7ad8a48fdfe8a21397
-  data.tar.gz: bb3c2e1d01fd2071ecc41e10b7622cb320e10dcd65ee90d254cc68465ee923db
+  metadata.gz: 02b47aaaf544e268b0d3a9a9d6a69d8165f165ef9e9e500753ec8c87787228b4
+  data.tar.gz: 673992076c584283ee340d683a5a0c5aa8d9c9b0c4203cac7c96250480a012a1
 SHA512:
-  metadata.gz: 0da4f8dc5d90d23b6f5f7bbc348c4b5c1a431924e7763d369a5bbfd3528bae260a7e627294ad9aebc2e67e88cc3f7a5843a5b269b3c90fba88d7af37cf981bf5
-  data.tar.gz: c8e92569daec346d3c9e2a963374fcefd9e04c23a3a6d298fc9b3474871e8edc2f899c0903a5bc19a032993ec53588d4683e2c08ad3c8471d1f8b45a64eed1fc
+  metadata.gz: a4a7fc7d2c74988b22482c4252023fb4ecd1f0216bb8f28b0e802b5ee7a4532a4fa6f995cca60f11e7015545d20566463d577f3cb656240563436490a4c70237
+  data.tar.gz: 2396303b7e6ce14afc455597477be362f3ea77a3540d6187ec743efe2a2f48dcc8ab5f2fc7a2175da3ad2e5addeeb72f6b1dabaeb2ebc9d2a85262ad8a70cdd8

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+0.9.3 (02/12/2019)
+------------------
+
+* add the ability to use a prefix for scoping secrets [470556a](https://github.com/simplygenius/atmos/commit/470556a)
+* add comment [b0592d5](https://github.com/simplygenius/atmos/commit/b0592d5)
+* fix pro recipe source url [9ddb8a8](https://github.com/simplygenius/atmos/commit/9ddb8a8)
+* handle flushing of data in plugin output handlers when streaming completes [e1e4707](https://github.com/simplygenius/atmos/commit/e1e4707)
+* refactor secrets config into a more general user config file in users home directory [06875b7](https://github.com/simplygenius/atmos/commit/06875b7)
+* expand warnings for missing aws auth [3ad6ec0](https://github.com/simplygenius/atmos/commit/3ad6ec0)
+
 0.9.2 (12/01/2018)
 ------------------
 

data/lib/simplygenius/atmos/config.rb

@@ -16,6 +16,7 @@ module SimplyGenius
       attr_accessor :atmos_env, :working_group,
                     :root_dir,
                     :config_file,
+                    :user_config_file,
                     :tmp_root
 
       def initialize(atmos_env, working_group = 'default')
@@ -23,6 +24,7 @@ module SimplyGenius
         @working_group = working_group
         @root_dir = File.expand_path(Dir.pwd)
         @config_file = File.join(root_dir, "config", "atmos.yml")
+        @user_config_file = "~/.atmos.yml"
         @tmp_root = File.join(root_dir, "tmp")
         @included_configs = []
       end
@@ -37,6 +39,12 @@ module SimplyGenius
         return result
       end
 
+      def []=(key, value)
+        load
+        result = @config.notation_put(key, value)
+        return result
+      end
+
       def to_h
         load
         @config.to_hash
@@ -100,6 +108,17 @@ module SimplyGenius
         end
       end
 
+      def save_user_config_file(data, merge_to_existing: true)
+        logger.debug("Saving to user config file (merging=#{merge_to_existing}): #{user_config_file}")
+
+        if merge_to_existing
+          existing = load_file(user_config_file, SettingsHash.new)
+          data = config_merge(existing, data, ["saving #{user_config_file}"])
+        end
+        File.write(user_config_file, YAML.dump(data.to_hash))
+        File.chmod(0600, user_config_file)
+      end
+
       private
 
       INTERP_PATTERN = /(\#\{([^\}]+)\})/
@@ -164,15 +183,7 @@ module SimplyGenius
           logger.debug("Expanded pattern: #{pattern}")
 
           Dir[pattern].each do |f|
-            logger.debug("Loading atmos config file: #{f}")
-            data = YAML.load_file(f)
-            if ! data.is_a?(Hash)
-              logger.debug("Skipping non-hash config file: #{f}")
-            else
-              h = SettingsHash.new(data)
-              config = config_merge(config, h, [f])
-              @included_configs << f
-            end
+            config = load_file(f, config)
           end
         end
 
@@ -182,16 +193,8 @@ module SimplyGenius
       def load_submap(relative_root, group, name, config)
         submap_dir = File.join(relative_root, 'atmos', group)
         submap_file = File.join(submap_dir, "#{name}.yml")
-        if File.exist?(submap_file)
-          logger.debug("Loading atmos #{group} config file: #{submap_file}")
-          data  = YAML.load_file(submap_file)
-          if ! data.is_a?(Hash)
-            logger.debug("Skipping non-hash config file: #{submap_file}")
-          else
-            h = SettingsHash.new({group => {name => data}})
-            config = config_merge(config, h, [submap_file])
-            @included_configs << submap_file
-          end
+        config = load_file(submap_file, config) do |d|
+          SettingsHash.new({group => {name => d}})
         end
 
         begin
@@ -209,15 +212,12 @@ module SimplyGenius
 
           logger.debug("Atmos env: #{atmos_env}")
 
+
           if ! File.exist?(config_file)
             logger.warn "Could not find an atmos config file at: #{config_file}"
             @full_config = SettingsHash.new
           else
-            logger.debug("Loading atmos config file #{config_file}")
-            data = YAML.load_file(config_file)
-            raise ArgumentError.new("Invalid main config file (not hash-like): #{config_file}") if ! data.is_a?(Hash)
-            @full_config = SettingsHash.new(data)
-            @included_configs << config_file
+            @full_config = load_file(config_file, SettingsHash.new)
           end
 
           @full_config = load_config_sources(File.dirname(config_file), @full_config, *Array(@full_config.notation_get("atmos.config_sources")))
@@ -227,6 +227,10 @@ module SimplyGenius
           @full_config = load_submap(File.dirname(config_file), 'providers', provider_name, @full_config)
           @full_config = load_submap(File.dirname(config_file), 'environments', atmos_env, @full_config)
 
+          @user_config_file = @full_config.notation_get("atmos.user_config") || @user_config_file
+          @user_config_file = File.expand_path(@user_config_file)
+          @full_config = load_file(user_config_file, @full_config)
+
           global = SettingsHash.new(@full_config.reject {|k, v| ['providers', 'environments'].include?(k) })
           conf = config_merge(global, {
               atmos_env: atmos_env,
@@ -237,6 +241,25 @@ module SimplyGenius
         end
       end
 
+      def load_file(file, config=SettingsHash.new, &block)
+        if File.exist?(file)
+          logger.debug("Loading atmos config file #{file}")
+          data = YAML.load_file(file)
+          if ! data.is_a?(Hash)
+            logger.debug("Skipping invalid atmos config file (not hash-like): #{file}")
+          else
+            data = SettingsHash.new(data)
+            data = block.call(data) if block
+            config = config_merge(config, data, [file])
+            @included_configs << file
+          end
+        else
+          logger.debug   "Could not find an atmos config file at: #{file}"
+        end
+
+        config
+      end
+
       def expand(config, obj)
         case obj
           when Hash

data/lib/simplygenius/atmos/generator.rb

@@ -51,6 +51,9 @@ module SimplyGenius
 
         end
 
+        # TODO: return all context so the calling template can see answers to
+        # template questions to use in customizing its output (e.g. service
+        # needs cluster name and ec2 backed state)
         return visited_templates
       end
 

data/lib/simplygenius/atmos/otp.rb

@@ -10,13 +10,9 @@ module SimplyGenius
       include GemLogger::LoggerSupport
 
       def initialize
-        @secret_file = Atmos.config["atmos.otp.secret_file"] || "~/.atmos.yml"
-        @secret_file = File.expand_path(@secret_file)
-        yml_hash = YAML.load_file(@secret_file) rescue Hash.new
-        @secret_store = SettingsHash.new(yml_hash)
-        @secret_store[Atmos.config[:org]] ||= {}
-        @secret_store[Atmos.config[:org]][:otp] ||= {}
-        @scoped_secret_store = @secret_store[Atmos.config[:org]][:otp]
+        @scoped_path = "atmos.otp.#{Atmos.config[:org]}"
+        Atmos.config[@scoped_path] ||= {}
+        @scoped_secret_store = Atmos.config[@scoped_path]
       end
 
       def add(name, secret)
@@ -32,8 +28,9 @@ module SimplyGenius
       end
 
       def save
-        File.write(@secret_file, YAML.dump(@secret_store.to_hash))
-        File.chmod(0600, @secret_file)
+        data = SettingsHash.new
+        data.notation_put(@scoped_path, @scoped_secret_store)
+        Atmos.config.save_user_config_file(data)
       end
 
       def generate(name)

data/lib/simplygenius/atmos/plugin_manager.rb

@@ -91,10 +91,10 @@ module SimplyGenius
         end
 
         def filter_block
-          return Proc.new do |data|
+          return Proc.new do |data, flushing: false|
             @filters.inject(data) do |memo, obj|
               begin
-                obj.filter(memo)
+                obj.filter(memo, flushing: flushing)
               rescue StandardError => e
                 logger.log_exception e, "Output filter failed during filter: #{obj.class}"
                 memo

data/lib/simplygenius/atmos/plugins/output_filter.rb

@@ -15,7 +15,7 @@ module SimplyGenius
           @context = context
         end
 
-        def filter(data)
+        def filter(data, flushing: false)
           raise "not implemented"
         end
 

data/lib/simplygenius/atmos/plugins/prompt_notify.rb

@@ -7,7 +7,7 @@ module SimplyGenius
 
       class PromptNotify < OutputFilter
 
-        def filter(data)
+        def filter(data, flushing: false)
           if data =~ /^[\e\[\dm\s]*Enter a value:[\e\[\dm\s]*$/
             notify(message: "Terraform is waiting for user input")
           end

data/lib/simplygenius/atmos/providers/aws/auth_manager.rb

@@ -25,8 +25,20 @@ module SimplyGenius
             profile = system_env['AWS_PROFILE']
             key = system_env['AWS_ACCESS_KEY_ID']
             secret = system_env['AWS_SECRET_ACCESS_KEY']
-            if profile.blank? && (key.blank? || secret.blank?)
-              logger.warn("An aws profile or key/secret should be supplied via the environment")
+            # dont warn if default profile
+            no_creds = ::Aws::SharedCredentials.new.credentials.nil? rescue true
+
+            if profile.blank? && (key.blank? || secret.blank?) && no_creds
+              logger.warn("No AWS credentials are active in the environment nor shared credential store")
+              logger.warn("Run 'aws configure' to add some to the shared credential store")
+            end
+            if profile.present? && key.present?
+              logger.warn("Ignoring AWS_PROFILE because AWS_ACCESS_KEY_ID is set in the environment")
+            end
+
+            if Atmos.config["auth.bypass"]
+              logger.warn("Bypassing atmos aws authentication")
+              return block.call(Hash[system_env])
             end
 
             # Handle bootstrapping a new env account.  Newly created organization

data/lib/simplygenius/atmos/providers/aws/s3_secret_manager.rb

@@ -13,26 +13,31 @@ module SimplyGenius
             @provider = provider
             logger.debug("Secrets config is: #{Atmos.config[:secret]}")
             @bucket_name = Atmos.config[:secret][:bucket]
+            @bucket_prefix = "#{Atmos.config[:secret][:prefix]}"
             @encrypt = Atmos.config[:secret][:encrypt]
           end
 
           def set(key, value)
             opts = {}
             opts[:server_side_encryption] = "AES256" if @encrypt
-            bucket.object(key).put(body: value, **opts)
+            bucket.object(@bucket_prefix + key).put(body: value, **opts)
           end
 
           def get(key)
-            bucket.object(key).get.body.read
+            bucket.object(@bucket_prefix + key).get.body.read
           end
 
           def delete(key)
-            bucket.object(key).delete
+            bucket.object(@bucket_prefix + key).delete
           end
 
           def to_h
-            Hash[bucket.objects.collect {|o|
-              [o.key, o.get.body.read]
+            Hash[bucket.objects(prefix: @bucket_prefix).collect {|o|
+              key = o.key
+              if @bucket_prefix.present?
+                key = o.key.gsub(/^#{@bucket_prefix}/, '')
+              end
+              [key, o.get.body.read]
             }]
           end
 

data/lib/simplygenius/atmos/terraform_executor.rb

@@ -259,18 +259,30 @@ module SimplyGenius
         end
       end
 
-      def pipe_stream(src, dest)
+      def pipe_stream(src, dest, &block)
          Thread.new do
            block_size = 1024
            begin
              while data = src.readpartial(block_size)
-               data = yield data if block_given?
+               data = block.call(data, flushing: false) if block
                dest.write(data)
              end
            rescue IOError, EOFError => e
              logger.log_exception(e, "Stream failure", level: :debug)
            rescue Exception => e
              logger.log_exception(e, "Stream failure")
+           ensure
+             begin
+               if block
+                 data = block.call('', flushing: true)
+                 dest.write(data)
+               end
+               dest.flush
+             rescue IOError, EOFError => e
+               logger.log_exception(e, "Stream failure while flushing", level: :debug)
+             rescue Exception => e
+               logger.log_exception(e, "Stream failure while flushing")
+             end
            end
          end
       end

data/lib/simplygenius/atmos/version.rb

@@ -1,5 +1,5 @@
 module SimplyGenius
   module Atmos
-    VERSION = "0.9.2"
+    VERSION = "0.9.3"
   end
 end

data/templates/new/config/atmos/runtime.yml

@@ -1,6 +1,11 @@
 # Configuration for controlling the behavior of the atmos runtime
 atmos:
 
+  # Configure the location of a user config file.  This is where secrets are
+  # stored, e.g. OTP secrets and atmos pro auth token
+  #
+  user_config_file: ~/.atmos.yml
+
   # Sources for atmos templates
   # To circumvent the version lock and get the latest git master recipes, use the git
   # location, e.g.:
@@ -9,11 +14,13 @@ atmos:
   template_sources:
   - name: atmos-recipes
     location: https://github.com/simplygenius/atmos-recipes/archive/v#{atmos_version}.zip#atmos-recipes-#{atmos_version}
+  # To access the Atmos Pro recipes, add your atmos pro auth token to
+  # ~/.atmos.yml and uncomment below
   #- name: atmos-pro-recipes
-  #  location: https://YOUR_API_KEY@releases.simplygenius.com/recipes/v#{atmos_version}#atmos-pro-recipes-#{atmos_version}
+  #  location: https://#{atmos.pro_auth_token}@releases.simplygenius.com/recipes/v#{atmos_version}.zip#atmos-pro-recipes-#{atmos_version}
 
   # To get the atmos-pro gem, add the gem source like:
-  #   gem sources -a https://YOUR_API_KEY@releases.simplygenius.com/gems/
+  #   gem sources -a https://ATMOS_PRO_AUTH_TOKEN@releases.simplygenius.com/gems/
   # then install the gem:
   #   gem install simplygenius-atmos-pro
   # then add simplygenius-atmos-pro to the plugins key below
@@ -36,11 +43,6 @@ atmos:
       # Disable notify callbacks
       disable: false
 
-  # Configure the integrated otp mechanism
-  otp:
-    # The file where otp secrets are stored
-    secret_file: ~/.atmos.yml
-
   # Configure the template generator
   generate:
     # The file storing the record of each generated template

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simplygenius-atmos
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - Matt Conway
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-01 00:00:00.000000000 Z
+date: 2019-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -526,8 +526,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Atmos provides a terraform scaffold for creating cloud system architectures