cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/clouds/aws.rb

@@ -28,6 +28,24 @@ module MU
 
       @@creds_loaded = {}
 
+      # Module used by {MU::Cloud} to insert additional instance methods into
+      # instantiated resources in this cloud layer.
+      module AdditionalResourceMethods
+      end
+
+      # A hook that is always called just before any of the instance method of
+      # our resource implementations gets invoked, so that we can ensure that
+      # repetitive setup tasks (like resolving +:resource_group+ for Azure
+      # resources) have always been done.
+      # @param cloudobj [MU::Cloud]
+      # @param deploy [MU::MommaCat]
+      def self.resourceInitHook(cloudobj, deploy)
+        class << self
+          attr_reader :cloudformation_data
+        end
+        cloudobj.instance_variable_set(:@cloudformation_data, {})
+      end
+
       # Load some credentials for using the AWS API
       # @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those.
       # @return [Aws::Credentials]
@@ -110,7 +128,7 @@ module MU
             else
               MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", MU::WARN
             end
-          rescue MU::Groomer::Chef::MuNoSuchSecret
+          rescue MU::Groomer::MuNoSuchSecret
             MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", MU::WARN
           end
         end
@@ -145,39 +163,99 @@ module MU
       # Given an AWS region, check the API to make sure it's a valid one
       # @param r [String]
       # @return [String]
-      def self.validate_region(r)
-        MU::Cloud::AWS.ec2(region: r).describe_availability_zones.availability_zones.first.region_name
+      def self.validate_region(r, credentials: nil)
+        begin
+          MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
+        rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
+          MU.log "Got '#{e.message}' trying to validate region #{r} (hosted: #{hosted?.to_s})", MU::ERR, details: loadCredentials(credentials)
+          raise MuError, "Got '#{e.message}' trying to validate region #{r} with credentials #{credentials ? credentials : "<default>"} (hosted: #{hosted?.to_s})"
+        end
+      end
+
+      # Tag a resource with all of our standard identifying tags.
+      #
+      # @param resource [String]: The cloud provider identifier of the resource to tag
+      # @param region [String]: The cloud provider region
+      # @return [void]
+      def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil)
+        tags = []
+        MU::MommaCat.listStandardTags.each_pair { |name, value|
+          if !value.nil?
+            tags << {key: name, value: value}
+          end
+        }
+        if MU::Cloud::CloudFormation.emitCloudFormation
+          return tags
+        end
+
+        attempts = 0
+        begin
+          MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_tags(
+            resources: [resource],
+            tags: tags
+          )
+        rescue Aws::EC2::Errors::ServiceError => e
+          MU.log "Got #{e.inspect} tagging #{resource} in #{region}, will retry", MU::WARN, details: caller.concat(tags) if attempts > 1
+          if attempts < 5
+            attempts = attempts + 1
+            sleep 15
+            retry
+          else
+            raise e
+          end
+        end
+        MU.log "Created standard tags for resource #{resource}", MU::DEBUG, details: caller
+      end
+
+      # If we reside in this cloud, return the VPC in which we, the Mu Master, reside.
+      # @return [MU::Cloud::VPC]
+      def self.myVPCObj
+        return nil if !hosted?
+        instance = MU.myCloudDescriptor
+        return nil if !instance or !instance.vpc_id
+        vpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: instance.vpc_id, dummy_ok: true)
+        return nil if vpc.nil? or vpc.size == 0
+        vpc.first
       end
 
       # If we've configured AWS as a provider, or are simply hosted in AWS, 
       # decide what our default region is.
-      def self.myRegion
+      def self.myRegion(credentials = nil)
         return @@myRegion_var if @@myRegion_var
 
         if credConfig.nil? and !hosted? and !ENV['EC2_REGION']
           return nil
         end
 
-
         if $MU_CFG and $MU_CFG['aws']
           $MU_CFG['aws'].each_pair { |credset, cfg|
+            next if credentials and credset != credentials
             next if !cfg['region']
-            if (cfg['default'] or !@@myRegion_var) and validate_region(cfg['region'])
+            if (cfg['default'] or !@@myRegion_var) and validate_region(cfg['region'], credentials: credset)
               @@myRegion_var = cfg['region']
-              break if cfg['default']
+              break if cfg['default'] or credentials
             end
           }
         elsif ENV.has_key?("EC2_REGION") and !ENV['EC2_REGION'].empty? and
-              validate_region(ENV['EC2_REGION'])
+              validate_region(ENV['EC2_REGION']) and
+              (
+               (ENV.has_key?("AWS_SECRET_ACCESS_KEY") and ENV.has_key?("AWS_SECRET_ACCESS_KEY") ) or
+               (Aws.config['access_key'] and Aws.config['access_secret'])
+              )
           # Make sure this string is valid by way of the API
           @@myRegion_var = ENV['EC2_REGION']
-        else
+        end
+
+        if hosted? and !@@myRegion_var
           # hacky, but useful in a pinch (and if we're hosted in AWS)
           az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone")
           @@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str
         end
+
+        @@myRegion_var
       end
 
+
       # Is the region we're dealing with a GovCloud region?
       # @param region [String]: The region in question, defaults to the Mu Master's local region
       def self.isGovCloud?(region = myRegion)
@@ -247,9 +325,8 @@ module MU
       # @param region [String]: The region to search.
       # @return [Array<String>]: The Availability Zones in this region.
       def self.listAZs(region: MU.curRegion, account: nil, credentials: nil)
-        if $MU_CFG and (!$MU_CFG['aws'] or !account_number)
-          return []
-        end
+        cfg = credConfig(credentials)
+        return [] if !cfg
         if !region.nil? and @@azs[region]
           return @@azs[region]
         end
@@ -265,6 +342,18 @@ module MU
         return @@azs[region]
       end
 
+      # Do cloud-specific deploy instantiation tasks, such as copying SSH keys
+      # around, sticking secrets in buckets, creating resource groups, etc
+      # @param deploy [MU::MommaCat]
+      def self.initDeploy(deploy)
+      end
+
+      # Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups,
+      # etc)
+      # @param deploy_id [MU::MommaCat]
+      def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
+      end
+
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
       # @param deploy_id [String]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
@@ -346,6 +435,14 @@ module MU
             instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
             if !instance_id.nil? and instance_id.size > 0
               @@is_in_aws = true
+              region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "")
+              begin
+                validate_region(region)
+              rescue MuError
+                @@creds_loaded.delete("#default")
+                @@is_in_aws = false
+                false
+              end
               return true
             end
           end
@@ -384,6 +481,16 @@ module MU
         sample
       end
 
+      # Return what we think of as a cloud object's habitat. In AWS, this means
+      # the +account_number+ in which it's resident. If this is not applicable,
+      # such as for a {Habitat} or {Folder}, returns nil.
+      # @param cloudobj [MU::Cloud::AWS]: The resource from which to extract the habitat id
+      # @return [String,nil]
+      def self.habitat(cloudobj, nolookup: false, deploy: nil)
+        cloudobj.respond_to?(:account_number) ? cloudobj.account_number : nil
+      end
+
+
       @@my_acct_num = nil
       @@my_hosted_cfg = nil
       @@acct_to_profile_map = {}
@@ -415,8 +522,29 @@ module MU
       # @param credentials [String]
       # @return [String]
       def self.adminBucketName(credentials = nil)
-         #XXX find a default if this particular account doesn't have a log_bucket_name configured
         cfg = credConfig(credentials)
+        return nil if !cfg
+        if !cfg['log_bucket_name']
+          cfg['log_bucket_name'] = $MU_CFG['hostname'] 
+          MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN
+        end
+        resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets
+        found = false
+        resp.buckets.each { |b|
+          if b.name == cfg['log_bucket_name']
+            found = true
+            break
+          end
+        }
+        if !found
+          MU.log "Attempting to create log bucket #{cfg['log_bucket_name']} for credentials #{credentials}", MU::WARN
+          begin
+            resp = MU::Cloud::AWS.s3(credentials: credentials).create_bucket(bucket: cfg['log_bucket_name'], acl: "private")
+          rescue Aws::S3::Errors::BucketAlreadyExists => e
+            raise MuError, "AWS credentials #{credentials} need a log bucket, and the name #{cfg['log_bucket_name']} is unavailable. Use mu-configure to edit credentials '#{credentials}' or 'hostname'"
+          end
+        end
+
         cfg['log_bucket_name']
       end
 
@@ -425,7 +553,8 @@ module MU
       # @param credentials [String]
       # @return [String]
       def self.adminBucketUrl(credentials = nil)
-        "s3://"+adminBucketName+"/"
+        return nil if !credConfig(credentials)
+        "s3://"+adminBucketName(credentials)+"/"
       end
 
       # Return the $MU_CFG data associated with a particular profile/name/set of
@@ -439,7 +568,9 @@ module MU
         # on a machine hosted in AWS, *and* that machine has an IAM profile,
         # fake it with those credentials and hope for the best.
         if !$MU_CFG['aws'] or !$MU_CFG['aws'].is_a?(Hash) or $MU_CFG['aws'].size == 0
-          return @@my_hosted_cfg if @@my_hosted_cfg
+          if @@my_hosted_cfg
+            return name_only ? "#default" : @@my_hosted_cfg
+          end
 
           if hosted?
             begin
@@ -464,9 +595,9 @@ module MU
         end
 
         if name.nil?
-          $MU_CFG['aws'].each_pair { |name, cfg|
+          $MU_CFG['aws'].each_pair { |set, cfg|
             if cfg['default']
-              return name_only ? name : cfg
+              return name_only ? set : cfg
             end
           }
         else
@@ -496,7 +627,6 @@ module MU
                 cfg['account_number'] = acct_num.to_s
                 @@acct_to_profile_map[name.to_s] = cfg
                 return name_only ? name.to_s : cfg
-                return cfg
               end
             }
           end
@@ -1026,10 +1156,14 @@ module MU
       # @return [void]
       def self.openFirewallForClients
         MU::Cloud.loadCloudType("AWS", :FirewallRule)
-        if File.exists?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
-          ::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+        begin
+          if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+            ::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+          end
+          ::Chef::Config[:environment] = MU.environment
+        rescue LoadError
+          # XXX why is Chef here
         end
-        ::Chef::Config[:environment] = MU.environment
 
         # This is the set of (TCP) ports we're opening to clients. We assume that
         # we can and and remove these without impacting anything a human has
@@ -1218,8 +1352,6 @@ module MU
 
           MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
           @api = Object.const_get("Aws::#{api}::Client").new(params)
-
-          @api
         end
 
         @instance_cache = {}
@@ -1239,7 +1371,7 @@ module MU
               retval = @api.method(method_sym).call
             end
             return retval
-          rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException => e
+          rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError => e
             if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
               MU.log e.inspect, MU::ERR
               raise e

data/modules/mu/clouds/aws/alarm.rb

@@ -17,21 +17,11 @@ module MU
     class AWS
       # A alarm as configured in {MU::Config::BasketofKittens::alarms}
       class Alarm < MU::Cloud::Alarm
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
 
-        @cloudformation_data = {}
-        attr_reader :cloudformation_data
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::alarms}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 

data/modules/mu/clouds/aws/bucket.rb

@@ -17,22 +17,14 @@ module MU
     class AWS
       # Support for AWS S3
       class Bucket < MU::Cloud::Bucket
-        @deploy = nil
-        @config = nil
 
         @@region_cache = {}
         @@region_cache_semaphore = Mutex.new
 
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::logs}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 
@@ -145,6 +137,51 @@ module MU
           end
         end
 
+        # Upload a file to a bucket.
+        # @param url [String]: Target URL, of the form s3://bucket/folder/file
+        # @param acl [String]: Canned ACL permission to assign to the object we upload
+        # @param file [String]: Path to a local file to write to our target location. One of +file+ or +data+ must be specified.
+        # @param data [String]: Data to write to our target location. One of +file+ or +data+ must be specified.
+        def self.upload(url, acl: "private", file: nil, data: nil, credentials: nil, region: nil)
+          if (!file or file.empty?) and !data
+            raise MuError, "Must specify a file or some data to upload to bucket #{s3_url}"
+          end
+
+          if file and !file.empty?
+            if !File.exist?(file) or !File.readable?(file)
+              raise MuError, "Unable to read #{file} for upload to #{url}"
+            else
+              data = File.read(file)
+            end
+          end
+
+          url.match(/^(?:s3:\/\/)([^\/:]+?)[\/:]\/?(.+)?/)
+          bucket = Regexp.last_match[1]
+          path = Regexp.last_match[2]
+          if !path 
+            if !file
+              raise MuError, "Unable to determine upload path from url #{url}"
+            end
+          end
+
+          begin
+puts data
+puts acl
+puts bucket
+puts path
+            MU.log "Writing #{path} to S3 bucket #{bucket}"
+            MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(
+              acl: acl,
+              bucket: bucket,
+              key: path,
+              body: data
+            )
+          rescue Aws::S3::Errors => e
+            raise MuError, "Got #{e.inspect} trying to write #{path} to #{bucket} (region: #{region}, credentials: #{credentials})"
+          end
+
+        end
+
         # Does this resource type exist as a global (cloud-wide) artifact, or
         # is it localized to a region/zone?
         # @return [Boolean]
@@ -172,13 +209,18 @@ module MU
                 if @@region_cache[bucket.name]
                   next if @@region_cache[bucket.name] != region
                 else
-                  location = MU::Cloud::AWS.s3(credentials: credentials, region: region).get_bucket_location(bucket: bucket.name).location_constraint
-
-                  if location.nil? or location.empty?
-                    @@region_cache[bucket.name] = region
-                  else
-                    @@region_cache[bucket.name] = location
+                  begin
+                    location = MU::Cloud::AWS.s3(credentials: credentials, region: region).get_bucket_location(bucket: bucket.name).location_constraint
+                    if location.nil? or location.empty?
+                      @@region_cache[bucket.name] = region
+                    else
+                      @@region_cache[bucket.name] = location
+                    end
+                  rescue Aws::S3::Errors::AccessDenied => e
+                    # this is routine- we saw a bucket that's not our business
+                    next
                   end
+
                 end
               }
 

data/modules/mu/clouds/aws/cache_cluster.rb

@@ -17,23 +17,11 @@ module MU
     class AWS
       # A cache cluster as configured in {MU::Config::BasketofKittens::cache_clusters}
       class CacheCluster < MU::Cloud::CacheCluster
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-
-        @cloudformation_data = {}
-        attr_reader :cloudformation_data
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::cache_clusters}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          # @mu_name = mu_name ? mu_name : @deploy.getResourceName(@config["name"])
 
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||=
             if @config["create_replication_group"]
               @deploy.getResourceName(@config["name"], max_length: 16, need_unique_string: true)
@@ -194,8 +182,8 @@ module MU
                 waiter.before_attempt do |attempts|
                   MU.log "Waiting for cache replication group #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
                 end
-                waiter.before_wait do |attempts, resp|
-                  throw :success if resp.replication_groups.first.status == "available"
+                waiter.before_wait do |attempts, r|
+                  throw :success if r.replication_groups.first.status == "available"
                   throw :failure if Time.now - wait_start_time > 1800
                 end
               end
@@ -251,8 +239,8 @@ module MU
                 waiter.before_attempt do |attempts|
                   MU.log "Waiting for cache cluster #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
                 end
-                waiter.before_wait do |attempts, resp|
-                  throw :success if resp.cache_clusters.first.cache_cluster_status  == "available"
+                waiter.before_wait do |attempts, r|
+                  throw :success if r.cache_clusters.first.cache_cluster_status  == "available"
                   throw :failure if Time.now - wait_start_time > 1800
                 end
               end