cloud-mu 1.9.0.pre.beta

data/modules/mu/clouds/aws/storage_pool.rb

@@ -0,0 +1,495 @@
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#	http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Cloud
+    class AWS
+      # A storage pool as configured in {MU::Config::BasketofKittens::storage_pools}
+      class StoragePool < MU::Cloud::StoragePool
+        @deploy = nil
+        @config = nil
+        attr_reader :mu_name
+        attr_reader :cloud_id
+        attr_reader :config
+
+        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
+        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::storage_pools}
+        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
+          @deploy = mommacat
+          @config = MU::Config.manxify(kitten_cfg)
+          @cloud_id ||= cloud_id
+          @mu_name ||= @deploy.getResourceName(@config['name'])
+        end
+
+        # Called automatically by {MU::Deploy#createResources}
+        # @return [String]: The cloud provider's identifier for this storage pool.
+        def create
+          MU.log "Creating storage pool #{@mu_name}"
+          resp = MU::Cloud::AWS.efs(@config['region']).create_file_system(
+            creation_token: @mu_name,
+            performance_mode: @config['storage_type']
+          )
+
+          attempts = 0
+          loop do
+            MU.log "Waiting for #{@mu_name}: #{resp.file_system_id} to become available" if attempts % 5 == 0
+            storage_pool = MU::Cloud::AWS.efs(@config['region']).describe_file_systems(
+              creation_token: @mu_name
+            ).file_systems.first
+            break if storage_pool.life_cycle_state == "available"
+            raise MuError, "Failed to create storage pool #{@mu_name}" if %w{deleting deleted}.include? storage_pool.life_cycle_state
+            sleep 10
+            attempts += 1
+            raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 20
+          end
+
+          addStandardTags(cloud_id: resp.file_system_id, region: @config['region'])
+          @cloud_id = resp.file_system_id
+
+          if @config['mount_points'] && !@config['mount_points'].empty?
+            mp_threads = []
+            parent_thread_id = Thread.current.object_id
+            @config['mount_points'].each { |target|
+              sgs = []
+              if target['add_firewall_rules']
+                target['add_firewall_rules'].each { |mount_sg|
+                  sg = @deploy.findLitterMate(type: "firewall_rule", name: mount_sg['rule_name'])
+                  sgs << sg.cloud_id if sg
+                }
+              end
+
+              if target.has_key?("vpc") and target['vpc'].has_key?("vpc_name")
+                vpc = @dependencies["vpc"][target['vpc']["vpc_name"]]
+                if target['vpc']["subnet_name"]
+                  subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
+                  if subnet_obj.nil?
+                    raise MuError, "Failed to locate subnet from #{subnet} in StoragePool #{@config['name']}:#{target['name']}"
+                  end
+                  target['vpc']['subnet_id'] = subnet_obj.cloud_id
+                end
+              elsif target['vpc']["subnets"] and !target['vpc']["subnets"].empty?
+                target['vpc']['subnet_id'] = target['vpc']["subnets"].first["subnet_id"]
+              end
+
+              mp_threads << Thread.new {
+                MU.dupGlobals(parent_thread_id)
+                mount_target = MU::Cloud::AWS::StoragePool.create_mount_target(
+                  cloud_id: @cloud_id,
+                  ip_address: target['ip_address'],
+                  subnet_id: target['vpc']['subnet_id'],
+                  security_groups: sgs,
+                  region: @config['region']
+                )
+                target['cloud_id'] = mount_target.mount_target_id
+              }
+            }
+
+            mp_threads.each { |t|
+              t.join
+            }
+          end
+
+          return @cloud_id
+        end
+
+        # Canonical Amazon Resource Number for this resource
+        # @return [String]
+        def arn
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@config['region']+":"+MU.account_number+":file-system/"+@cloud_id
+        end
+
+        # Locate an existing storage pool and return an array containing matching AWS resource descriptors for those that match.
+        # @param cloud_id [String]: The cloud provider's identifier for this resource.
+        # @param region [String]: The cloud provider region
+        # @param tag_key [String]: A tag key to search.
+        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
+        # @param flags [Hash]: Optional flags
+        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching storage pool
+        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
+          map = {}
+          if cloud_id
+            storge_pool = MU::Cloud::AWS.efs(region).describe_file_systems(
+              file_system_id: cloud_id
+            ).file_systems.first
+            
+            map[cloud_id] = storge_pool if storge_pool
+          end
+
+          if tag_value
+            storage_pools = MU::Cloud::AWS.efs(region).describe_file_systems.file_systems
+          
+            if !storage_pools.empty?
+              storage_pools.each{ |pool|
+                tags = MU::Cloud::AWS.efs(region).describe_tags(
+                  file_system_id: pool.file_system_id
+                ).tags
+
+                value = nil
+                tags.each{ |tag|
+                  if tag.key == tag_key
+                    value = tag.value
+                    break
+                  end
+                }
+                
+                if value == tag_value
+                  map[pool.file_system_id] = pool
+                  break
+                end
+              }
+            end
+          end
+
+          return map
+        end
+
+        # Add our standard tag set to an Amazon EFS File System.
+        # @param cloud_id [String]: The cloud provider's identifier for this resource.
+        # @param region [String]: The cloud provider region
+        def addStandardTags(cloud_id: nil, region: MU.curRegion)
+          if cloud_id
+            tags = []
+            MU::MommaCat.listStandardTags.each_pair { |name, value|
+              tags << {key: name, value: value}
+            }
+
+            name_tag = false
+            if @config['tags']
+              @config['tags'].each { |tag|
+                tags << {key: tag['key'], value: tag['value']}
+                name_tag = true if tag['key'] == "Name"
+              }
+            end
+
+            if @config['optional_tags']
+              MU::MommaCat.listOptionalTags.each_pair { |name, value|
+                tags << {key: name, value: value}
+              }
+            end
+
+            tags << {key: "Name", value: @mu_name} unless name_tag
+
+            MU::Cloud::AWS.efs(region).create_tags(
+              file_system_id: cloud_id,
+              tags: tags
+            )
+          else
+            MU.log "cloud_id not provided, not tagging resources", MU::WARN
+          end
+        end
+
+        # Called automatically by {MU::Deploy#createResources}
+        def groom
+          #Nothing to do
+        end
+
+        # Create a mount point for an existing storage pool and attach it to a given subnet
+        # @param cloud_id [String]: The cloud provider's identifier of the storage pool.
+        # @param ip_address [String]: A private IP address that will be associated with mount point's network interface
+        # @param subnet_id [String]: The subnet_id to associate the mount point with
+        # @param security_groups [Array]: A list of security groups to associate with the mount point.
+        # @param region [String]: The cloud provider region
+        def self.create_mount_target(cloud_id: nil, ip_address: nil, subnet_id: nil, security_groups: [], region: MU.curRegion)
+          MU.log "Creating mount target for filesystem #{cloud_id}"
+
+          resp = MU::Cloud::AWS.efs(region).create_mount_target(
+            file_system_id: cloud_id,
+            subnet_id: subnet_id,
+            ip_address: ip_address,
+            security_groups: security_groups
+          )
+
+          attempts = 0
+          retries = 0
+          loop do
+            MU.log "Waiting for #{resp.mount_target_id} to become available", MU::NOTICE if attempts % 10 == 0
+            begin
+              mount_target = MU::Cloud::AWS.efs(region).describe_mount_targets(
+                mount_target_id: resp.mount_target_id 
+              ).mount_targets.first
+            rescue Aws::EFS::Errors::MountTargetNotFound
+              if retries <= 3
+                sleep 5
+                retry
+              else
+                return nil
+              end
+            end
+
+            break if mount_target.life_cycle_state == "available"
+            raise MuError, "Failed to create mount target #{resp.mount_target_id }" if %w{deleting deleted}.include? mount_target.life_cycle_state
+            sleep 10
+            attempts += 1
+            raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 40
+          end
+
+          return resp
+        end
+
+        # Modify the security groups associated with an existing mount point 
+        # @param cloud_id [String]: The cloud provider's identifier of the mount point.
+        # @param replace [TrueClass, FalseClass]: If the provided security groups will replace or be added to the existing ones
+        # @param security_groups [Array]: A list of security groups to associate with the mount point.
+        # @param region [String]: The cloud provider region
+        def self.modify_security_groups(cloud_id: nil, replace: false , security_groups: [], region: MU.curRegion)
+          unless replace
+            extisting_sgs = MU::Cloud::AWS.efs(region).describe_mount_target_security_groups(
+              mount_target_id: cloud_id
+            ).security_groups
+
+            security_groups.concat extisting_sgs
+          end
+
+          security_groups.uniq!
+          resp = MU::Cloud::AWS.efs(region).modify_mount_target_security_groups(
+            mount_target_id: cloud_id,
+            security_groups: security_groups
+          )
+        end
+
+        # Register a description of this storage pool with this deployment's metadata.
+        def notify
+          deploy_struct = {}
+          storage_pool = MU::Cloud::AWS.efs(@config['region']).describe_file_systems(
+            creation_token: @mu_name
+          ).file_systems.first
+
+          targets = {}
+
+          if @config['mount_points'] && !@config['mount_points'].empty?
+            @config['mount_points'].each { |mp|
+              subnet = nil
+              dependencies
+              mp_vpc = if mp['vpc'] and mp['vpc']['vpc_name']
+                @deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'])
+# XXX non-sibling, findStray version
+              end
+
+              mount_targets = MU::Cloud::AWS.efs(@config['region']).describe_mount_targets(
+                file_system_id: storage_pool.file_system_id
+              ).mount_targets
+
+              mount_target = nil
+              subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
+              mp_vpc.subnets.each { |subnet_obj|
+                mount_targets.map { |t|
+                  if subnet_cidr_obj.contains(t.ip_address)
+                    mount_target = t
+                    subnet = subnet_obj.cloud_desc
+                  end
+                }
+                break if mount_target
+              }
+
+              # mount_target = MU::Cloud::AWS.efs(@config['region']).describe_mount_targets(
+                # mount_target_id: mp["cloud_id"]
+              # ).mount_targets.first
+
+
+              targets[mp["name"]] = {
+                "owner_id" => mount_target.owner_id,
+                "cloud_id" => mount_target.mount_target_id,
+                "file_system_id" => mount_target.file_system_id,
+                "mount_directory" => mp["directory"],
+                "subnet_id" => mount_target.subnet_id,
+                "vpc_id" => subnet.vpc_id,
+                "availability_zone" => subnet.availability_zone,
+                "state" => mount_target.life_cycle_state,
+                "ip_address" => mount_target.ip_address,
+                "endpoint" => "#{subnet.availability_zone}.#{mount_target.file_system_id}.efs.#{@config['region']}.amazonaws.com",
+                "network_interface_id" => mount_target.network_interface_id
+              }
+            }
+          end
+
+          deploy_struct = {
+            "owner_id" => storage_pool.owner_id,
+            "creation_token" => storage_pool.creation_token,
+            "identifier" => storage_pool.file_system_id,
+            "creation_time" => storage_pool.creation_time,
+            "number_of_mount_targets" => storage_pool.number_of_mount_targets,
+            "size_in_bytes" => storage_pool.size_in_bytes.value,
+            "type" => storage_pool.performance_mode,
+            "mount_targets" => targets
+          }
+
+          return deploy_struct
+        end
+
+        # Called by {MU::Cleanup}. Locates resources that were created by the
+        # currently-loaded deployment, and purges them.
+        # @param noop [Boolean]: If true, will only print what would be done
+        # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
+        # @param region [String]: The cloud provider region in which to operate
+        # @return [void]
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
+          supported_regions = %w{us-west-2 us-east-1 eu-west-1}
+          if supported_regions.include?(region)
+            begin 
+              resp = MU::Cloud::AWS.efs(region).describe_file_systems
+              return if resp.nil? or resp.file_systems.nil?
+              storage_pools = resp.file_systems
+            rescue Aws::EFS::Errors::AccessDeniedException
+              MU.log "Storage Pools not supported in this account", MU::NOTICE
+              return nil
+            end
+
+            our_pools = []
+            our_replication_group_ids = []
+
+            if !storage_pools.empty?
+              storage_pools.each{ |pool|
+                tags = MU::Cloud::AWS.efs(region).describe_tags(
+                  file_system_id: pool.file_system_id
+                ).tags
+
+                found_muid = false
+                found_master = false
+                tags.each { |tag|
+                  found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id
+                  found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
+                }
+                next if !found_muid
+
+                if ignoremaster
+                  our_pools << pool if found_muid
+                else
+                  our_pools << pool if found_muid && found_master
+                end
+              }
+            end
+
+            # How to identify mount points in a reliable way? Mount points are not tagged, which means we can only reliably identify mount points based on a filesystem ID. We can you our deployment metadata, but it isn’t necessarily reliable
+            # begin
+              # resp = MU::Cloud::AWS.efs(region).delete_mount_target(
+                # mount_target_id: "MountTargetId"
+              # )
+              # MU.log "Deleted mount target"
+            # rescue Aws::EFS::Errors::BadRequest => e
+              # MU.log "Mount target already deleted", MU::NOTICE if e.to_s.start_with?("invalid mount target ID")
+            # end
+
+            if !our_pools.empty?
+              our_pools.each{ |pool|
+                mount_targets = MU::Cloud::AWS.efs(region).describe_mount_targets(
+                  file_system_id: pool.file_system_id
+                ).mount_targets
+
+                if !mount_targets.empty?
+                  mount_targets.each{ |mp|
+                    MU.log "Deleting mount target #{mp.mount_target_id} for filesystem #{pool.name}: #{pool.file_system_id}"
+                    unless noop
+                      begin
+                        resp = MU::Cloud::AWS.efs(region).delete_mount_target(
+                          mount_target_id: mp.mount_target_id
+                          )
+                      rescue Aws::EFS::Errors::BadRequest => e
+                        MU.log "Mount target #{mp.mount_target_id} already deleted", MU::NOTICE if e.to_s.start_with?("invalid mount target ID")
+                      end
+                    end
+                  }
+                end
+
+                MU.log "Deleting filesystem #{pool.name}: #{pool.file_system_id}"
+                unless noop
+                  attempts = 0
+                  begin
+                    resp = MU::Cloud::AWS.efs(region).delete_file_system(
+                      file_system_id: pool.file_system_id
+                    )
+                  rescue Aws::EFS::Errors::BadRequest => e
+                    MU.log "Filesystem #{pool.name}: #{pool.file_system_id} already deleted", MU::NOTICE if e.to_s.start_with?("invalid file system ID")
+                  rescue Aws::EFS::Errors::FileSystemInUse
+                    MU.log "Filesystem #{pool.name}: #{pool.file_system_id} is still in use, retrying", MU::NOTICE
+                    sleep 10
+                    attempts += 1
+                    raise MuError, "Failed to delete filesystem #{pool.name}: #{pool.file_system_id}, still in use." if attempts >= 6
+                    retry
+                  end
+                end
+              }
+            end
+          end
+        end
+
+        # Cloud-specific configuration properties.
+        # @param config [MU::Config]: The calling MU::Config object
+        # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
+        def self.schema(config)
+          toplevel_required = []
+          schema = {
+            "ingress_rules" => {
+              "type" => "array",
+              "description" => "Firewall rules to apply to our mountpoints",
+              "items" => {
+                "type" => "object",
+                "description" => "Firewall rules to apply to our mountpoints",
+                "properties" => {
+                  "sgs" => {
+                    "type" => "array",
+                    "items" => {
+                      "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
+                      "type" => "string"
+                    }
+                  },
+                  "lbs" => {
+                    "type" => "array",
+                    "items" => {
+                      "description" => "AWS Load Balancers which will have this rule applied to their traffic",
+                      "type" => "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+          [toplevel_required, schema]
+        end
+
+        # Cloud-specific pre-processing of {MU::Config::BasketofKittens::storage_pools}, bare and unvalidated.
+        # @param pool [Hash]: The resource to process and validate
+        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @return [Boolean]: True if validation succeeded, False otherwise
+        def self.validateConfig(pool, configurator)
+          ok = true
+          supported_regions = %w{us-west-2 us-east-1 us-east-2 eu-west-1}
+
+          if !supported_regions.include?(pool['region'])
+            MU.log "Region #{pool['region']} not supported. Only #{supported_regions.join(',  ')} are supported", MU::ERR
+            ok = false
+          end
+
+          if pool['mount_points'] && !pool['mount_points'].empty?
+            pool['mount_points'].each{ |mp|
+              if mp['ingress_rules']
+                fwname = "storage-#{mp['name']}"
+                acl = {"name" => fwname, "rules" => mp['ingress_rules'], "region" => pool['region'], "optional_tags" => pool['optional_tags']}
+                acl["tags"] = pool['tags'] if pool['tags'] && !pool['tags'].empty?
+                acl["vpc"] = mp['vpc'].dup if mp['vpc']
+                ok = false if !configurator.insertKitten(acl, "firewall_rules")
+                mp["add_firewall_rules"] = [] if mp["add_firewall_rules"].nil?
+                mp["add_firewall_rules"] << {"rule_name" => fwname}
+              end
+  
+            }
+          end
+
+          ok
+        end
+
+        private
+      end #class
+    end #class
+  end
+end #module