cloud-mu 3.1.3 → 3.3.0

data/modules/mu/providers/aws/vpc_subnet.rb

@@ -0,0 +1,286 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Cloud
+    class AWS
+
+      # Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
+      class VPC < MU::Cloud::VPC
+
+        # Subnets are almost a first-class resource. So let's kinda sorta treat
+        # them like one. This should only be invoked on objects that already
+        # exists in the cloud layer.
+        class Subnet < MU::Cloud::AWS::VPC
+
+          attr_reader :cloud_id
+          attr_reader :ip_block
+          attr_reader :mu_name
+          attr_reader :name
+          attr_reader :az
+          attr_reader :cloud_desc
+
+          # @param parent [MU::Cloud::AWS::VPC]: The parent VPC of this subnet.
+          # @param config [Hash<String>]:
+          def initialize(parent, config)
+            @parent = parent
+            @config = MU::Config.manxify(config)
+            @cloud_id = config['cloud_id']
+            @mu_name = config['mu_name']
+            @name = config['name']
+            @deploydata = config # This is a dummy for the sake of describe()
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
+            @az = resp.availability_zone
+            @ip_block = resp.cidr_block
+            @cloud_desc = resp # XXX this really isn't the cloud implementation's business
+
+          end
+
+          # Return the cloud identifier for the default route of this subnet.
+          # @return [String,nil]
+          def defaultRoute
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+                filters: [{name: "association.subnet-id", values: [@cloud_id]}]
+            )
+            if resp.route_tables.size == 0 # use default route table for the VPC
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+                 filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
+              )
+            end
+            resp.route_tables.each { |route_table|
+              route_table.routes.each { |route|
+                if route.destination_cidr_block =="0.0.0.0/0" and route.state != "blackhole"
+                  return route.instance_id if !route.instance_id.nil?
+                  return route.gateway_id if !route.gateway_id.nil?
+                  return route.vpc_peering_connection_id if !route.vpc_peering_connection_id.nil?
+                  return route.network_interface_id if !route.network_interface_id.nil?
+                end
+              }
+            }
+            return nil
+          end
+
+          # Is this subnet privately-routable only, or public?
+          # @return [Boolean]
+          def private?
+            return false if @cloud_id.nil?
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+                filters: [{name: "association.subnet-id", values: [@cloud_id]}]
+            )
+            if resp.route_tables.size == 0 # use default route table for the VPC
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+                 filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
+              )
+            end
+            resp.route_tables.each { |route_table|
+              route_table.routes.each { |route|
+                return false if !route.gateway_id.nil? and route.gateway_id != "local" # you can have an IgW and route it to a subset of IPs instead of 0.0.0.0/0
+                if route.destination_cidr_block == "0.0.0.0/0"
+                  return true if !route.instance_id.nil?
+                  return true if route.nat_gateway_id
+                end
+              }
+            }
+            return true
+          end
+        end # VPC::Subnet class
+
+        private
+
+        def create_subnets
+          return [] if @config['subnets'].nil? or @config['subnets'].empty?
+          nat_gateways = []
+
+          @eip_allocation_ids ||= []
+
+          subnetthreads = Array.new
+
+          azs = MU::Cloud::AWS.listAZs(region: @config['region'], credentials: @config['credentials'])
+          @config['subnets'].each { |subnet|
+            subnet_name = @config['name']+"-"+subnet['name']
+            az = subnet['availability_zone'] ? subnet['availability_zone'] : azs.op
+            MU.log "Creating Subnet #{subnet_name} (#{subnet['ip_block']}) in #{az}", details: subnet
+
+            subnetthreads << Thread.new {
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_subnet(
+                vpc_id: @cloud_id,
+                cidr_block: subnet['ip_block'],
+                availability_zone: az
+              ).subnet
+              subnet_id = subnet['subnet_id'] = resp.subnet_id
+
+              tag_me(subnet_id, @mu_name+"-"+subnet['name'])
+
+              loop_if = Proc.new {
+                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
+                (!resp or resp.state != "available")
+              }
+
+              MU.retrier([Aws::EC2::Errors::InvalidSubnetIDNotFound, NoMethodError], wait: 5, loop_if: loop_if) { |retries, _wait|
+                MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to become available", MU::NOTICE if retries > 0 and (retries % 3) == 0
+              }
+
+              if !subnet['route_table'].nil?
+                routes = {}
+                @config['route_tables'].each { |tbl|
+                  routes[tbl['name']] = tbl
+                }
+                if routes[subnet['route_table']].nil?
+                  raise "Subnet #{subnet_name} references nonexistent route #{subnet['route_table']}"
+                end
+                MU.log "Associating Route Table '#{subnet['route_table']}' (#{routes[subnet['route_table']]['route_table_id']}) with #{subnet_name}"
+                MU.retrier([Aws::EC2::Errors::InvalidRouteTableIDNotFound], wait: 10, max: 10) {
+                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_route_table(
+                    route_table_id: routes[subnet['route_table']]['route_table_id'],
+                    subnet_id: subnet_id
+                  )
+                }
+              end
+
+              if subnet.has_key?("map_public_ips")
+                MU.retrier([Aws::EC2::Errors::InvalidSubnetIDNotFound], wait: 10, max: 10) {
+                  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_subnet_attribute(
+                    subnet_id: subnet_id,
+                    map_public_ip_on_launch: {
+                      value: subnet['map_public_ips'],
+                    }
+                  )
+                }
+              end
+
+              if subnet['is_public'] and subnet['create_nat_gateway']
+                nat_gateways << create_nat_gateway(subnet)
+              end
+
+              if subnet["enable_traffic_logging"]
+                loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
+                logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
+                MU.log "Enabling traffic logging on Subnet #{subnet_name} in VPC #{@mu_name} to log group #{loggroup.mu_name}"
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
+                  resource_ids: [subnet_id],
+                  resource_type: "Subnet",
+                  traffic_type: subnet["traffic_type_to_log"],
+                  log_group_name: loggroup.mu_name,
+                  deliver_logs_permission_arn: logrole.cloudobj.arn
+                )
+              end
+            }
+          }
+
+          subnetthreads.each { |t|
+            t.join
+          }
+
+          nat_gateways
+        end
+
+        def allocate_eip_for_nat
+          MU::MommaCat.lock("nat-gateway-eipalloc")
+
+          eips = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(
+            filters: [
+              {
+                name: "domain",
+                values: ["vpc"]
+              }
+            ]
+          ).addresses
+
+          allocation_id = nil
+          eips.each { |eip|
+            next if !eip.association_id.nil? and !eip.association_id.empty?
+            if (eip.private_ip_address.nil? || eip.private_ip_address.empty?) and MU::MommaCat.lock(eip.allocation_id, true, true)
+              if !@eip_allocation_ids.include?(eip.allocation_id)
+                allocation_id = eip.allocation_id
+                break
+              end
+            end
+          }
+
+          if allocation_id.nil?
+            allocation_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).allocate_address(domain: "vpc").allocation_id
+            MU::MommaCat.lock(allocation_id, false, true)
+          end
+
+          @eip_allocation_ids << allocation_id
+
+          MU::MommaCat.unlock("nat-gateway-eipalloc")
+
+          allocation_id
+        end
+
+        def create_nat_gateway(subnet)
+          allocation_id = allocate_eip_for_nat
+
+          nat_gateway_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_nat_gateway(
+            subnet_id: subnet['subnet_id'],
+            allocation_id: allocation_id,
+          ).nat_gateway.nat_gateway_id
+
+          ensure_unlock = Proc.new { MU::MommaCat.unlock(allocation_id, true) }
+          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
+          loop_if = Proc.new {
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
+            resp.class != Aws::EC2::Types::NatGateway or resp.state == "pending"
+          }
+
+          MU.retrier([Aws::EmptyStructure, NoMethodError], wait: 5, max: 30, always: ensure_unlock, loop_if: loop_if) { |retries, _wait|
+            MU.log "Waiting for nat gateway #{nat_gateway_id} to become available (EIP allocation: #{allocation_id})" if retries % 5 == 0
+          }
+
+          raise MuError, "NAT Gateway failed #{nat_gateway_id}: #{resp}" if resp.state == "failed"
+
+          tag_me(nat_gateway_id)
+
+          {'id' => nat_gateway_id, 'availability_zone' => subnet['availability_zone']}
+        end
+
+        # Remove all subnets associated with the currently loaded deployment.
+        # @param noop [Boolean]: If true, will only print what would be done
+        # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
+        # @param region [String]: The cloud provider region
+        # @return [void]
+        def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_subnets(
+            filters: tagfilters
+          )
+          subnets = resp.data.subnets
+
+          return if subnets.nil? or subnets.size == 0
+
+          subnets.each { |subnet|
+            on_retry = Proc.new {
+              MU::Cloud::AWS::VPC.purge_interfaces(noop, [{name: "subnet-id", values: [subnet.subnet_id]}], region: region, credentials: credentials)
+            }
+
+            MU.log "Deleting Subnet #{subnet.subnet_id}"
+            MU.retrier([Aws::EC2::Errors::DependencyViolation], ignoreme: [Aws::EC2::Errors::InvalidSubnetIDNotFound], max: 20, on_retry: on_retry) { |_retries, wait|
+              begin
+                if subnet.state != "available"
+                  MU.log "Waiting for #{subnet.subnet_id} to be in a removable state...", MU::NOTICE
+                  sleep wait
+                else
+                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
+                end
+              end while subnet.state != "available"
+            }
+          }
+        end
+        private_class_method :purge_subnets
+
+      end # VPC class
+
+    end #class
+  end
+end #module

data/modules/mu/{clouds → providers}/azure.rb

@@ -47,6 +47,11 @@ module MU
         guid_chunks.join("-")
       end
 
+      # List all Azure subscriptions available to our credentials
+      def self.listHabitats(credentials = nil, use_cache: true)
+        []
+      end
+
       # A hook that is always called just before any of the instance method of
       # our resource implementations gets invoked, so that we can ensure that
       # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -77,6 +82,11 @@ module MU
         [:resource_group]
       end
 
+      # Is this a "real" cloud provider, or a stub like CloudFormation?
+      def self.virtual?
+        false
+      end
+
       # Stub class to represent Azure's resource identifiers, which look like:
       # /subscriptions/3d20ddd8-4652-4074-adda-0d127ef1f0e0/resourceGroups/mu/providers/Microsoft.Network/virtualNetworks/mu-vnet
       # Various API calls need chunks of this in different contexts, and this
@@ -101,11 +111,11 @@ module MU
         def initialize(*args)
           if args.first.is_a?(String)
             @raw = args.first
-            junk, junk, @subscription, junk, @resource_group, junk, @provider, @resource_type, @name = @raw.split(/\//)
+            _junk, _junk2, @subscription, _junk3, @resource_group, _junk4, @provider, @resource_type, @name = @raw.split(/\//)
             if @subscription.nil? or @resource_group.nil? or @provider.nil? or @resource_type.nil? or @name.nil?
               # Not everything has a resource group
               if @raw.match(/^\/subscriptions\/#{Regexp.quote(@subscription)}\/providers/)
-                junk, junk, @subscription, junk, @provider, @resource_type, @name = @raw.split(/\//)
+                _junk, _junk2, @subscription, _junk3, @provider, @resource_type, @name = @raw.split(/\//)
                 if @subscription.nil? or @provider.nil? or @resource_type.nil? or @name.nil?
                   raise MuError, "Failed to parse Azure resource id string #{@raw} (got subscription: #{@subscription}, provider: #{@provider}, resource_type: #{@resource_type}, name: #{@name}"
                 end
@@ -266,7 +276,7 @@ module MU
         
         begin
           sdk_response = MU::Cloud::Azure.subs(credentials: credentials).subscriptions().list_locations(subscription)
-        rescue Exception => e
+        rescue StandardError => e
           MU.log e.inspect, MU::ERR, details: e.backtrace
           #pp "Error Getting the list of regions from Azure" #TODO: SWITCH THIS TO MU LOG
           if @@regions and @@regions.size > 0
@@ -274,6 +284,9 @@ module MU
           end
           raise e
         end
+        if !sdk_response
+          raise MuError, "Nil response from Azure API attempting list_locations(#{subscription})"
+        end
 
         sdk_response.value.each do | region |
           @@regions.push(region.name)
@@ -378,7 +391,7 @@ module MU
         rg_obj = MU::Cloud::Azure.resources(:ResourceGroup).new
         rg_obj.location = region
         rg_obj.tags = MU::MommaCat.listStandardTags
-        rg_obj.tags.reject! { |k, v| v.nil? }
+        rg_obj.tags.reject! { |_k, v| v.nil? }
 
         MU::Cloud::Azure.resources(credentials: credentials).resource_groups.list.each { |rg|
           if rg.name == name and rg.location == region and rg.tags == rg_obj.tags
@@ -519,7 +532,7 @@ module MU
           end
           return @@metadata
 
-        rescue Timeout::Error => e
+        rescue Timeout::Error
           # MU.log "Timeout querying Azure Metadata"
           return nil
         rescue
@@ -906,7 +919,6 @@ module MU
 # END SDK STUBS
 
 # BEGIN SDK CLIENT
-      private
 
       @@authorization_api = {}
       @@subscriptions_api = {}
@@ -967,7 +979,7 @@ module MU
             begin
               modelpath = "::Azure::#{api}::Mgmt::#{profile}::#{@subclass}"
               @api = Object.const_get(modelpath).new(@cred_obj)
-            rescue NameError => e
+            rescue NameError
               raise MuError, "Unable to locate a profile #{profile} of Azure API #{api}. I tried:\n#{stdpath}\n#{modelpath}"
             end
           end
@@ -1017,6 +1029,7 @@ module MU
           # @param arguments [Array]
           def method_missing(method_sym, *arguments)
             MU.log "Calling #{@parentname}.#{@myname}.#{method_sym.to_s}", MU::DEBUG, details: arguments
+            retries = 0
             begin
               if !arguments.nil? and arguments.size == 1
                 retval = @myobject.method(method_sym).call(arguments[0])
@@ -1025,9 +1038,16 @@ module MU
               else
                 retval = @myobject.method(method_sym).call
               end
-            rescue ::Net::ReadTimeout, ::Faraday::TimeoutError => e
+            rescue ::Net::ReadTimeout, ::Faraday::TimeoutError, ::Faraday::ConnectionFailed => e
               sleep 5
-              retry
+              if retries < 12
+                MU.log e.message+" calling #{@parentname}.#{@myname}.#{method_sym.to_s}(#{arguments.map { |a| a.to_s }.join(", ")})", MU::DEBUG, details: caller
+                retries += 1
+                retry
+              else
+                MU.log e.message+" calling #{@parentname}.#{@myname}.#{method_sym.to_s}(#{arguments.map { |a| a.to_s }.join(", ")})", MU::ERR, details: caller
+                raise e
+              end
             rescue ::MsRestAzure::AzureOperationError, ::MsRest::HttpOperationError => e
               MU.log "Error calling #{@parent.api.class.name}.#{@myname}.#{method_sym.to_s}", MU::DEBUG, details: arguments
               begin

data/modules/mu/{clouds → providers}/azure/container_cluster.rb

@@ -119,7 +119,6 @@ module MU
 
         # Register a description of this cluster instance with this deployment's metadata.
         def notify
-          base = {}
           base = MU.structToHash(cloud_desc)
           base["cloud_id"] = @cloud_id.name
           base.merge!(@config.to_h)
@@ -146,9 +145,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "flavor" => {
@@ -219,11 +218,7 @@ module MU
               "Azure Kubernetes Service Cluster Admin Role"
             ]
           }
-          cluster['dependencies'] ||= []
-          cluster['dependencies'] << {
-            "type" => "user",
-            "name" => cluster["name"]+"user"
-          }
+          MU::Config.addDependency(cluster, cluster['name']+"user", "user")
 
           ok = false if !configurator.insertKitten(svcacct_desc, "users")
 

data/modules/mu/{clouds → providers}/azure/firewall_rule.rb

@@ -53,7 +53,7 @@ module MU
               oldrules[rule.name] = rule
             end
           }
-          used_priorities = oldrules.values.map { |r| r.priority }
+#          used_priorities = oldrules.values.map { |r| r.priority }
 
           newrules_semaphore = Mutex.new
           num_rules = 0
@@ -136,12 +136,12 @@ module MU
             }
           end
 
-          resolved_lbs = []
-          if lbs
-            lbs.each { |lb|
+#          resolved_lbs = []
+#          if lbs
+#            lbs.each { |lb|
 # TODO awaiting LoadBalancer implementation
-            }
-          end
+#            }
+#          end
 
           if egress
             rule_obj.direction = MU::Cloud::Azure.network(:SecurityRuleDirection)::Outbound
@@ -295,7 +295,7 @@ module MU
                 resp = MU::Cloud::Azure.network(credentials: args[:credentials]).network_security_groups.get(rg, id_str)
                 next if resp.nil?
                 found[Id.new(resp.id)] = resp
-              rescue MU::Cloud::Azure::APIError => e
+              rescue MU::Cloud::Azure::APIError
                 # this is fine, we're doing a blind search after all
               end
             }
@@ -336,16 +336,23 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil)
-          bok = {}
+        def toKitten(**args)
+
+          bok = {
+            "cloud" => "Azure",
+            "name" => cloud_desc.name,
+            "project" => @config['project'],
+            "credentials" => @config['credentials'],
+            "cloud_id" => @cloud_id.to_s
+          }
 
           bok
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config = nil)
+        def self.schema(_config = nil)
           toplevel_required = []
           hosts_schema = MU::Config::CIDR_PRIMITIVE
           hosts_schema["pattern"] = "^(\\d+\\.\\d+\\.\\d+\\.\\d+\/[0-9]{1,2}|\\*)$"