cloud-mu 3.0.0beta → 3.0.0

data/modules/mu/clouds/azure/vpc.rb

@@ -494,19 +494,29 @@ module MU
             MU.log "Creating VPC #{@mu_name} (#{@config['ip_block']}) in #{@config['region']}", details: vpc_obj
             need_apply = true
           elsif ext_vpc.location != vpc_obj.location or
-                ext_vpc.tags != vpc_obj.tags or
+#                ext_vpc.tags != vpc_obj.tags or
+#                XXX updating tags is a different API call
                 ext_vpc.address_space.address_prefixes != vpc_obj.address_space.address_prefixes
             MU.log "Updating VPC #{@mu_name} (#{@config['ip_block']}) in #{@config['region']}", MU::NOTICE, details: vpc_obj
+MU.structToHash(ext_vpc).diff(MU.structToHash(vpc_obj))
             need_apply = true
           end
 
           if need_apply
-            resp = MU::Cloud::Azure.network(credentials: @config['credentials']).virtual_networks.create_or_update(
-              @resource_group,
-              @mu_name,
-              vpc_obj
-            )
-            @cloud_id = Id.new(resp.id)
+            begin
+              resp = MU::Cloud::Azure.network(credentials: @config['credentials']).virtual_networks.create_or_update(
+                @resource_group,
+                @mu_name,
+                vpc_obj
+              )
+              @cloud_id = Id.new(resp.id)
+            rescue ::MU::Cloud::Azure::APIError => e
+              if e.message.match(/InUseSubnetCannotBeDeleted: /)
+                MU.log "Cannot delete an in-use Azure subnet", MU::WARN
+              else
+                raise e
+              end
+            end
           end
 
           # this is slow, so maybe thread it
@@ -676,17 +686,26 @@ module MU
                       ext_subnet.network_security_group.nil? and !subnet_obj.network_security_group.nil? or
                       (!ext_subnet.network_security_group.nil? and !subnet_obj.network_security_group.nil? and ext_subnet.network_security_group.id != subnet_obj.network_security_group.id)
                   MU.log "Updating Subnet #{subnet_name} in VPC #{@mu_name}", MU::NOTICE, details: subnet_obj
+MU.structToHash(ext_subnet).diff(MU.structToHash(subnet_obj))
                   need_apply = true
 
                 end
 
                 if need_apply
-                  MU::Cloud::Azure.network(credentials: @config['credentials']).subnets.create_or_update(
-                    @resource_group,
-                    @cloud_id.to_s,
-                    subnet_name,
-                    subnet_obj
-                  )
+                  begin
+                    MU::Cloud::Azure.network(credentials: @config['credentials']).subnets.create_or_update(
+                      @resource_group,
+                      @cloud_id.to_s,
+                      subnet_name,
+                      subnet_obj
+                    )
+                  rescue ::MU::Cloud::Azure::APIError => e
+                    if e.message.match(/InUseSubnetCannotBeUpdated: /)
+                      MU.log "Cannot alter an in-use Azure subnet", MU::WARN
+                    else
+                      raise e
+                    end
+                  end
                 end
               }
             }

data/modules/mu/clouds/cloudformation/server.rb

@@ -304,7 +304,7 @@ module MU
                     role_name: baserole.role_name,
                     policy_name: name
                 )
-                policies[name] = URI.decode_www_form(resp.policy_document)
+                policies[name] = URI.decode(resp.policy_document)
               }
             }
           end

data/modules/mu/clouds/google.rb

@@ -352,8 +352,7 @@ module MU
           )
           f.unlink
         rescue ::Google::Apis::ClientError => e
-# XXX comment for NCBI tests
-#          raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}"
+          raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}"
         end
       end
 
@@ -406,7 +405,7 @@ module MU
 MU.log e.message, MU::WARN, details: e.inspect
           if e.inspect.match(/body: "Not Found"/)
             raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials"
-          elsif e.message.match(/notFound: /)
+          elsif e.message.match(/notFound: |Unknown user:/)
             if retries < 5
               sleep 5
               retries += 1

data/modules/mu/clouds/google/container_cluster.rb

@@ -464,7 +464,7 @@ module MU
             )
           end
 
-          MU.log %Q{How to interact with your Kubernetes cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
+          MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
         end
 
 
@@ -1059,6 +1059,14 @@ module MU
             }
           end
 
+          if cluster['dependencies']
+            cluster['dependencies'].each { |dep|
+              if dep['type'] == "vpc"
+                dep['phase'] = "groom"
+              end
+            }
+          end
+
           if (cluster['pod_ip_block_name'] or cluster['services_ip_block_name']) and
              cluster['custom_subnet']
             MU.log "GKE cluster #{cluster['name']} cannot specify pod_ip_block_name or services_ip_block_name when using a custom subnet", MU::ERR

data/modules/mu/clouds/google/firewall_rule.rb

@@ -423,6 +423,12 @@ end
 
           if acl['vpc']
             acl['vpc']['project'] ||= acl['project']
+            acl['vpc'] = MU::Cloud::Google::VPC.pickVPC(
+              acl['vpc'],
+              acl,
+              "firewall_rule",
+              config
+            )
           end
 
           acl['rules'] ||= []

data/modules/mu/clouds/google/role.rb

@@ -491,12 +491,10 @@ module MU
                     MU::Cloud::Google.admin_directory(credentials: credentials).delete_role(customer, id)
                   end
                 end
-              elsif id.match(/^projects\//)
+              elsif id.match(/^projects\/.*?\/roles\//)
                 begin
                   resp = MU::Cloud::Google.iam(credentials: credentials).get_project_role(id)
                 rescue ::Google::Apis::ClientError => e
-#MU.log e.message, MU::ERR, details: id
-#next
                   next if e.message.match(/notFound/)
                   raise e
                 end

data/modules/mu/clouds/google/server.rb

@@ -242,6 +242,8 @@ next if !create
           subnet_cfg = config['vpc']
           if config['vpc']['subnets'] and
              !subnet_cfg['subnet_name'] and !subnet_cfg['subnet_id']
+            # XXX if illegal subnets somehow creep in here, we'll need to be
+            # picky by region or somesuch
             subnet_cfg = config['vpc']['subnets'].sample
 
           end
@@ -249,6 +251,7 @@ next if !create
           if subnet.nil?
             raise MuError, "Couldn't find subnet details for #{subnet_cfg['subnet_name'] || subnet_cfg['subnet_id']} while configuring Server #{config['name']} (VPC: #{vpc.mu_name})"
           end
+
           base_iface_obj = {
             :network => vpc.url,
             :subnetwork => subnet.url
@@ -268,11 +271,19 @@ next if !create
         def create
           @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
 
-          sa = MU::Config::Ref.get(@config['service_account'])
+          sa = nil
+          retries = 0
+          begin
+            sa = MU::Config::Ref.get(@config['service_account'])
+            if !sa or !sa.kitten or !sa.kitten.cloud_desc
+              sleep 10
+            end
+          end while !sa or !sa.kitten or !sa.kitten.cloud_desc and retries < 5
 
           if !sa or !sa.kitten or !sa.kitten.cloud_desc
             raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
           end
+          
 
           @service_acct = MU::Cloud::Google.compute(:ServiceAccount).new(
             email: sa.kitten.cloud_desc.email,
@@ -344,7 +355,7 @@ next if !create
 
             instanceobj = MU::Cloud::Google.compute(:Instance).new(desc)
 
-            MU.log "Creating instance #{@mu_name}", MU::NOTICE, details: instanceobj
+            MU.log "Creating instance #{@mu_name} in #{@project_id} #{@config['availability_zone']}", details: instanceobj
 
             begin
               instance = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_instance(
@@ -409,8 +420,10 @@ next if !create
           return {
             "cloud" => "Google",
             "size" => "g1-small",
-            "run_list" => [ "mu-utility::nat" ],
+            "run_list" => [ "mu-nat" ],
+            "groomer" => "Ansible",
             "platform" => "centos7",
+            "src_dst_check" => false,
             "ssh_user" => "centos",
             "associate_public_ip" => true,
             "static_ip" => { "assign_ip" => true },
@@ -1339,7 +1352,10 @@ next if !create
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "Google"
               cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes)[cloudbase.myRegion]
+              foreign_types = (cloudbase.listInstanceTypes).values.first
+              if foreign_types.size == 1
+                foreign_types = foreign_types.values.first
+              end
               if foreign_types and foreign_types.size > 0 and foreign_types.has_key?(size)
                 vcpu = foreign_types[size]["vcpu"]
                 mem = foreign_types[size]["memory"]
@@ -1378,6 +1394,7 @@ next if !create
           ok = true
 
           server['project'] ||= MU::Cloud::Google.defaultProject(server['credentials'])
+
           size = validateInstanceType(server["size"], server["region"], project: server['project'], credentials: server['credentials'])
 
           if size.nil?
@@ -1458,6 +1475,10 @@ next if !create
             end
           end
 
+          if server['vpc']
+            server['vpc']['project'] ||= server['project']
+          end
+
           if server['image_id'].nil?
             img_id = MU::Cloud.getStockImage("Google", platform: server['platform'])
             if img_id

data/modules/mu/clouds/google/user.rb

@@ -199,7 +199,7 @@ module MU
             end
           else
             @config['type'] ||= "service"
-            return MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
+            MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
           end
 
         end

data/modules/mu/clouds/google/userdata/linux.erb

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+# Copyright:: Copyright (c) 2017 eGlobalTech, Inc., all rights reserved
 #
 # Licensed under the BSD-3 license (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,13 @@
 updates_run=0
 need_reboot=0
 instance_id="`curl http://metadata.google.internal/computeMetadata/v1/instance/name`"
+
+for f in /etc/rc.local /etc/rc.d/rc.local;do
+  if [ -f $f ];then
+    chmod 755 $f
+  fi
+done
+
 if [ -f /etc/debian_version ];then
 	if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
 		echo "/bin/sh /var/lib/cloud/instance/user-data.txt" >> /etc/rc.local
@@ -65,9 +72,7 @@ elif [ -x /usr/bin/yum ];then
 
   sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
 
-	if [ $version == 7 ];then
-		chmod 755 /etc/rc.d/rc.local
-	fi
+	chmod 755 /etc/rc.d/rc.local
 	if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi
 	# Ugh, rando EPEL mirror
 	if [ ! -f /etc/yum.repos.d/epel.repo ];then
@@ -127,7 +132,6 @@ print Base64.urlsafe_encode64(key.public_encrypt(File.read("<%= $mu.muID %>-secr
 ' > encrypt_deploy_secret.rb
 
 deploykey="<%= $mu.deployKey %>"
-instance_id="`curl http://metadata.google.internal/computeMetadata/v1/instance/name`"
 
 # Make double-sure sshd is actually up
 service sshd restart

data/modules/mu/clouds/google/vpc.rb

@@ -147,7 +147,7 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
 
-          rtb = @config['route_tables'].first
+          rtb = @config['route_tables'].first # there's only ever one
 
           rtb['routes'].each { |route|
             # If we had a sibling server being spun up as a NAT, rig up the 
@@ -236,7 +236,7 @@ end
           resp = {}
           if args[:cloud_id] and args[:project]
             begin
-            vpc = MU::Cloud::Google.compute(credentials: args[:credentials]).get_network(
+              vpc = MU::Cloud::Google.compute(credentials: args[:credentials]).get_network(
               args[:project],
               args[:cloud_id].to_s.sub(/^.*?\/([^\/]+)$/, '\1')
             )
@@ -255,7 +255,7 @@ end
 
             if vpcs and vpcs.items
               vpcs.items.each { |v|
-                resp[vpc.name] = v
+                resp[v.name] = v
               }
             end
           end
@@ -405,36 +405,48 @@ end
             dummy_ok: true,
             calling_deploy: @deploy
           )
-# XXX wat
+
           return nil if found.nil? || found.empty?
-          if found.size > 1
+
+          if found.size == 1
+            return found.first
+          elsif found.size > 1
             found.each { |nat|
+              next if !nat.cloud_desc
               # Try some cloud-specific criteria
-              cloud_desc = nat.cloud_desc
-              if !nat_host_ip.nil? and
-# XXX this is AWS code, is wrong here
-                  (cloud_desc.private_ip_address == nat_host_ip or cloud_desc.public_ip_address == nat_host_ip)
-                return nat
-              elsif cloud_desc.vpc_id == @cloud_id
-                # XXX Strictly speaking we could have different NATs in
-                # different subnets, so this can be wrong in corner cases.
-                return nat
-              end
+              nat.cloud_desc.network_interfaces.each { |iface|
+                if !nat_ip.nil?
+                  return nat if iface.network_ip == nat_ip
+                  if iface.access_configs
+                    iface.access_configs.each { |public_iface|
+                      return if public_iface.nat_ip == nat_ip
+                    }
+                  end
+                end
+                if iface.network == @url
+                  # XXX Strictly speaking we could have different NATs in
+                  # different subnets, so this can be wrong in corner cases.
+                  return nat
+                end
+              }
             }
-          elsif found.size == 1
-            return found.first
           end
+
           return nil
         end
 
         # Check for a subnet in this VPC matching one or more of the specified
         # criteria, and return it if found.
-        def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil)
+        def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil, region: nil)
           if !cloud_id.nil? and cloud_id.match(/^https:\/\//)
+            cloud_id.match(/\/regions\/([^\/]+)\/subnetworks\/([^\/]+)$/)
+            region = Regexp.last_match[1]
+            cloud_id = Regexp.last_match[2]
             cloud_id.gsub!(/.*?\//, "")
           end
-          MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block})", MU::DEBUG, details: caller[0]
+          MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region})", MU::DEBUG, details: caller[0]
           subnets.each { |subnet|
+            next if region and subnet.az != region
             if !cloud_id.nil? and !subnet.cloud_id.nil? and subnet.cloud_id.to_s == cloud_id.to_s
               return subnet
             elsif !name.nil? and !subnet.name.nil? and
@@ -694,6 +706,68 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           [toplevel_required, schema]
         end
 
+        # If the VPC a config block was set to one that's been "split," try to
+        # figure out which of the new VPCs we really want to be in. For use by
+        # resource types that don't go in subnets, but do tie to VPCs.
+        # @param vpc_block [Hash]
+        # @param configurator [MU::Config]
+        # @return [Hash]
+        def self.pickVPC(vpc_block, my_config, my_type, configurator)
+          _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(my_type)
+          return if vpc_block.nil?
+          vpc_block['name'] ||= vpc_block['vpc_name']
+          return if !vpc_block['name']
+
+          vpcs = configurator.haveLitterMate?(
+            nil,
+            "vpcs",
+            has_multiple: true
+          )
+          # drop all virtual vpcs that aren't real anymore
+          vpcs.reject! { |v| v['virtual_name'] == v['name'] }
+          # drop the ones that have nothing to do with us
+          vpcs.reject! { |v| v['virtual_name'] != vpc_block['name'] }
+
+          return vpc_block if vpcs.size == 0
+
+          # see if one of this thing's siblings declared a subnet_pref we can
+          # use to guess which one we should marry ourselves to
+          configurator.kittens.each_pair { |type, siblings|
+            siblings.each { |sibling|
+              next if !sibling['dependencies']
+              sibling['dependencies'].each { |dep|
+                if [cfg_name, cfg_plural].include?(dep['type']) and
+                   dep['name'] == my_config['name']
+                  vpcs.each { |v|
+                    if sibling['vpc']['name'] == v['name']
+                      vpc_block['name'] = v['name']
+                      return vpc_block
+                    end
+                  }
+                  if sibling['vpc']['subnet_pref']
+                    vpcs.each { |v|
+                      gateways = v['route_tables'].map { |rtb|
+                        rtb['routes'].map { |r| r["gateway"] }
+                      }.flatten.uniq
+                      if ["public", "all_public"].include?(sibling['vpc']['subnet_pref']) and
+                         gateways.include?("#INTERNET")
+                        vpc_block['name'] = v['name']
+                        return vpc_block
+                      elsif ["private", "all_private"].include?(sibling['vpc']['subnet_pref']) and
+                         !gateways.include?("#INTERNET")
+                        vpc_block['name'] = v['name']
+                        return vpc_block
+                      end
+                    }
+
+                  end
+                end
+              }
+            }
+          }
+
+          vpc_block
+        end
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::vpcs}, bare and unvalidated.
         # @param vpc [Hash]: The resource to process and validate
@@ -720,7 +794,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             vpc['route_tables'].each { |t|
               is_public = false
               t['routes'].each { |r|
-                if !vpc["virtual_name"] and !vpc["create_nat_gateway"] and
+                if !vpc["virtual_name"] and
+                   !vpc["create_nat_gateway"] and
+                   !vpc['bastion'] and
                    r["gateway"] == "#NAT"
                   r["gateway"] = "#DENY"
                 end
@@ -781,6 +857,11 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                 next if ["name", "route_tables", "subnets", "ip_block"].include?(key)
                 newvpc[key] = val
               }
+              if vpc["bastion"] and
+                 !tbl["routes"].map { |r| r["gateway"] }.include?("#INTERNET")
+                newvpc["bastion"] = vpc["bastion"]
+                vpc.delete("bastion")
+              end
               newvpc['peers'] ||= []
 # Add the peer connections we're generating, in addition 
               peernames.each { |peer|
@@ -809,7 +890,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             else
               ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], credentials: vpc['credentials'])
             end
-            if has_nat and !has_deny
+            if has_nat and !has_deny and !vpc['bastion']
               vpc['route_tables'].first["routes"] << {
                 "gateway"=>"#DENY",
                 "destination_network"=>"0.0.0.0/0"