cloud-mu 3.3.0 → 3.5.1

data/modules/mu/providers/google/folder.rb

@@ -265,8 +265,12 @@ module MU
 
           if args[:cloud_id]
             raw_id = args[:cloud_id].sub(/^folders\//, "")
-            resp = MU::Cloud::Google.folder(credentials: args[:credentials]).get_folder("folders/"+raw_id)
-            found[resp.name] = resp if resp
+            begin
+              resp = MU::Cloud::Google.folder(credentials: args[:credentials]).get_folder("folders/"+raw_id)
+              found[resp.name] = resp if resp
+            rescue ::Google::Apis::ClientError => e
+              raise e if e.message !~ /forbidden: /
+            end
 
           elsif args[:flags] and args[:flags]['display_name']
 

data/modules/mu/providers/google/function.rb

@@ -119,6 +119,9 @@ module example.com/cloudfunction
         # Called automatically by {MU::Deploy#createResources}
         def groom
           desc = {}
+
+          func_obj = buildDesc
+
           labels = Hash[@tags.keys.map { |k|
             [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
           ]
@@ -140,6 +143,10 @@ module example.com/cloudfunction
           if cloud_desc.available_memory_mb != @config['memory']
             need_update = true
           end
+          if cloud_desc.service_account_email != func_obj.service_account_email
+            need_update = true
+          end
+
           if @config['environment_variable']
             @config['environment_variable'].each { |var|
               if !cloud_desc.environment_variables or
@@ -161,7 +168,17 @@ module example.com/cloudfunction
             File.read("#{dir}/current.zip")
           }
 
-          new = if @config['code']['zip_file']
+          tempfile = nil
+          new = if @config['code']['zip_file'] or @config['code']['path']
+            if @config['code']['path']
+              tempfile = Tempfile.new(["function", ".zip"])
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
+#            @code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp
             File.read(@config['code']['zip_file'])
           elsif @config['code']['gs_url']
             @config['code']['gs_url'].match(/^gs:\/\/([^\/]+)\/(.*)/)
@@ -172,25 +189,31 @@ module example.com/cloudfunction
               File.read(dir+"/new.zip")
             }
           end
+
           if @config['code']['gs_url'] and
              (@config['code']['gs_url'] != cloud_desc.source_archive_url or
              current != new)
             need_update = true
-          elsif @config['code']['zip_file'] and current != new
+          elsif (@config['code']['zip_file'] or @config['code']['path']) and current != new
             need_update = true
-            desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
+          end
+
+          if @config['vpc_connector']
+            if cloud_desc.vpc_connector != @config['vpc_connector'] or
+               cloud_desc.vpc_connector_egress_settings != (@config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY")
+              need_update = true
+            end
           end
 
           if need_update
-            func_obj = buildDesc
-            MU.log "Updating Cloud Function #{@mu_name}", MU::NOTICE, details: func_obj
+            MU.log "Updating Cloud Function #{@cloud_id}", MU::NOTICE, details: func_obj
             begin
-#              MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
-#                @cloud_id, 
-#                func_obj
-#              )
-            rescue ::Google::Apis::ClientError
-              MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR
+              MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
+                @cloud_id, 
+                func_obj
+              )
+            rescue ::Google::Apis::ClientError => e
+              MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR, e.message
               if desc[:source_archive_url]
                 main_file = nil
                 HELLO_WORLDS.each_pair { |runtime, code|
@@ -207,6 +230,11 @@ module example.com/cloudfunction
 #            service_account_email: sa.kitten.cloud_desc.email,
 #            labels: labels,
 
+          if tempfile
+            tempfile.close
+            tempfile.unlink
+          end
+
         end
 
         # Return the metadata for this project's configuration
@@ -354,6 +382,7 @@ module example.com/cloudfunction
         def self.schema(config)
           toplevel_required = ["runtime"]
           schema = {
+            "roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
             "triggers" => {
               "type" => "array",
               "items" => {
@@ -448,6 +477,7 @@ module example.com/cloudfunction
             content_type: "application/zip",
             name: filename
           )
+
           MU::Cloud::Google.storage(credentials: credentials).insert_object(
             bucket,
             obj_obj,
@@ -487,7 +517,7 @@ module example.com/cloudfunction
           end
 # XXX list_project_locations
 
-          if !function['code'] or (!function['code']['zip_file'] and !function['code']['gs_url'])
+          if !function['code'] or (!function['code']['zip_file'] and !function['code']['gs_url'] and !function['code']['path'])
             MU.log "Must specify a code source in Cloud Function #{function['name']}", MU::ERR
             ok = false
           elsif function['code']['zip_file']
@@ -557,22 +587,14 @@ module example.com/cloudfunction
 
           location = "projects/"+@config['project']+"/locations/"+@config['region']
           sa = nil
-          retries = 0
-          begin
-            sa_ref = MU::Config::Ref.get(@config['service_account'])
-            sa = @deploy.findLitterMate(name: sa_ref.name, type: "users")
-            if !sa or !sa.cloud_desc
-              sleep 10
-            end
-          rescue ::Google::Apis::ClientError => e
-            if e.message.match(/notFound:/)
-              sleep 10
-              retries += 1
-              retry
-            end
-          end while !sa or !sa.cloud_desc and retries < 5
+          need_sa = Proc.new {
+            !sa or !sa.kitten or !sa.kitten.cloud_desc
+          }
+          MU.retrier(loop_if: need_sa, wait: 10, max: 6) { |retries, _wait|
+            sa = MU::Config::Ref.get(@config['service_account'])
+          }
 
-          if !sa or !sa.cloud_desc
+          if need_sa.call()
             raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
           end
 
@@ -583,7 +605,7 @@ module example.com/cloudfunction
 #            entry_point: "hello_world",
             entry_point: @config['handler'],
             description: @deploy.deploy_id,
-            service_account_email: sa.cloud_desc.email,
+            service_account_email: sa.kitten.cloud_desc.email,
             labels: labels,
             available_memory_mb: @config['memory']
           }
@@ -596,7 +618,6 @@ module example.com/cloudfunction
           if @config['vpc_connector']
             desc[:vpc_connector] = @config['vpc_connector']
             desc[:vpc_connector_egress_settings] = @config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY"
-            pp desc
           elsif @vpc
             desc[:network] = @vpc.url.sub(/^.*?\/projects\//, 'projects/')
           end
@@ -627,8 +648,22 @@ module example.com/cloudfunction
 #          }
           if @config['code']['gs_url']
             desc[:source_archive_url] = @config['code']['gs_url']
-          elsif @config['code']['zip_file']
+          elsif @config['code']['zip_file'] or @config['code']['path']
+            tempfile = nil
+            if @config['code']['path']
+              tempfile = Tempfile.new(["function", ".zip"])
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
             desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
+
+            if tempfile
+              tempfile.close
+              tempfile.unlink
+            end
           end
 
 #          Dir.mktmpdir(@mu_name) { |dir|

data/modules/mu/providers/google/role.rb

@@ -581,7 +581,7 @@ module MU
               }
             end
             if args[:cloud_id]
-              found.reject! { |k, _v| k != role.name }
+              found.reject! { |k, _v| k != args[:cloud_id] }
             end
 
             # Now go get everything that's bound here
@@ -745,6 +745,7 @@ module MU
                   end
 
                   entity_types.each_pair { |entity_type, entities|
+                    next if entity_type == "deleted"
                     mu_entitytype = (entity_type == "serviceAccount" ? "user" : entity_type)+"s"
                     entities.each { |entity|
                       next if entity.nil?

data/modules/mu/providers/google/vpc.rb

@@ -364,6 +364,12 @@ end
             }
           end
 
+
+          # The API is filled with lies
+          @subnets.reject! { |s|
+            !MU::Cloud::Google.listRegions(credentials: @credentials).include?(s.az)
+          }
+
           return @subnets
         end
 
@@ -442,14 +448,19 @@ end
 
         # Check for a subnet in this VPC matching one or more of the specified
         # criteria, and return it if found.
-        def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil, region: nil)
+        def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil, region: nil, subnet_mu_name: nil)
           if !cloud_id.nil? and cloud_id.match(/^https:\/\//)
             cloud_id.match(/\/regions\/([^\/]+)\/subnetworks\/([^\/]+)$/)
             region = Regexp.last_match[1]
             cloud_id = Regexp.last_match[2]
             cloud_id.gsub!(/.*?\//, "")
           end
-          MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region})", MU::DEBUG, details: caller[0]
+          
+          if name
+            subnet_mu_name ||= @config['scrub_mu_isms'] ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
+          end
+
+          MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region}, subnet_mu_name: #{subnet_mu_name})", MU::DEBUG, details: caller[0]
           subnets.each { |subnet|
             next if region and subnet.az != region
             if !cloud_id.nil? and !subnet.cloud_id.nil? and subnet.cloud_id.to_s == cloud_id.to_s
@@ -457,6 +468,9 @@ end
             elsif !name.nil? and !subnet.name.nil? and
                   subnet.name.downcase.to_s == name.downcase.to_s
               return subnet
+            elsif !subnet_mu_name.nil? and !subnet.name.nil? and
+                  subnet.name.downcase.to_s == subnet_mu_name.downcase.to_s
+              return subnet
             end
           }
           return nil
@@ -931,6 +945,14 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                 else
                   route['nat_host_name'] = nat['name']
                   route['priority'] = 100
+                  MU::Config.addDependency(vpc, nat['name'], "server", their_phase: "groom", my_phase: "groom")
+                  vpc["bastion"] = MU::Config::Ref.get(
+                    name: nat['name'],
+                    cloud: vpc['cloud'],
+                    credentials: vpc['credentials'],
+                    type: "servers"
+                  )
+
                 end
               end
             }
@@ -1172,6 +1194,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
               if e.message.match(/notFound: /)
                 MU.log "Failed to fetch cloud description for Google subnet #{@cloud_id}", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
                 return nil
+              elsif e.message.match(/Unknown region\. /)
+                MU.log "Google subnet #{@cloud_id} seems like it should live in #{@az}, but that's not a valid region", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
+                return nil
               else
                 raise e
               end

data/modules/tests/aws-servers-with-handrolled-iam.yaml

@@ -0,0 +1,37 @@
+# clouds: AWS
+# groomers: Chef
+---
+appname: smoketest
+vpcs:
+- name: svrtest
+roles:
+- name: handrolled
+  scrub_mu_isms: true
+  can_assume:
+  - entity_id: ec2.amazonaws.com
+    entity_type: service
+  import:
+  - arn:aws:iam::aws:policy/AmazonRDSFullAccess
+servers:
+- name: iamtest1
+  size: t3.medium
+  iam_role: handrolled
+  platform: centos6
+  generate_iam_role: false
+  vpc:
+    name: svrtest
+server_pools:
+- name: iamtest2
+  scrub_mu_isms: true
+  min_size: 1
+  max_size: 1
+  wait_for_nodes: 1
+  platform: centos6
+  vpc:
+    name: svrtest
+  basis:
+    launch-config:
+      name: iamtest2
+      size: t3.medium
+      iam_role: handrolled
+      generate_iam_role: false

data/modules/tests/k8s.yaml

@@ -27,7 +27,7 @@ container_clusters:
     comment: meep
   vpc:
     vpc_name: k8s
-    subnet_pref: all_private
+    subnet_pref: all_public
   kubernetes_resources:
   - apiVersion: v1
     kind: Service

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-mu
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.5.1
 platform: ruby
 authors:
 - John Stange
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-22 00:00:00.000000000 Z
+date: 2021-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -28,47 +28,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.5'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: azure_sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.65'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.65'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: 2.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: chronic_duration
   requirement: !ruby/object:Gem::Requirement
@@ -131,14 +131,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.36.4
+        version: 0.50.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.36.4
+        version: 0.50.0
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -448,6 +448,7 @@ executables:
 - mu-gen-docs
 - mu-tunnel-nagios
 - mu-ssh
+- mu-refresh-ssl
 - mu-gen-env
 - mu-configure
 - mu-momma-cat
@@ -524,6 +525,7 @@ files:
 - bin/mu-load-config.rb
 - bin/mu-momma-cat
 - bin/mu-node-manage
+- bin/mu-refresh-ssl
 - bin/mu-run-tests
 - bin/mu-self-update
 - bin/mu-ssh
@@ -663,8 +665,10 @@ files:
 - cookbooks/mu-master/templates/default/389-directory-setup.inf.erb
 - cookbooks/mu-master/templates/default/chef-server.rb.erb
 - cookbooks/mu-master/templates/default/dhclient-eth0.conf.erb
+- cookbooks/mu-master/templates/default/mods/rewrite.conf.erb
 - cookbooks/mu-master/templates/default/mu-momma-cat.erb
 - cookbooks/mu-master/templates/default/mu.rc.erb
+- cookbooks/mu-master/templates/default/nagios.conf.erb
 - cookbooks/mu-master/templates/default/openssl.cnf.erb
 - cookbooks/mu-master/templates/default/sssd.conf.erb
 - cookbooks/mu-master/templates/default/web_app.conf.erb
@@ -736,6 +740,7 @@ files:
 - cookbooks/mu-tools/attributes/default.rb
 - cookbooks/mu-tools/attributes/ebs_rolling_snapshots.rb
 - cookbooks/mu-tools/files/amazon/etc/freshclam.conf
+- cookbooks/mu-tools/files/centos-6/CentOS-Base.repo
 - cookbooks/mu-tools/files/centos-6/README_MU
 - cookbooks/mu-tools/files/centos-6/etc/audit/stig.rules
 - cookbooks/mu-tools/files/centos-6/etc/bashrc
@@ -841,6 +846,7 @@ files:
 - cookbooks/mu-tools/templates/amazon/sshd_config.erb
 - cookbooks/mu-tools/templates/centos-6/sshd_config.erb
 - cookbooks/mu-tools/templates/centos-7/sshd_config.erb
+- cookbooks/mu-tools/templates/centos-8/sshd_config.erb
 - cookbooks/mu-tools/templates/default/0-mu-log-client.conf.erb
 - cookbooks/mu-tools/templates/default/conf.maldet.erb
 - cookbooks/mu-tools/templates/default/etc_hosts.erb
@@ -907,34 +913,44 @@ files:
 - environments/dev.json
 - environments/development.json
 - environments/prod.json
+- extras/Gemfile.lock.bootstrap
 - extras/README.md
 - extras/admin-role-binding.yaml
 - extras/admin-user.yaml
 - extras/alpha.png
 - extras/aws-auth-cm.yaml.erb
 - extras/beta.png
+- extras/bucketstubs/error.html
+- extras/bucketstubs/index.html
 - extras/clean-stock-amis
 - extras/generate-stock-images
 - extras/git-fix-permissions-hook
+- extras/git_rpm/build.sh
+- extras/git_rpm/mugit.spec
 - extras/gitlab-eks-helper.sh.erb
 - extras/image-generators/AWS/centos6.yaml
 - extras/image-generators/AWS/centos7-govcloud.yaml
 - extras/image-generators/AWS/centos7.yaml
-- extras/image-generators/AWS/rhel7.yaml
-- extras/image-generators/AWS/win2k12.yaml
+- extras/image-generators/AWS/rhel71.yaml
+- extras/image-generators/AWS/win2k12r2.yaml
 - extras/image-generators/AWS/win2k16.yaml
 - extras/image-generators/AWS/win2k19.yaml
 - extras/image-generators/Google/centos6.yaml
 - extras/image-generators/Google/centos7.yaml
 - extras/image-generators/README.md
+- extras/image-generators/VMWare/centos8.yaml
 - extras/lambda_waf_domain_blacklist.py
 - extras/list-stock-amis
+- extras/openssl_rpm/build.sh
+- extras/openssl_rpm/mussl.spec
 - extras/platform_berksfile_base
 - extras/python_rpm/build.sh
 - extras/python_rpm/muthon.spec
 - extras/release.png
 - extras/ruby_rpm/build.sh
 - extras/ruby_rpm/muby.spec
+- extras/sqlite_rpm/build.sh
+- extras/sqlite_rpm/muqlite.spec
 - extras/vault_tools/README.md
 - extras/vault_tools/export_vaults.sh
 - extras/vault_tools/recreate_vaults.sh
@@ -1114,6 +1130,7 @@ files:
 - modules/tests/auto_scaling.inc
 - modules/tests/aws-iam.yaml
 - modules/tests/aws-jobs-functions.yaml
+- modules/tests/aws-servers-with-handrolled-iam.yaml
 - modules/tests/aws-sgs.yaml
 - modules/tests/bucket.yml
 - modules/tests/centos6.yaml