cloud-mu 3.0.1 → 3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0ab46aed6ed1aaf58e3d8de76de9ea27adc49cbd754b51cc4ef7e6354a78a26
-  data.tar.gz: 5b341646d7533692813097af86fb37c46f1ba23fb0faf6acfe09688a3456e13c
+  metadata.gz: 1d41e45463915304c58cf78023563876314b9877087d44263e0ad3360ab6aea1
+  data.tar.gz: e3926bc8af2bb1cd7d6e2f32fe84e5de21b6073a7e7eda1be908a626f08becd4
 SHA512:
-  metadata.gz: e9c62154b428aa93f247fbf61589372744a167814f0403a339640e1e96042c8ece774b57617127860e7cfa46dc95c0c666022c3872beac77987f496cea65ad00
-  data.tar.gz: 12a4ced1bb6cbfe64ef666699a34bafe1aaf0228d080d6f8ee3584bcaec967084eca3d3a4167a405e05d1f8bc0d171093f7bab46e5648d1ff3339750a36c60e8
+  metadata.gz: 745bd6c7016676750b1306c324eca784821d026a6a255f4e0e33e09d5ca2843501ce3a774fa5cc9dc1b0e6a53270bce0ac79963fe25c1031d7ead2ddb1263c98
+  data.tar.gz: d0bf047ad4aa61a160368daa0a4897fa2904d183f1e478207e93c6e3cacecc1833dab04e9b3d634414298bee5a38ad0987a09e2226df79008b52fc67bac9cc95

data/bin/mu-configure

@@ -107,6 +107,12 @@ $CONFIGURABLES = {
     "desc" => "The local system's value for HOSTNAME",
     "changes" => ["chefrun", "hostname"]
   },
+  "disable_mommacat" => {
+    "title" => "Disable Momma Cat",
+    "default" => false,
+    "desc" => "Disable the Momma Cat grooming daemon. Nodes which require asynchronous Ansible/Chef bootstraps will not function. This option is only honored in gem-based installations.",
+    "boolean" => true
+  },
   "mommacat_port" => {
     "title" => "Momma Cat Listen Port",
     "pattern" => /^[0-9]+$/i,
@@ -804,7 +810,7 @@ if !$NOOP
   # Converts the current $CONFIGURABLES object to a Hash suitable for merging
   # with $MU_CFG.
   def setConfigTree(tree = $CONFIGURABLES)
-    cfg = {}
+    cfg = $MU_CFG.nil? ? {} : $MU_CFG.dup
     tree.each_pair { |key, data|
       next if !AMROOT and data['rootonly']
       if data.has_key?("subtree")

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.0.1'
-  s.date        = '2019-11-22'
+  s.version     = '3.0.2'
+  s.date        = '2019-12-03'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -59,4 +59,6 @@ EOF
   s.add_runtime_dependency 'addressable', '~> 2.5'
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
   s.add_runtime_dependency 'azure_sdk', "~> 0.37"
+  s.add_runtime_dependency 'rack', "~> 2.0"
+  s.add_runtime_dependency 'thin', "~> 1.7"
 end

data/modules/mu/cloud.rb

@@ -1381,8 +1381,16 @@ module MU
             if !@config['vpc']["id"].nil? and @config['vpc']["id"].is_a?(Hash)
               @config['vpc']["id"] = MU::Config::Ref.new(@config['vpc']["id"])
             end
-            if !@config['vpc']["id"].nil? and @config['vpc']["id"].is_a?(MU::Config::Ref) and !@config['vpc']["id"].kitten.nil?
-              @vpc = @config['vpc']["id"].kitten
+            if !@config['vpc']["id"].nil?
+              if @config['vpc']["id"].is_a?(MU::Config::Ref) and !@config['vpc']["id"].kitten.nil?
+                @vpc = @config['vpc']["id"].kitten
+              else
+                if @config['vpc']['habitat']
+                  @config['vpc']['habitat'] = MU::Config::Ref.get(@config['vpc']['habitat'])
+                end
+                vpc_ref = MU::Config::Ref.get(@config['vpc'])
+                @vpc = vpc_ref.kitten
+              end
             elsif !@config['vpc']["name"].nil? and @deploy
               MU.log "Attempting findLitterMate on VPC for #{self}", loglevel, details: @config['vpc']
 
@@ -2054,6 +2062,15 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
               cloudclass.cleanup(params)
             rescue MuCloudResourceNotImplemented
               MU.log "No #{cloud} implementation of #{shortname}.cleanup, skipping", MU::DEBUG, details: flags
+            rescue Exception => e
+              in_msg = cloud
+              if params and params[:region]
+                in_msg += " "+params[:region]
+              end
+              if params and params[:flags] and params[:flags]["project"]
+                in_msg += " project "+params[:flags]["project"]
+              end
+              MU.log "Skipping #{shortname} cleanup method in #{in_msg} due to exception: #{e.message}", MU::WARN
             end
           }
           MU::MommaCat.unlockAll

data/modules/mu/clouds/google/container_cluster.rb

@@ -1044,6 +1044,9 @@ module MU
               "credentials" => cluster["credentials"],
               "type" => "service"
             }
+            if user["name"].length < 6
+              user["name"] += Password.pronounceable(6)
+            end
             configurator.insertKitten(user, "users", true)
             cluster['dependencies'] ||= []
             cluster['service_account'] = MU::Config::Ref.get(

data/modules/mu/clouds/google/firewall_rule.rb

@@ -45,8 +45,17 @@ module MU
         def create
           @cloud_id = @mu_name.downcase.gsub(/[^-a-z0-9]/, "-")
 
-          vpc_id = @vpc.url if !@vpc.nil?
-          vpc_id ||= @config['vpc']['vpc_id'] if @config['vpc'] and @config['vpc']['vpc_id']
+          vpc_id = if @vpc
+            @vpc.url if !@vpc.nil?
+          else
+            vpc_ref = MU::Config::Ref.get(@config['vpc'])
+            if !vpc_ref.kitten
+              MU.log "Failed to resolve VPC for FirewallRule #{@mu_name}", MU::ERR, details: @config['vpc']
+              raise MuError, "Failed to resolve VPC for FirewallRule #{@mu_name}"
+            else
+              vpc_ref.kitten.url
+            end
+          end
 
           if vpc_id.nil?
             raise MuError, "Failed to resolve VPC for #{self}"
@@ -419,16 +428,22 @@ end
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(acl, config)
           ok = true
-          acl['project'] ||= MU::Cloud::Google.defaultProject(acl['credentials'])
 
           if acl['vpc']
-            acl['vpc']['project'] ||= acl['project']
-            acl['vpc'] = MU::Cloud::Google::VPC.pickVPC(
+            if !acl['vpc']['habitat']
+              acl['vpc']['project'] ||= acl['project']
+            elsif acl['vpc']['habitat'] and acl['vpc']['habitat']['id']
+              acl['vpc']['project'] = acl['vpc']['habitat']['id']
+            elsif acl['vpc']['habitat'] and acl['vpc']['habitat']['name']
+              acl['vpc']['project'] = acl['vpc']['habitat']['name']
+            end
+            correct_vpc = MU::Cloud::Google::VPC.pickVPC(
               acl['vpc'],
               acl,
               "firewall_rule",
               config
             )
+            acl['vpc'] = correct_vpc if correct_vpc
           end
 
           acl['rules'] ||= []

data/modules/mu/clouds/google/server.rb

@@ -1424,6 +1424,9 @@ next if !create
               "credentials" => server["credentials"],
               "type" => "service"
             }
+            if user["name"].length < 6
+              user["name"] += Password.pronounceable(6)
+            end
             if server['roles']
               user['roles'] = server['roles'].dup
             end
@@ -1432,13 +1435,13 @@ next if !create
             server['service_account'] = MU::Config::Ref.get(
               type: "users",
               cloud: "Google",
-              name: server["name"],
-              project: server["project"],
-              credentials: server["credentials"]
+              name: user["name"],
+              project: user["project"],
+              credentials: user["credentials"]
             )
             server['dependencies'] << {
               "type" => "user",
-              "name" => server["name"]
+              "name" => user["name"]
             }
           end
 

data/modules/mu/clouds/google/server_pool.rb

@@ -380,6 +380,9 @@ start = Time.now
               "credentials" => pool["credentials"],
               "type" => "service"
             }
+            if user["name"].length < 6
+              user["name"] += Password.pronounceable(6)
+            end
             configurator.insertKitten(user, "users", true)
             pool['dependencies'] ||= []
             pool['service_account'] = MU::Config::Ref.get(

data/modules/mu/clouds/google.rb

@@ -165,8 +165,8 @@ module MU
 
         # blow up if this resource *has* to live in a project
         if cloudobj.cloudclass.canLiveIn == [:Habitat]
-          MU.log "Failed to find project for cloudobj of class #{cloudobj.cloudclass.class.name}", MU::ERR, details: cloudobj
-          raise MuError, "Failed to find project for cloudobj of class #{cloudobj.cloudclass.class.name}"
+          MU.log "Failed to find project for cloudobj #{cloudobj.to_s}", MU::ERR, details: cloudobj
+          raise MuError, "Failed to find project for cloudobj #{cloudobj.to_s}"
         end
 
         nil
@@ -816,6 +816,10 @@ MU.log e.message, MU::WARN, details: e.inspect
       def self.admin_directory(subclass = nil, credentials: nil)
         require 'google/apis/admin_directory_v1'
 
+        # fill in the default credential set name so we don't generate
+        # dopey extra warnings about falling back on scopes
+        credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true)
+
         writescopes = ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema']
         readscopes = ['admin.directory.group.member.readonly', 'admin.directory.group.readonly', 'admin.directory.user.readonly', 'admin.directory.domain.readonly', 'admin.directory.orgunit.readonly', 'admin.directory.rolemanagement.readonly', 'admin.directory.customer.readonly', 'admin.directory.user.alias.readonly', 'admin.directory.userschema.readonly']
         @@readonly_semaphore.synchronize {
@@ -826,7 +830,7 @@ MU.log e.message, MU::WARN, details: e.inspect
 
           if subclass.nil?
             begin
-              @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
+              @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
             rescue Signet::AuthorizationError => e
               MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
               @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
@@ -1035,7 +1039,7 @@ MU.log e.message, MU::WARN, details: e.inspect
         # Create a Google Cloud Platform API client
         # @param api [String]: Which API are we wrapping?
         # @param scopes [Array<String>]: Google auth scopes applicable to this API
-        def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil)
+        def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false)
           @credentials = credentials
           @scopes = scopes.map { |s|
             if !s.match(/\//) # allow callers to use shorthand
@@ -1052,11 +1056,16 @@ MU.log e.message, MU::WARN, details: e.inspect
               @api.authorization.sub = @masquerade
               @api.authorization.fetch_access_token!
             rescue Signet::AuthorizationError => e
-              MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::ERROR, details: @scopes
-              if e.message.match(/client not authorized for any of the scopes requested/)
+              if auth_error_quiet
+                MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes
+              else
+                MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::ERROR, details: @scopes
+                if e.message.match(/client not authorized for any of the scopes requested/)
 # XXX it'd be helpful to list *all* scopes we like, as well as the API client's numeric id
-                MU.log "To grant access to API scopes for this service account, see:", MU::ERR, details: "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"
+                  MU.log "To grant access to API scopes for this service account, see:", MU::ERR, details: "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"
+                end
               end
+
               raise e
             end
           end

data/modules/mu/config.rb

@@ -525,7 +525,7 @@ end
               region: @region,
               habitats: hab_arg,
               credentials: @credentials,
-              dummy_ok: (["habitats", "folders", "users", "groups"].include?(@type))
+              dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
             )
             @obj ||= found.first if found
           rescue ThreadError => e
@@ -1786,6 +1786,9 @@ $CONFIGURABLES
 
 
       acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
+      if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"]
+        acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"]
+      end
       acl.delete("vpc") if !acl["vpc"]
       if !resclass.isGlobal? and !region.nil? and !region.empty?
         acl["region"] = region
@@ -1972,6 +1975,13 @@ $CONFIGURABLES
 
     # Namespace magic to pass to ERB's result method.
     def get_binding(keyset)
+      environment = $environment
+      myPublicIp = $myPublicIp
+      myRoot = $myRoot
+      myAZ = $myAZ
+      myRegion = $myRegion
+      myAppName = $myAppName
+
 #      return MU::Config.global_bindings[keyset] if MU::Config.global_bindings[keyset]
       MU::Config.global_bindings[keyset] = binding
       MU::Config.global_bindings[keyset]

data/modules/mu/mommacat.rb

@@ -2762,6 +2762,9 @@ MESSAGE_END
 		# Start the Momma Cat daemon and return the exit status of the command used
     # @return [Integer]
     def self.start
+      if MU.inGem? and MU.muCfg['disable_mommacat']
+        return
+      end
       base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
       [base, "#{base}/log", "#{base}/run"].each { |dir|
        if !Dir.exist?(dir)
@@ -2784,20 +2787,19 @@ MESSAGE_END
 
       MU.log cmd, MU::NOTICE
 
-      output = %x{#{cmd}}
-
-      Dir.chdir(origdir)
-
       retries = 0
       begin
+        output = %x{#{cmd}}
         sleep 1
         retries += 1
         if retries >= 10
-          MU.log "MommaCat failed to start (command was #{cmd})", MU::WARN, details: output
+          MU.log "MommaCat failed to start (command was #{cmd}, working directory #{MU.myRoot}/modules)", MU::WARN, details: output
           pp caller
           return $?.exitstatus
         end
       end while !status
+
+      Dir.chdir(origdir)
     
       if $?.exitstatus != 0
         exit 1
@@ -2809,6 +2811,9 @@ MESSAGE_END
     # Return true if the Momma Cat daemon appears to be running
     # @return [Boolean]
     def self.status
+      if MU.inGem? and MU.muCfg['disable_mommacat']
+        return true
+      end
       if File.exist?(daemonPidFile)
         pid = File.read(daemonPidFile).chomp.to_i
         begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-mu
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.2
 platform: ruby
 authors:
 - John Stange
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-22 00:00:00.000000000 Z
+date: 2019-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubis
@@ -355,6 +355,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.37'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 description: |+
   The eGTLabs Mu toolkit for unified cloud deployments. This gem contains the Mu deployment interface to cloud provider APIs. It will generate a sample configuration the first time it is invoked.