cloud-mu 3.0.0beta → 3.0.0

data/modules/mu/logger.rb

@@ -33,6 +33,7 @@ module MU
     # Show DEBUG log entries and extra call stack and threading info
     LOUD = 2.freeze
 
+    attr_accessor :verbosity
     @verbosity = MU::Logger::NORMAL
     @quiet = false
     @html = false
@@ -52,7 +53,6 @@ module MU
     end
 
     attr_reader :summary
-    attr_accessor :verbosity
     attr_accessor :color
     attr_accessor :quiet
     attr_accessor :html
@@ -71,8 +71,10 @@ module MU
             handle: @handle,
             color: @color
     )
-      verbosity = MU::Logger::NORMAL if verbosity.nil?
+      verbosity ||= @verbosity
       return if verbosity == MU::Logger::SILENT
+      return if verbosity < MU::Logger::LOUD and level == DEBUG
+      return if verbosity < MU::Logger::NORMAL and level == INFO
 
       # By which we mean, "get the filename (with the .rb stripped off) which
       # originated the call to this method. Which, for our purposes, is the
@@ -159,7 +161,7 @@ module MU
             else
               handle.puts "#{time} - #{caller_name} - #{msg}"
             end
-            if verbosity >= MU::Logger::LOUD
+            if verbosity >= MU::Logger::QUIET
               if @html
                 html_out "#{caller_name} - #{msg}"
               elsif color
@@ -178,7 +180,7 @@ module MU
             else
               handle.puts "#{time} - #{caller_name} - #{msg}"
             end
-            if verbosity >= MU::Logger::LOUD
+            if verbosity >= MU::Logger::SILENT
               if @html
                 html_out "#{caller_name} - #{msg}"
               elsif color

data/modules/mu/mommacat.rb

@@ -238,9 +238,9 @@ module MU
         end
         credsets = {}
 
-        MU::Cloud.resource_types.values { |data|
-          if !@original_config[data[:cfg_plural]].nil? and @original_config[data[:cfg_plural]].size > 0
-            @original_config[data[:cfg_plural]].each { |resource|
+        MU::Cloud.resource_types.values.each { |attrs|
+          if !@original_config[attrs[:cfg_plural]].nil? and @original_config[attrs[:cfg_plural]].size > 0
+            @original_config[attrs[:cfg_plural]].each { |resource|
 
               credsets[resource['cloud']] ||= []
               credsets[resource['cloud']] << resource['credentials']
@@ -257,7 +257,6 @@ module MU
         end
         MU.log "Creating deploy secret for #{MU.deploy_id}"
         @deploy_secret = Password.random(256)
-
         if !@original_config['scrub_mu_isms']
           credsets.each_pair { |cloud, creds|
             creds.uniq!
@@ -282,7 +281,7 @@ module MU
       loadDeploy(set_context_to_me: set_context_to_me)
       if !deploy_secret.nil?
         if !authKey(deploy_secret)
-          raise DeployInitializeError, "Invalid or incorrect deploy key."
+          raise DeployInitializeError, "Client request did not include a valid deploy authorization secret. Verify that userdata runs correctly?"
         end
       end
 
@@ -328,9 +327,16 @@ module MU
                 next
               end
 
-              if orig_cfg['vpc']
-                ref = MU::Config::Ref.get(orig_cfg['vpc'])
-                orig_cfg['vpc']['id'] = ref if ref.kitten
+              if orig_cfg['vpc'] and orig_cfg['vpc'].is_a?(Hash)
+                ref = if orig_cfg['vpc']['id'] and orig_cfg['vpc']['id'].is_a?(Hash)
+                  orig_cfg['vpc']['id']['mommacat'] = self
+                  MU::Config::Ref.get(orig_cfg['vpc']['id'])
+                else
+                  orig_cfg['vpc']['mommacat'] = self
+                  MU::Config::Ref.get(orig_cfg['vpc'])
+                end
+                orig_cfg['vpc'].delete('mommacat')
+                orig_cfg['vpc'] = ref if ref.kitten
               end
 
               begin
@@ -921,7 +927,7 @@ module MU
         MU.log "Creating #{ssh_dir}", MU::DEBUG
         Dir.mkdir(ssh_dir, 0700)
         if Process.uid == 0 and @mu_user != "mu"
-          ssh_dir.chown(Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid)
+          File.chown(Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir)
         end
       end
       if !File.exist?("#{ssh_dir}/#{@ssh_key_name}")
@@ -1104,10 +1110,10 @@ module MU
       MU::MommaCat.listDeploys.each { |deploy_id|
         next if File.exist?(deploy_dir(deploy_id)+"/.cleanup")
         MU.log "Checking for dead wood in #{deploy_id}", MU::DEBUG
+        need_reload = false
         @cleanup_threads << Thread.new {
           MU.dupGlobals(parent_thread_id)
-          # We can't use cached litter information because we will then try to delete the same node over and over again until we restart the service
-          deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true, use_cache: false)
+          deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true)
           purged_this_deploy = 0
           if deploy.kittens.has_key?("servers")
             deploy.kittens["servers"].values.each { |nodeclasses|
@@ -1120,6 +1126,7 @@ module MU
                   elsif !server.active?
                     next if File.exist?(deploy_dir(deploy_id)+"/.cleanup-"+server.cloud_id)
                     deletia << mu_name
+                    need_reload = true
                     MU.log "Cleaning up metadata for #{server} (#{nodeclass}), formerly #{server.cloud_id}, which appears to have been terminated", MU::NOTICE
                     begin
                       server.destroy
@@ -1134,6 +1141,9 @@ module MU
                     purged_this_deploy = purged_this_deploy + 1
                   end
                 }
+                deletia.each { |mu_name|
+                  servers.delete(mu_name)
+                }
                 if purged_this_deploy > 0
                   # XXX some kind of filter (obey sync_siblings on nodes' configs)
                   deploy.syncLitter(servers.keys)
@@ -1141,6 +1151,10 @@ module MU
               }
             }
           end
+          if need_reload
+            deploy.save!
+            MU::MommaCat.getLitter(deploy_id, use_cache: false)
+          end
           MU.purgeGlobals
         }
       }
@@ -1191,7 +1205,8 @@ module MU
         flags: {},
         habitats: [],
         dummy_ok: false,
-        debug: false
+        debug: false,
+        no_deploy_search: false
     ) 
       start = Time.now
       callstr = "findStray(cloud: #{cloud}, type: #{type}, deploy_id: #{deploy_id}, calling_deploy: #{calling_deploy.deploy_id if !calling_deploy.nil?}, name: #{name}, cloud_id: #{cloud_id}, tag_key: #{tag_key}, tag_value: #{tag_value}, credentials: #{credentials}, habitats: #{habitats ? habitats.to_s : "[]"}, dummy_ok: #{dummy_ok.to_s}, flags: #{flags.to_s}) from #{caller[0]}"
@@ -1253,7 +1268,7 @@ module MU
 
         kittens = {}
         # Search our other deploys for matching resources
-        if (deploy_id or name or mu_name or cloud_id)
+        if !no_deploy_search and (deploy_id or name or mu_name or cloud_id)
           MU.log "findStray: searching my deployments (#{cfg_plural}, name: #{name}, deploy_id: #{deploy_id}, mu_name: #{mu_name}) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
 
           # Check our in-memory cache of live deploys before resorting to
@@ -1569,7 +1584,7 @@ end
       rescue Exception => e
         MU.log e.inspect, MU::ERR, details: e.backtrace
       end
-      MU.log "findStray: returning #{matches.size.to_s} matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel
+      MU.log "findStray: returning #{matches ? matches.size.to_s : "0"} matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel
 
       matches
     end
@@ -2015,7 +2030,7 @@ end
         return
       end
       if ssh_key_name.nil? or ssh_key_name.empty?
-        MU.log "Failed to extract canonical_ip for #{ssh_key_name.mu_name} in addHostToSSHConfig", MU::ERR
+        MU.log "Failed to extract ssh_key_name for #{ssh_key_name.mu_name} in addHostToSSHConfig", MU::ERR
         return
       end
 
@@ -2287,7 +2302,7 @@ MESSAGE_END
         MU::MommaCat.listDeploys.sort.each { |deploy_id|
           begin
             # We don't want to use cached litter information here because this is also called by cleanTerminatedInstances.
-            deploy = MU::MommaCat.getLitter(deploy_id, use_cache: false)
+            deploy = MU::MommaCat.getLitter(deploy_id)
             if deploy.ssh_key_name.nil? or deploy.ssh_key_name.empty?
               MU.log "Failed to extract ssh key name from #{deploy_id} in syncMonitoringConfig", MU::ERR if deploy.kittens.has_key?("servers")
               next
@@ -2298,6 +2313,7 @@ MESSAGE_END
               deploy.kittens["servers"].values.each { |nodeclasses|
                 nodeclasses.values.each { |nodes|
                   nodes.values.each { |server|
+                    next if !server.cloud_desc
                     MU.dupGlobals(parent_thread_id)
                     threads << Thread.new {
                       MU::MommaCat.setThreadContext(deploy)
@@ -2528,10 +2544,14 @@ MESSAGE_END
       update_servers.each { |node|
         # Not clear where this pollution comes from, but let's stick a temp
         # fix in here.
-        if node.deploydata['nodename'] != node.mu_name
+        if node.deploydata['nodename'] != node.mu_name and
+           !node.deploydata['nodename'].nil? and !node.deploydata['nodename'].emty?
           MU.log "Node #{node.mu_name} had wrong or missing nodename (#{node.deploydata['nodename']}), correcting", MU::WARN
           node.deploydata['nodename'] = node.mu_name
-          @deployment[svrs][node.config['name']][node.mu_name]['nodename'] = node.mu_name
+          if @deployment[svrs] and @deployment[svrs][node.config['name']] and
+             @deployment[svrs][node.config['name']][node.mu_name]
+            @deployment[svrs][node.config['name']][node.mu_name]['nodename'] = node.mu_name
+          end
           save!
         end
       }
@@ -2670,7 +2690,7 @@ MESSAGE_END
       Dir.chdir(MU.myRoot+"/modules")
 
       # XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs?
-      cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.mySSLDir}/mommacat.key --ssl-cert-file #{MU.mySSLDir}/mommacat.pem --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
+      cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
       MU.log cmd, MU::NOTICE
       output = %x{#{cmd}}
       Dir.chdir(origdir)

data/modules/mu/mu.yaml.rb

@@ -0,0 +1,276 @@
+# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}.
+#
+# Example:
+#
+# <pre>
+#      ---    
+#      public_address: 1.2.3.4    
+#      mu_admin_email: egtlabs@eglobaltech.com    
+#      mu_admin_name: Joe Schmoe    
+#      mommacat_port: 2260    
+#      banner: My Example Mu Master    
+#      mu_repository: git://github.com/cloudamatic/mu.git    
+#      repos:    
+#      - https://github.com/cloudamatic/mu_demo_platform    
+#      allow_invade_foreign_vpcs: true    
+#      ansible_dir:    
+#      aws:    
+#        egtdev:    
+#          region: us-east-1    
+#          log_bucket_name: egt-mu-log-bucket    
+#          default: true    
+#          name: egtdev    
+#        personal:    
+#          region: us-east-2    
+#          log_bucket_name: my-mu-log-bucket    
+#          name: personal    
+#        google:    
+#          egtlabs:    
+#            project: egt-labs-admin    
+#            credentials_file: /opt/mu/etc/google.json    
+#            region: us-east4    
+#            log_bucket_name: hexabucket-761234    
+#            default: true    
+# </pre>
+module MuYAML
+	# The configuration file format for Mu's main config file.
+	# Amazon Web Services
+	class aws
+		# @!group Required parameters
+		
+		# **REQUIRED** - 
+		# S3 bucket into which we'll synchronize deploy secrets, and if we're hosted in AWS, collected system logs
+		#
+		# @return [String]
+		attr_accessor :log_bucket_name
+		# @!endgroup
+		# @!group Optional parameters
+		
+		# **Must match pattern `(?i-mx:^[a-z0-9]+$)`** - 
+		# Credentials used for accessing the AWS API (looks like: AKIAINWLOOAA24PBRBZA)
+		#
+		# @return [String]
+		attr_accessor :access_key
+		
+		# Credentials used for accessing the AWS API (looks like: +Z16iRP9QAq7EcjHINyEMs3oR7A76QpfaSgCBogp).
+		#
+		# @return [String]
+		attr_accessor :access_secret
+		
+		# **Must match pattern `(?-mix:^\d+$)`** - 
+		# Default target account for resources managed using these credentials. This is an AWS account number, e.g. 918972669773. If not specified, we will use the account number which owns these API keys.
+		#
+		# @return [String]
+		attr_accessor :account_number
+		
+		# A secure Chef vault and item from which to retrieve an AWS access key and secret. The vault item should have 'access_key' and 'access_secret' elements.
+		#
+		# @return [String]
+		attr_accessor :credentials
+		
+		# An INI-formatted AWS credentials file, of the type used by the AWS command-line tools. This is less secure than using 'credentials' to store these in a Chef vault. See: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html
+		#
+		# @return [String]
+		attr_accessor :credentials_file
+		
+		# **Default: `false`** - 
+		# If set to true, Mu will default to these AWS credentials when targeting AWS resources
+		#
+		# @return [Boolean]
+		attr_accessor :default
+		
+		# Default Amazon Web Services region in which these credentials should operate
+		#
+		# @return [String]
+		attr_accessor :region
+		# @!endgroup
+	end
+	# Microsoft Azure Cloud Computing Platform & Services
+	class azure
+		# @!group Optional parameters
+		
+		# App client id used to authenticate to our subscription. From https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview
+		#
+		# @return [String]
+		attr_accessor :client_id
+		
+		# App client secret used to authenticate to our subscription. From https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview under the 'Certificates & secrets' tab, 'Client secrets.' This can only be retrieved upon initial secret creation.
+		#
+		# @return [String]
+		attr_accessor :client_secret
+		
+		# JSON file which contains a hash of directory_id, client_id, client_secret, and subscription values. If found, these will be override values entered directly in mu-configure.
+		#
+		# @return [String]
+		attr_accessor :credentials_file
+		
+		# **Default: `false`** - 
+		# If set to true, Mu will use this set of Azure credentials when targeting Azure without a specific account having been requested
+		#
+		# @return [Boolean]
+		attr_accessor :default
+		
+		# AKA Tenant ID; the default Microsoft Azure Directory project in which we operate and deploy, from https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview
+		#
+		# @return [String]
+		attr_accessor :directory_id
+		
+		# **Default: `eastus`** - 
+		# Default Microsoft Azure region in which we operate and deploy
+		#
+		# @return [String]
+		attr_accessor :region
+		
+		# Default Microsoft Azure Subscription we will use to deploy, from https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade
+		#
+		# @return [String]
+		attr_accessor :subscription
+		# @!endgroup
+	end
+	# Google Cloud Platform
+	class google
+		# @!group Required parameters
+		
+		# **REQUIRED** - 
+		# Cloud Storage bucket into which we'll synchronize deploy secrets, and if we're hosted in GCP, collected system logs
+		#
+		# @return [String]
+		attr_accessor :log_bucket_name
+		
+		# **REQUIRED** - 
+		# Default Google Cloud Platform project in which we operate and deploy. Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON, and import that key to the vault specified here. Import example: knife vault create secrets google -J my-google-service-account.json
+		#
+		# @return [String]
+		attr_accessor :project
+		# @!endgroup
+		# @!group Optional parameters
+		
+		# A secure Chef vault and item from which to retrieve the JSON-formatted Service Account credentials for our GCP account, in the format vault:itemname (e.g. 'secrets:google'). Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON, and import that key to the vault specified here. Import example: knife vault create secrets google -J my-google-service-account.json 
+		#
+		# @return [String]
+		attr_accessor :credentials
+		
+		# JSON-formatted Service Account credentials for our GCP account, b64-encoded and dropped directly into mu.yaml. Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON and point this argument to the file. This is less secure than using 'credentials' to store in a vault.
+		#
+		# @return [String]
+		attr_accessor :credentials_encoded
+		
+		# JSON-formatted Service Account credentials for our GCP account, stored in plain text in a file. Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON and point this argument to the file. This is less secure than using 'credentials' to store in a vault.
+		#
+		# @return [String]
+		attr_accessor :credentials_file
+		
+		# For Google Cloud projects which are attached to a GSuite domain. Some API calls (groups, users, etc) require this identifier. From admin.google.com, choose Security, the Single Sign On, and look for the Entity ID field. The value after idpid= in the URL there should be the customer ID.
+		#
+		# @return [String]
+		attr_accessor :customer_id
+		
+		# **Default: `false`** - 
+		# If set to true, Mu will use this set of GCP credentials when targeting the Google Cloud without a specific account having been requested
+		#
+		# @return [Boolean]
+		attr_accessor :default
+		
+		# For Google Cloud projects which are attached to a GSuite domain. GCP service accounts cannot view or manage GSuite resources (groups, users, etc) directly, but must instead masquerade as a GSuite user which has delegated authority to the service account. See also: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority
+		#
+		# @return [String]
+		attr_accessor :masequerade_as
+		
+		# **Default: `us-east4`** - 
+		# Default Google Cloud Platform region in which we operate and deploy
+		#
+		# @return [String]
+		attr_accessor :region
+		# @!endgroup
+	end
+	# @!group Required parameters
+	
+	# **REQUIRED**,
+	# **Must match pattern `(?i-mx:^[a-z0-9\-_]+$)`** - 
+	# The local system's value for HOSTNAME
+	#
+	# @return [String]
+	attr_accessor :hostname
+	
+	# **REQUIRED**,
+	# **Must match pattern `(?i-mx:\A([\w+\-].?)+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z)`** - 
+	# Administative contact email
+	#
+	# @return [String]
+	attr_accessor :mu_admin_email
+	
+	# **REQUIRED**,
+	# **Must match pattern `(?-mix:^(127\.0\.0\.1|localhost)$)`** - 
+	# IP address or hostname
+	#
+	# @return [String]
+	attr_accessor :public_address
+	# @!endgroup
+	# @!group Optional parameters
+	
+	# If set to true, Mu will be allowed to modify routing and peering behavior of VPCs which it did not create, but for which it has permissions.
+	#
+	# @return [Boolean]
+	attr_accessor :allow_invade_foreign_vpcs
+	
+	# Intended for use with minimal installs which use Ansible as a groomer and which do not store Ansible artifacts in a dedicated git repository. This allows simply pointing to a local directory.
+	#
+	# @return [String]
+	attr_accessor :ansible_dir
+	
+	# Amazon Web Services
+	#
+	# @return [aws]
+	# @see aws
+	attr_accessor :aws
+	
+	# Microsoft Azure Cloud Computing Platform & Services
+	#
+	# @return [azure]
+	# @see azure
+	attr_accessor :azure
+	
+	# Login banner, displayed in various locations
+	#
+	# @return [String]
+	attr_accessor :banner
+	
+	# Google Cloud Platform
+	#
+	# @return [google]
+	# @see google
+	attr_accessor :google
+	
+	# Optional extra Chef roles or recipes to invoke when running chef-client on this Master (ex: recipe[mycookbook::mumaster])
+	#
+	# @return [Array<String>]
+	attr_accessor :master_runlist_extras
+	
+	# **Default: `2260`**,
+	# **Must match pattern `(?i-mx:^[0-9]+$)`** - 
+	# Listen port for the Momma Cat grooming daemon
+	#
+	# @return [String]
+	attr_accessor :mommacat_port
+	
+	# **Default: `Mu Administrator`** - 
+	# Administative contact's full name
+	#
+	# @return [String]
+	attr_accessor :mu_admin_name
+	
+	# **Default: `git://github.com/cloudamatic/mu.git`**,
+	# **Must match pattern `(?-mix:(((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?))?([\w\.@\:\/\-~]+)(\.git)?(\/)?)`** - 
+	# Source repository for Mu tools
+	#
+	# @return [String]
+	attr_accessor :mu_repository
+	
+	# **Default: `["https://github.com/cloudamatic/mu_demo_platform"]`**,
+	# **Must match pattern `(?-mix:(((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?))?([\w\.@\:\/\-~]+)(\.git)?(\/)?)`** - 
+	# Optional platform repositories, as a Git URL or Github repo name (ex: eGT-Labs/fema_platform.git)
+	#
+	# @return [Array<String>]
+	attr_accessor :repos
+	# @!endgroup
+end