cloud-mu 3.1.2 → 3.2.0

data/bin/mu-deploy

@@ -105,7 +105,7 @@ if $opts[:dryrun]
     Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :never) {
       begin
         Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :immediate) {
-          MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
+          MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
           Thread.current.exit
         }
       ensure
@@ -124,7 +124,7 @@ if $opts[:dryrun]
       )
       cost_dummy_deploy.run
     rescue MU::Cloud::MuCloudResourceNotImplemented, MU::Cloud::MuCloudFlagNotImplemented
-      MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
+      MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
     end
   end
   exit
@@ -135,7 +135,7 @@ if $opts[:update]
 # TODO consider whether this is useful/valid
 #  old_conf = JSON.parse(File.read(deploy.deploy_dir+"/basket_of_kittens.json"))
 #  stack_conf = old_conf.merge(stack_conf)
-  deploy.updateBasketofKittens(stack_conf)
+  deploy.updateBasketofKittens(stack_conf, skip_validation: true)
   deployer = MU::Deploy.new(
     deploy.environment,
     verbosity: verbosity,

data/bin/mu-findstray-tests

@@ -0,0 +1,25 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, debug: true)

data/bin/mu-gen-docs

@@ -79,8 +79,7 @@ EOF
       impl_counts[type] ||= 0
       [a, b].each { |cloud|
         begin
-          myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-          case myclass.quality
+          case MU::Cloud.resourceClass(cloud, type).quality
           when MU::Cloud::RELEASE
             cloud_is_useful[cloud] = true
             counts[cloud] += 4
@@ -114,8 +113,7 @@ EOF
     cloudlist.each { |cloud|
       readme += "<td><center>"
       begin
-        myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-        case myclass.quality
+        case MU::Cloud.resourceClass(cloud, type).quality
         when MU::Cloud::RELEASE
           readme += "<img src='release.png' style='#{icon_style}' title='Release Quality' alt='[Release Quality]'>"
         when MU::Cloud::BETA

data/bin/mu-load-config.rb

@@ -36,8 +36,7 @@ def validateClouds(cfg)
         cfg[cloud]["default"]["default"] = true
         found_default = true
       else
-        missing_alias = false
-        cfg[cloud].each_pair { |acctalias, acct|
+        cfg[cloud].values.each { |acct|
           if acct["default"]
             if found_default
               puts "Multiple accounts have 'default' set in #{cloud}"
@@ -205,7 +204,7 @@ def cfgPath
     end
   end
   home = Etc.getpwuid(Process.uid).dir
-  username = Etc.getpwuid(Process.uid).name
+  Etc.getpwuid(Process.uid).name # validates existence of a username
   if Process.uid == 0 and !in_gem
     if ENV.include?('MU_INSTALLDIR')
       ENV['MU_INSTALLDIR']+"/etc/mu.yaml"

data/bin/mu-node-manage

@@ -29,9 +29,9 @@ Usage:
   opt :all, "Operate on all nodes/deploys. Use with caution.", :require => false, :default => false, :type => :boolean
   opt :platform, "Operate exclusively on one nodes of a particular operating system. Can be used in conjunction with -a or -d. Valid platforms: linux, windows", :require => false, :type => :string
   opt :environment, "Operate exclusively on one nodes with a particular environment (e.g. dev, prod). Can be used in conjunction with -a or -d.", :require => false, :type => :string
-  opt :override_chef_runlist, "An alternate runlist to pass to Chef, in chefrun mode.", :require => false, :type => :string
+  opt :override_chef_runlist, "An alternate runlist to pass to Chef, in groomeronly mode.", :require => false, :type => :string
   opt :xecute, "Run a shell command on matching nodes. Overrides --mode and suppresses some informational output in favor of scriptability.", :require => false, :type => :string
-  opt :mode, "Action to perform on matching nodes. Valid actions: groom, chefrun, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "chefrun", :type => :string
+  opt :mode, "Action to perform on matching nodes. Valid actions: groom, groomeronly, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "groomeronly", :type => :string
   opt :verbose, "Show output from Chef runs, etc", :require => false, :default => false, :type => :boolean
   opt :winrm, "Force WinRM connection. Disable SSH fallback", :require => false, :default => false, :type => :boolean
   opt :info, "List a particular node attribute", :require => false, :default => 'nodename', :type => :string
@@ -39,8 +39,10 @@ end
 
 MU.setLogging(MU::Logger::LOUD) if $opts[:verbose]
 
-if !["groom", "chefrun", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
-  Optimist::die(:mode, "--mode must be one of: groom, chefrun, awsmeta, vaults, certs, chefupgrade")
+$opts[:mode] = "groomeronly" if $opts[:mode] == "chefrun"
+
+if !["groom", "groomeronly", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
+  Optimist::die(:mode, "--mode must be one of: groom, groomeronly, awsmeta, vaults, certs, chefupgrade")
 end
 if $opts[:platform] and !["linux", "windows"].include?($opts[:platform])
   Optimist::die(:platform, "--platform must be one of: linux, windows")
@@ -176,7 +178,7 @@ end
 exit 1 if !ok
 
 
-def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
+def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false, groomeronly: false)
   badnodes = []
   count = 0
   deploys.each { |muid|
@@ -196,6 +198,8 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
                 server.config["vault_access"].each { |v|
                   MU::Groomer::Chef.grantSecretAccess(mu_name, v['vault'], v['item'])
                 }
+              elsif groomeronly
+                server.groomer.run
               else
                 mommacat.groomNode(server.cloud_id, nodeclass, type, mu_name: mu_name)
               end
@@ -227,7 +231,7 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
   end
 end
 
-def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false, chefrun: false, chef_runlist: nil)
+def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false)
   badnodes = []
   count = 0
   deploys.each { |muid|
@@ -247,12 +251,6 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
             next
           end
 
-          # Generate the command if attemting a chef run
-          if chefrun
-            cmd = serverobj.windows? ? "powershell -Command chef-client" : "chef-client || sudo chef-client"
-            cmd += " -o '#{chef_runlist}'" if chef_runlist
-          end
-
           MU.log "Running '#{cmd}' on #{nodename} (##{count})" if !print_output
 
           # Set Variables to catch the output and exit code of the execution
@@ -363,7 +361,7 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
   }
 
   if badnodes.size > 0
-    cmd = "Chef" if $opts[:mode] == "chefrun"
+    cmd = "Chef" if $opts[:mode] == "groomeronly"
     if !print_output
       MU.log "Not all `#{cmd}` runs exited cleanly", MU::WARN, details: badnodes
     else
@@ -687,12 +685,13 @@ elsif $opts[:mode] == "vaults"
   reGroom(do_deploys, do_nodes, vaults_only: true)
 elsif $opts[:mode] == "chefupgrade"
   chefUpgrade(do_deploys, do_nodes)
-elsif $opts[:mode] == "chefrun"
+elsif $opts[:mode] == "groomeronly"
   print_output = $opts[:verbose] || do_nodes.size == 1
   if $opts[:override_chef_runlist]
-    runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], chefrun: true, print_output: print_output)
+#    runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], groomeronly: true, print_output: print_output)
   else
-    runCommand(do_deploys, do_nodes, chefrun: true, print_output: print_output)
+#    runCommand(do_deploys, do_nodes, groomeronly: true, print_output: print_output)
+    reGroom(do_deploys, do_nodes, groomeronly: true)
   end
 elsif $opts[:mode] == "userdata" or $opts[:mode] == "awsmeta"
 # Need Google equiv and to select nodes correctly based on what cloud they're in

data/bin/mu-run-tests

@@ -13,53 +13,139 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+bindir = File.realpath(File.expand_path(File.dirname(__FILE__)))
+dir = File.realpath(File.expand_path(bindir+"/../modules/tests"))
+Dir.chdir(dir)
+
 require 'colorize'
 require 'optimist'
-bindir = File.realpath(File.expand_path(File.dirname(__FILE__)))
 require bindir+"/mu-load-config.rb"
 require 'mu'
 
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'json-schema'
+
 $opts = Optimist::options do
   banner <<-EOS
 Usage:
-#{$0} [-e <environment>] [-r region] [-v] [-d] [-w] [-c] [-n] [-s] [-j] [-p parameter=value] /path/to/stack/config.[json|yaml] [-u deploy_id [-l]] [-r deploy_id]
+#{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]]
   EOS
   opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer
   opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
+  opt :verbose, "Show more information while running", :require => false, :default => false
 end
 
-dir = File.realpath(File.expand_path(bindir+"/../modules/tests"))
-Dir.chdir(dir)
+only = ARGV
 
 files = Dir.glob("*.yaml", base: dir)
 files.concat(Dir.glob("*.yml", base: dir))
-baseclouds = MU::Cloud.supportedClouds.reject { |c| c == "CloudFormation" }
+baseclouds = MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
 
 commands = {}
+failures = []
+if only and only.size > 0
+  files.reject! { |f| !only.include?(f) }
+  if files.size == 0
+    MU.log "No files in #{dir} matched requested list", MU::WARN, details: only
+    exit 1
+  end
+end
 
 files.each { |f|
   clouds = baseclouds.dup
+  groomer_match = true
   File.open(dir+"/"+f).readlines.each { |l|
     l.chomp!
-    next if !l.match(/^\s*#\s*clouds: (.*)/)
-    clouds = []
-    cloudstr = Regexp.last_match[1]
-    cloudstr.split(/\s*,\s*/).each { |c|
-      baseclouds.each { |cloud|
-        if cloud.match(/^#{Regexp.quote(c)}$/i)
-          clouds << cloud
+    if l.match(/^\s*#\s*clouds: (.*)/)
+      clouds = []
+      cloudstr = Regexp.last_match[1]
+      cloudstr.split(/\s*,\s*/).each { |c|
+        baseclouds.each { |cloud|
+          if cloud.match(/^#{Regexp.quote(c)}$/i)
+            clouds << cloud
+          end
+        }
+      }
+    elsif l.match(/^\s*#\s*groomers: (.*)/)
+      groomerstr = Regexp.last_match[1]
+      groomerstr.split(/\s*,\s*/).each { |g|
+        if !MU::Groomer.availableGroomers.include?(g)
+          MU.log "#{f} requires groomer #{g}, which is not available. This test will be skipped.", MU::NOTICE
+          groomer_match = false
         end
       }
-    }
-    break
+    end
   }
+  if !groomer_match
+    next
+  end
+
   clouds.each { |cloud|
-    commands["mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"] = { "file" => f, "cloud" => cloud }
+    cmd = "mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"
+    commands[cmd] = {
+      "file" => f,
+      "cloud" => cloud,
+    }
+    if $opts[:full]
+      $environment = "dev"
+      begin
+        conf_engine = MU::Config.new(f, cloud: cloud)
+      rescue StandardError => e
+        MU.log e.message+" parsing "+f+" with cloud "+cloud, MU::WARN, details: e.backtrace
+        failures << f+" ("+commands[cmd]["cloud"]+")"
+        next 
+      end
+      parsed = MU::Config.stripConfig(conf_engine.config)
+      types = []
+      MU::Cloud.resource_types.values.each { |attrs|
+        if parsed.has_key?(attrs[:cfg_plural])
+          types << attrs[:cfg_plural]
+        end
+      }
+      commands[cmd]["parsed"] = parsed
+      commands[cmd]["types"] = types
+    end
   }
 }
 
 puts "Running #{commands.size.to_s.bold} #{$opts[:full] ? "full deploy" : "parse"} tests from #{files.size.to_s.bold} Baskets of Kittens across #{baseclouds.size.to_s.bold} clouds"
 
+@output_semaphore = Mutex.new
+
+def execCommand(cmd, results_stash)
+  @output_semaphore.synchronize {
+    puts cmd if $opts[:verbose]
+  }
+
+  ok = true
+  output = %x{#{cmd} 2>&1}
+  ok = false if $?.exitstatus != 0
+
+  results_stash["output"] += output
+
+  @output_semaphore.synchronize {
+    if ok
+      if $opts[:verbose]
+        puts "#{cmd} SUCCEEDED".green
+      else
+        print ".".green
+      end
+    else
+      if $opts[:verbose]
+        puts "#{cmd} FAILED:".light_red
+        puts output
+      else
+        print ".".light_red
+      end
+    end
+  }
+
+  ok
+end
+
 threads = []
 results = {}
 commands.keys.each { |cmd|
@@ -72,21 +158,40 @@ commands.keys.each { |cmd|
   end
 
   threads << Thread.new(cmd) { |cmd_thr|
-    results[cmd_thr] = {}
-    puts %Q{#{cmd_thr} 2>&1}
-    results[cmd_thr]["output"] = %x{#{cmd_thr} 2>&1}
-    if $?.exitstatus != 0
-      results[cmd_thr]["status"] = false
-      print ".".light_red
-    else
-      results[cmd_thr]["status"] = true
-      print ".".green
+    results[cmd_thr] = { "output" => "", "failed" => [] }
+    if !execCommand(cmd_thr, results[cmd_thr])
+      results[cmd_thr]["failed"] << "main"
     end
+
     if $opts[:full] and results[cmd_thr]["output"].match(/deploy - Deployment id: .*? \((.*?)\)/)
       deploy_id = Regexp.last_match[1]
-      results[cmd_thr]["output"] += %x{mu-cleanup #{deploy_id} --skipsnapshots}
-      if $?.exitstatus != 0
-        results[cmd_thr]["status"] = false
+      adoptdir = Dir.mktmpdir(commands[cmd_thr]["file"].gsub(/[^a-z0-9]|yaml$/i, ""))
+      if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
+        adopt = "cd #{adoptdir} && mu-adopt --appname adoptone --grouping omnibus  --clouds #{commands[cmd_thr]["cloud"]} --types #{commands[cmd_thr]["types"].join(" ")} 2>&1"
+        if !execCommand(adopt, results[cmd_thr])
+          results[cmd_thr]["failed"] << "adopt"
+        end
+      end
+
+      if File.exist?(dir+"/regrooms/"+commands[cmd_thr]["file"])
+        regroom = "mu-deploy regrooms/#{commands[cmd_thr]["file"]} --cloud #{commands[cmd_thr]["cloud"]} --update #{deploy_id} 2>&1"
+        if !execCommand(regroom, results[cmd_thr])
+          results[cmd_thr]["failed"] << "regroom"
+        end
+        if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
+          re_adopt = "cd #{adoptdir} && mu-adopt --appname adopttwo --grouping omnibus --clouds #{commands[cmd_thr]["cloud"]} --types #{commands[cmd_thr]["types"].join(" ")} 2>&1"
+          if !execCommand(re_adopt, results[cmd_thr])
+            results[cmd_thr]["failed"] << "second adopt"
+          end
+        end
+# TODO big flex is to read back both adopted BoKs and .diff them, but without
+# all resources having implemented adoption this isn't much of a test yet
+      end
+
+      FileUtils.remove_entry(adoptdir)
+      cleanup = %Q{mu-cleanup #{deploy_id} --skipsnapshots}
+      if !execCommand(cleanup, results[cmd_thr])
+        results[cmd_thr]["failed"] << "cleanup"
       end
     end
   }
@@ -96,21 +201,14 @@ threads.each { |t|
 }
 puts ""
 
-failures = []
 results.keys.sort { |a, b|
-      if results[b]["status"] and !results[a]["status"]
-        1
-      elsif !results[b]["status"] and results[a]["status"]
-        -1
-      else
-        0
-      end
+      results[b]["failed"].size <=> results[a]["failed"].size
     }.each { |cmd|
-  if !results[cmd]["status"]
+  if results[cmd]["failed"].size > 0
     puts cmd+" failed:".light_red
     puts results[cmd]["output"].yellow
     puts "^ #{cmd}".light_red
-    failures << commands[cmd]["file"]+" ("+commands[cmd]["cloud"]+")"
+    failures << commands[cmd]["file"]+" in "+commands[cmd]["cloud"]+" ("+results[cmd]["failed"].join(", ")+")"
   else
     puts cmd+" passed".green
   end

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.1.2'
-  s.date        = '2020-01-27'
+  s.version     = '3.2.0'
+  s.date        = '2020-06-16'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -35,31 +35,33 @@ EOF
   s.homepage    =
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'
-  s.add_runtime_dependency 'erubis', "~> 2.7"
-  s.add_runtime_dependency 'inifile', "~> 3.0"
-  s.add_runtime_dependency 'bundler', "~> 1.17"
-  s.add_runtime_dependency 'yard', "~> 0.9"
-  s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
+  s.add_runtime_dependency 'addressable', '~> 2.5'
   s.add_runtime_dependency "aws-sdk-core", "< 3"
+  s.add_runtime_dependency 'azure_sdk', "~> 0.52"
+  s.add_runtime_dependency 'bundler', "~> 1.17"
   s.add_runtime_dependency 'chronic_duration', "~> 0.10"
-  s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
-  s.add_runtime_dependency 'optimist', "~> 3.0"
-  s.add_runtime_dependency 'json-schema', "~> 2.8"
-  s.add_runtime_dependency 'colorize', "~> 0.8"
   s.add_runtime_dependency 'color', "~> 1.8"
-  s.add_runtime_dependency 'netaddr', '~> 2.0'
-  s.add_runtime_dependency 'nokogiri', "~> 1.10"
-  s.add_runtime_dependency 'solve', '~> 4.0'
+  s.add_runtime_dependency 'colorize', "~> 0.8"
+  s.add_runtime_dependency 'erubis', "~> 2.7"
+  s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
+  s.add_runtime_dependency 'googleauth', "~> 0.6"
+  s.add_runtime_dependency 'inifile', "~> 3.0"
+  s.add_runtime_dependency 'json-schema', "~> 2.8"
   s.add_runtime_dependency 'net-ldap', "~> 0.16"
   s.add_runtime_dependency 'net-ssh', "~> 4.2"
   s.add_runtime_dependency 'net-ssh-multi', '~> 1.2', '>= 1.2.1'
-  s.add_runtime_dependency 'googleauth', "~> 0.6"
-  s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
+  s.add_runtime_dependency 'netaddr', '~> 2.0'
+  s.add_runtime_dependency 'nokogiri', "~> 1.10"
+  s.add_runtime_dependency 'openssl-oaep', "~> 0.1"
+  s.add_runtime_dependency 'optimist', "~> 3.0"
+  s.add_runtime_dependency 'rack', "~> 2.0"
+  s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
-  s.add_runtime_dependency 'addressable', '~> 2.5'
+  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
-  s.add_runtime_dependency 'azure_sdk', "~> 0.37"
-  s.add_runtime_dependency 'rack', "~> 2.0"
+  s.add_runtime_dependency 'solve', '~> 4.0'
   s.add_runtime_dependency 'thin', "~> 1.7"
-  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'winrm', "~> 2.3", ">= 2.3.4"
+  s.add_runtime_dependency 'yard', "~> 0.9"
 end