cloud-mu 3.1.3 → 3.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5db54ee8a4ea5ea9884c3f468e162e8b27d60de12a0ccc8cbb30006a30818eb
-  data.tar.gz: e6fee0a6d491ebb38a13754d41c9d4df4b7d360f820769f49a07a93cd94ee042
+  metadata.gz: 63e9eb2d1b4db9fe4dc96bfdc81fc6ae47c6e0b6afa1113226ebf4d798a4990a
+  data.tar.gz: 60ea7c4c936d2f66e300705b1cbaaca178476581ad106d05f7f48dac310f2003
 SHA512:
-  metadata.gz: 66ffb8861d10e000a7afda4d7047a32e89f7cd348a43c6058d7cee9fb4a218e7c0db7b97d25d38eb5e08a53ed9003d4916497bb103fec1edb8c45694eddd4add
-  data.tar.gz: d843093658cb63138abc07290274c599f1e084cf39fd905699516bf2b0785a4b7859561084300762f63152fb3d240762032b65acd86ebc0a072b5809166497ab
+  metadata.gz: f7862eacf10e60e1ee974b7a518177baf4d16271724d4ad9b0fd02a27dd3a7a9b5de5caa04aa6ce5bfe171449ffcbebdafd0012c2299c4ac2e39442fbbd0fcfd
+  data.tar.gz: a72c384253bc6b1649060ca33ceaafb28a1f9fb9d5cd699af2bcc63f7b1bbad3efb9ca1b67fbcc08b4fed0efa2d7f533ef3556abb15ed977490912f09e6350de

data/Dockerfile

@@ -4,17 +4,25 @@ RUN mkdir -p /opt/mu/etc/ /home/mu /usr/local/ruby-current/lib/ruby/gems/2.5.0/g
 
 WORKDIR /home/mu
 
+RUN df -h
+
 RUN apt-get update
 
 RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential
 
 RUN apt-get upgrade -y
 
+RUN df -h
+
 COPY ./cloud-mu-*.gem /home/mu
 
-RUN gem install ./cloud-mu-*.gem thin -N
+RUN gem install ./cloud-mu-*.gem -N
+
+RUN df -h
+
+RUN ls -la
 
-RUN rm cloud-mu-*.gem
+#RUN rm --verbose -f cloud-mu-*.gem
 
 RUN apt-get remove -y build-essential ruby2.5-dev
 

data/bin/mu-adopt

@@ -103,7 +103,11 @@ end
 
 
 adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats])
-adoption.scrapeClouds
+found = adoption.scrapeClouds
+if found.nil? or found.empty?
+  MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }
+  exit
+end
 MU.log "Generating baskets", MU::DEBUG
 boks = adoption.generateBaskets(prefix: $opt[:appname])
 

data/bin/mu-load-config.rb

@@ -36,8 +36,7 @@ def validateClouds(cfg)
         cfg[cloud]["default"]["default"] = true
         found_default = true
       else
-        missing_alias = false
-        cfg[cloud].each_pair { |acctalias, acct|
+        cfg[cloud].values.each { |acct|
           if acct["default"]
             if found_default
               puts "Multiple accounts have 'default' set in #{cloud}"
@@ -205,7 +204,7 @@ def cfgPath
     end
   end
   home = Etc.getpwuid(Process.uid).dir
-  username = Etc.getpwuid(Process.uid).name
+  Etc.getpwuid(Process.uid).name # validates existence of a username
   if Process.uid == 0 and !in_gem
     if ENV.include?('MU_INSTALLDIR')
       ENV['MU_INSTALLDIR']+"/etc/mu.yaml"

data/bin/mu-run-tests

@@ -13,29 +13,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+bindir = File.realpath(File.expand_path(File.dirname(__FILE__)))
+dir = File.realpath(File.expand_path(bindir+"/../modules/tests"))
+Dir.chdir(dir)
+
 require 'colorize'
 require 'optimist'
-bindir = File.realpath(File.expand_path(File.dirname(__FILE__)))
 require bindir+"/mu-load-config.rb"
 require 'mu'
 
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'json-schema'
+
 $opts = Optimist::options do
   banner <<-EOS
 Usage:
-#{$0} [-e <environment>] [-r region] [-v] [-d] [-w] [-c] [-n] [-s] [-j] [-p parameter=value] /path/to/stack/config.[json|yaml] [-u deploy_id [-l]] [-r deploy_id]
+#{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]]
   EOS
   opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer
   opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
+  opt :verbose, "Show more information while running", :require => false, :default => false
 end
 
-dir = File.realpath(File.expand_path(bindir+"/../modules/tests"))
-Dir.chdir(dir)
+only = ARGV
 
 files = Dir.glob("*.yaml", base: dir)
 files.concat(Dir.glob("*.yml", base: dir))
 baseclouds = MU::Cloud.supportedClouds.reject { |c| c == "CloudFormation" }
 
 commands = {}
+failures = []
+if only and only.size > 0
+  files.reject! { |f| !only.include?(f) }
+  if files.size == 0
+    MU.log "No files in #{dir} matched requested list", MU::WARN, details: only
+    exit 1
+  end
+end
 
 files.each { |f|
   clouds = baseclouds.dup
@@ -54,12 +71,68 @@ files.each { |f|
     break
   }
   clouds.each { |cloud|
-    commands["mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"] = { "file" => f, "cloud" => cloud }
+    cmd = "mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"
+    commands[cmd] = {
+      "file" => f,
+      "cloud" => cloud,
+    }
+    if $opts[:full]
+      $environment = "dev"
+      begin
+        conf_engine = MU::Config.new(f, cloud: cloud)
+      rescue StandardError => e
+        MU.log e.message+" parsing "+f+" with cloud "+cloud, MU::WARN, details: e.backtrace
+        failures << f+" ("+commands[cmd]["cloud"]+")"
+        next 
+      end
+      parsed = MU::Config.stripConfig(conf_engine.config)
+      types = []
+      MU::Cloud.resource_types.values.each { |attrs|
+        if parsed.has_key?(attrs[:cfg_plural])
+          types << attrs[:cfg_plural]
+        end
+      }
+      commands[cmd]["parsed"] = parsed
+      commands[cmd]["types"] = types
+    end
   }
 }
 
 puts "Running #{commands.size.to_s.bold} #{$opts[:full] ? "full deploy" : "parse"} tests from #{files.size.to_s.bold} Baskets of Kittens across #{baseclouds.size.to_s.bold} clouds"
 
+@output_semaphore = Mutex.new
+
+def execCommand(cmd, results_stash)
+  @output_semaphore.synchronize {
+    puts cmd if $opts[:verbose]
+  }
+
+  ok = true
+  output = %x{#{cmd} 2>&1}
+  ok = false if $?.exitstatus != 0
+
+  results_stash["output"] += output
+
+  @output_semaphore.synchronize {
+    if ok
+      if $opts[:verbose]
+        puts "#{cmd} SUCCEEDED".green
+      else
+        print ".".green
+      end
+    else
+      if $opts[:verbose]
+        puts "#{cmd} FAILED:".light_red
+        puts output
+      else
+        print ".".light_red
+      end
+    end
+  }
+
+  ok
+end
+
 threads = []
 results = {}
 commands.keys.each { |cmd|
@@ -72,21 +145,40 @@ commands.keys.each { |cmd|
   end
 
   threads << Thread.new(cmd) { |cmd_thr|
-    results[cmd_thr] = {}
-    puts %Q{#{cmd_thr} 2>&1}
-    results[cmd_thr]["output"] = %x{#{cmd_thr} 2>&1}
-    if $?.exitstatus != 0
-      results[cmd_thr]["status"] = false
-      print ".".light_red
-    else
-      results[cmd_thr]["status"] = true
-      print ".".green
+    results[cmd_thr] = { "output" => "", "failed" => [] }
+    if !execCommand(cmd_thr, results[cmd_thr])
+      results[cmd_thr]["failed"] << "main"
     end
+
     if $opts[:full] and results[cmd_thr]["output"].match(/deploy - Deployment id: .*? \((.*?)\)/)
       deploy_id = Regexp.last_match[1]
-      results[cmd_thr]["output"] += %x{mu-cleanup #{deploy_id} --skipsnapshots}
-      if $?.exitstatus != 0
-        results[cmd_thr]["status"] = false
+      adoptdir = Dir.mktmpdir(commands[cmd_thr]["file"].gsub(/[^a-z0-9]|yaml$/i, ""))
+      if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
+        adopt = "cd #{adoptdir} && mu-adopt --appname adoptone --grouping omnibus  --clouds #{commands[cmd_thr]["cloud"]} --types #{commands[cmd_thr]["types"].join(" ")} 2>&1"
+        if !execCommand(adopt, results[cmd_thr])
+          results[cmd_thr]["failed"] << "adopt"
+        end
+      end
+
+      if File.exist?(dir+"/regrooms/"+commands[cmd_thr]["file"])
+        regroom = "mu-deploy regrooms/#{commands[cmd_thr]["file"]} --cloud #{commands[cmd_thr]["cloud"]} --update #{deploy_id} 2>&1"
+        if !execCommand(regroom, results[cmd_thr])
+          results[cmd_thr]["failed"] << "regroom"
+        end
+        if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
+          re_adopt = "cd #{adoptdir} && mu-adopt --appname adopttwo --grouping omnibus --clouds #{commands[cmd_thr]["cloud"]} --types #{commands[cmd_thr]["types"].join(" ")} 2>&1"
+          if !execCommand(re_adopt, results[cmd_thr])
+            results[cmd_thr]["failed"] << "second adopt"
+          end
+        end
+# TODO big flex is to read back both adopted BoKs and .diff them, but without
+# all resources having implemented adoption this isn't much of a test yet
+      end
+
+      FileUtils.remove_entry(adoptdir)
+      cleanup = %Q{mu-cleanup #{deploy_id} --skipsnapshots}
+      if !execCommand(cleanup, results[cmd_thr])
+        results[cmd_thr]["failed"] << "cleanup"
       end
     end
   }
@@ -96,21 +188,14 @@ threads.each { |t|
 }
 puts ""
 
-failures = []
 results.keys.sort { |a, b|
-      if results[b]["status"] and !results[a]["status"]
-        1
-      elsif !results[b]["status"] and results[a]["status"]
-        -1
-      else
-        0
-      end
+      results[b]["failed"].size <=> results[a]["failed"].size
     }.each { |cmd|
-  if !results[cmd]["status"]
+  if results[cmd]["failed"].size > 0
     puts cmd+" failed:".light_red
     puts results[cmd]["output"].yellow
     puts "^ #{cmd}".light_red
-    failures << commands[cmd]["file"]+" ("+commands[cmd]["cloud"]+")"
+    failures << commands[cmd]["file"]+" in "+commands[cmd]["cloud"]+" ("+results[cmd]["failed"].join(", ")+")"
   else
     puts cmd+" passed".green
   end

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.1.3'
-  s.date        = '2020-01-31'
+  s.version     = '3.1.4'
+  s.date        = '2020-02-14'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -35,31 +35,31 @@ EOF
   s.homepage    =
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'
-  s.add_runtime_dependency 'erubis', "~> 2.7"
-  s.add_runtime_dependency 'inifile', "~> 3.0"
-  s.add_runtime_dependency 'bundler', "~> 1.17"
-  s.add_runtime_dependency 'yard', "~> 0.9"
-  s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
+  s.add_runtime_dependency 'addressable', '~> 2.5'
   s.add_runtime_dependency "aws-sdk-core", "< 3"
+  s.add_runtime_dependency 'azure_sdk', "~> 0.52"
+  s.add_runtime_dependency 'bundler', "~> 1.17"
   s.add_runtime_dependency 'chronic_duration', "~> 0.10"
-  s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
-  s.add_runtime_dependency 'optimist', "~> 3.0"
-  s.add_runtime_dependency 'json-schema', "~> 2.8"
-  s.add_runtime_dependency 'colorize', "~> 0.8"
   s.add_runtime_dependency 'color', "~> 1.8"
-  s.add_runtime_dependency 'netaddr', '~> 2.0'
-  s.add_runtime_dependency 'nokogiri', "~> 1.10"
-  s.add_runtime_dependency 'solve', '~> 4.0'
+  s.add_runtime_dependency 'colorize', "~> 0.8"
+  s.add_runtime_dependency 'erubis', "~> 2.7"
+  s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
+  s.add_runtime_dependency 'googleauth', "~> 0.6"
+  s.add_runtime_dependency 'inifile', "~> 3.0"
+  s.add_runtime_dependency 'json-schema', "~> 2.8"
   s.add_runtime_dependency 'net-ldap', "~> 0.16"
   s.add_runtime_dependency 'net-ssh', "~> 4.2"
   s.add_runtime_dependency 'net-ssh-multi', '~> 1.2', '>= 1.2.1'
-  s.add_runtime_dependency 'googleauth', "~> 0.6"
-  s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
+  s.add_runtime_dependency 'netaddr', '~> 2.0'
+  s.add_runtime_dependency 'nokogiri', "~> 1.10"
+  s.add_runtime_dependency 'optimist', "~> 3.0"
+  s.add_runtime_dependency 'rack', "~> 2.0"
+  s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
-  s.add_runtime_dependency 'addressable', '~> 2.5'
+  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
-  s.add_runtime_dependency 'azure_sdk', "~> 0.37"
-  s.add_runtime_dependency 'rack', "~> 2.0"
+  s.add_runtime_dependency 'solve', '~> 4.0'
   s.add_runtime_dependency 'thin', "~> 1.7"
-  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'yard', "~> 0.9"
 end

data/cookbooks/mu-tools/libraries/helper.rb

@@ -49,7 +49,6 @@ module Mutools
     @authorizer = nil
     def set_gcp_cfg_params
       begin
-        require "google/cloud"
         require "googleauth"
         @project ||= get_google_metadata("project/project-id")
         @authorizer ||= ::Google::Auth.get_application_default(['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'])
@@ -202,6 +201,8 @@ module Mutools
           Chef::Log.info("Fetching deploy secret: #{gsutil} cp gs://#{bucket}/#{filename} -")
           cmd = if File.exist?("/usr/bin/python2.7")
             %Q{CLOUDSDK_PYTHON=/usr/bin/python2.7 #{gsutil} cp gs://#{bucket}/#{filename} -}
+          elsif File.exist?("/opt/rh/python27/root/usr/bin/python")
+            %Q{CLOUDSDK_PYTHON=/opt/rh/python27/root/usr/bin/python #{gsutil} cp gs://#{bucket}/#{filename} -}
           else
             %Q{#{gsutil} cp gs://#{bucket}/#{filename} -}
           end

data/cookbooks/mu-tools/libraries/monkey.rb

@@ -0,0 +1,35 @@
+class Chef
+  class Provider
+    class Package
+      class Rubygems < Chef::Provider::Package
+
+        def install_via_gem_command(name, version)
+          src = []
+          if new_resource.source.is_a?(String) && new_resource.source =~ /\.gem$/i
+            name = new_resource.source
+          else
+            src << "--clear-sources" if new_resource.clear_sources
+            src += gem_sources.map { |s| "--source=#{s}" }
+          end
+          src_str = src.empty? ? "" : " #{src.join(" ")}"
+          cmd = if !version.nil? && !version.empty?
+            "#{gem_binary_path} install #{name} -q --no-rdoc --no-ri -v \"#{version}\"#{src_str}#{opts}"
+          else
+            "#{gem_binary_path} install \"#{name}\" -q --no-rdoc --no-ri #{src_str}#{opts}"
+          end
+
+          begin
+            shell_out_with_timeout!(cmd, env: nil)
+          rescue StandardError => e
+            if cmd.match(/--no-rdoc|--no-ri/)
+              cmd.gsub!(/--no-rdoc --no-ri/, "--no-document")
+              retry
+            end
+            raise e
+          end
+        end
+
+      end
+    end
+  end
+end

data/cookbooks/mu-tools/recipes/google_api.rb

@@ -16,10 +16,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-["google-api-client", "google-cloud", "googleauth"].each { |gem|
+["google-api-client", "googleauth"].each { |gem|
   chef_gem gem do
     compile_time true
     action :install
-		only_if { !get_google_metadata("name").nil? }
+		only_if { !get_google_metadata("instance/name").nil? }
   end
 }

data/cookbooks/mu-tools/resources/disk.rb

@@ -15,7 +15,7 @@ action :create do
   devicename = device
 
   if set_gcp_cfg_params
-    devicename.gsub!(/.*?\//, "")
+    devicename= devicename.gsub(/.*?\//, "")
     device = "/dev/disk/by-id/google-"+devicename
   end
 

data/extras/image-generators/Google/centos6.yaml

@@ -16,3 +16,4 @@
     create_image:
       image_then_destroy: true
       public: true
+      family: mu-centos-6

data/extras/image-generators/Google/centos7.yaml

@@ -4,7 +4,6 @@
   - name: centos7
     cloud: Google
     image_id: "centos-cloud/centos-7"
-    platform: centos6
     ssh_user: centos
     size: g1-small
     associate_public_ip: true
@@ -16,3 +15,4 @@
     create_image:
       image_then_destroy: true
       public: true
+      family: mu-centos-7

data/modules/mommacat.ru

@@ -16,7 +16,7 @@ require 'pp'
 require 'base64'
 require 'etc'
 
-home = Etc.getpwuid(Process.uid).dir
+Etc.getpwuid(Process.uid).dir
 
 if !ENV.include?('MU_INSTALLDIR')
   ENV['MU_INSTALLDIR'] = "/opt/mu"
@@ -52,7 +52,7 @@ end
 
 begin
   MU::MommaCat.syncMonitoringConfig(false)
-rescue Exception => e
+rescue StandardError => e
   MU.log e.inspect, MU::ERR, details: e.backtrace
   # ...but don't die!
 end
@@ -64,14 +64,12 @@ Thread.new {
     MU::MommaCat.cleanTerminatedInstances
     MU::Master.cleanExpiredScratchpads if $ENABLE_SCRATCHPAD
     sleep 60
-  rescue Exception => e
+  rescue StandardError => e
     MU.log "Error in cleanTerminatedInstances thread: #{e.inspect}", MU::ERR, details: e.backtrace
     retry
   end while true
 }
 
-required_vars = ["mu_id", "mu_deploy_secret", "mu_resource_name", "mu_resource_type", "mu_instance_id"]
-
 # Use a template to generate a pleasant-looking HTML page for simple messages
 # and errors.
 def genHTMLMessage(title: "", headline: "", msg: "", template: $MU_CFG['html_template'], extra_vars: {})
@@ -193,7 +191,6 @@ def releaseKitten(mu_id)
 end
 
 app = proc do |env|
-  ok = false
   returnval = [
       200,
       {
@@ -318,7 +315,7 @@ app = proc do |env|
         elsif !filter or !path
           returnval = throw404 env['REQUEST_PATH']
         else
-          MU::MommaCat.addInstanceToEtcHosts(path, filter)
+          MU::Master.addInstanceToEtcHosts(path, filter)
           returnval = [
             200,
             {
@@ -356,7 +353,6 @@ app = proc do |env|
 
     elsif !env["rack.input"].nil?
       req = Rack::Utils.parse_nested_query(env["rack.input"].read)
-      ok = true
 
       if req["mu_user"].nil?
         req["mu_user"] = "mu"
@@ -373,7 +369,6 @@ app = proc do |env|
       kittenpile = getKittenPile(req)
       if kittenpile.nil? or kittenpile.original_config.nil? or kittenpile.original_config[req["mu_resource_type"]+"s"].nil?
         returnval = throw500 "Couldn't find config data for #{req["mu_resource_type"]} in deploy_id #{req["mu_id"]}"
-        ok = false
         next
       end
       server_cfg = nil
@@ -385,7 +380,6 @@ app = proc do |env|
       }
       if server_cfg.nil?
         returnval = throw500 "Couldn't find config data for #{req["mu_resource_type"]} name: #{req["mu_resource_name"]} deploy_id: #{req["mu_id"]}"
-        ok = false
         next
       end
 
@@ -426,7 +420,6 @@ MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
           instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
         else
           returnval = throw500 "I don't know how to add a volume for #{instance}"
-          ok = false
         end
       elsif !instance.nil?
         if !req["mu_bootstrap"].nil?
@@ -434,16 +427,13 @@ MU.log "ADDVOLUME REQUEST", MU::WARN, details: params
           returnval[2] = ["Grooming asynchronously, check Momma Cat logs on the master for details."]
         else
           returnval = throw500 "Didn't get 'mu_bootstrap' parameter from instance id '#{req["mu_instance_id"]}'"
-          ok = false
         end
       else
         returnval = throw500 "No such instance id '#{req["mu_instance_id"]}' nor was this an SSL signing request"
-        ok = false
       end
     end
-  rescue Exception => e
+  rescue StandardError => e
     returnval = throw500 "Invalid request: #{e.inspect} (#{req})", e.backtrace
-    ok = false
   ensure
     if !req.nil?
       releaseKitten(req['mu_id'])