cloud-mu 2.1.0beta → 3.0.0beta

data/bin/mu-deploy

@@ -42,6 +42,8 @@ Usage:
   opt :cloudformation_output, "When emitting a CloudFormation template, put the final product in this location instead of in /tmp. Takes a local file path or an s3:// URI. S3 uploads will be given AUTHENTICATED-READ permissions.", :require => false, :type => :string
   opt :verbose, "Display debugging output.", :require => false, :default => false, :type => :boolean
   opt :quiet, "Display minimal output.", :require => false, :default => false, :type => :boolean
+  opt :color, "Display log output in human-friendly colors.", :require => false, :default => true, :type => :boolean
+  opt :credentials, "Set the default credential set to use for resources which do not specify a default", :require => false, :type => :string
 end
 verbosity = MU::Logger::NORMAL
 verbosity = MU::Logger::LOUD if $opts[:verbose]
@@ -58,7 +60,7 @@ if $opts[:liveupdate] and !$opts[:update]
 end
 
 MU.setVar("curRegion", $opts[:region]) if $opts[:region]
-MU.setLogging(verbosity, $opts[:web])
+MU.setLogging(verbosity, $opts[:web], STDOUT, $opts[:color])
 
 # Parse any paramater options into something useable.
 params = Hash.new
@@ -87,11 +89,11 @@ end
 
 MU.log "Loading #{config}", html: $opts[:web], details: $opts
 
-conf_engine = MU::Config.new(config, $opts[:skipinitialupdates], params: params, updating: $opts[:update])
+conf_engine = MU::Config.new(config, $opts[:skipinitialupdates], params: params, updating: $opts[:update], default_credentials: $opts[:credentials])
 stack_conf = conf_engine.config
 
 if $opts[:dryrun] or $opts[:verbose]
-  puts stack_conf.to_yaml
+  puts MU::Config.stripConfig(stack_conf).to_yaml
   conf_engine.visualizeDependencies
 end
 
@@ -112,6 +114,7 @@ if $opts[:dryrun]
       cost_dummy_deploy = MU::Deploy.new(
         $opts[:environment],
         verbosity: MU::Logger::SILENT,
+        color: $opts[:color],
         force_cloudformation: true,
         cloudformation_path: "/dev/null",
         nocleanup: false,
@@ -135,6 +138,7 @@ if $opts[:update]
   deployer = MU::Deploy.new(
     deploy.environment,
     verbosity: verbosity,
+    color: $opts[:color],
     webify_logs: $opts[:web],
     nocleanup: true, # don't accidentally blow up an existing deploy
     stack_conf: stack_conf,
@@ -156,6 +160,7 @@ end
 deployer = MU::Deploy.new(
   $opts[:environment],
   verbosity: verbosity,
+  color: $opts[:color],
   webify_logs: $opts[:web],
   nocleanup: $opts[:nocleanup],
   cloudformation_path: cfm_path,

data/bin/mu-gcp-setup

@@ -31,6 +31,7 @@ require 'erb'
 require 'optimist'
 require 'json-schema'
 require 'mu'
+require 'mu/master/ssl'
 Dir.chdir(MU.installDir)
 
 $opts = Optimist::options do
@@ -64,7 +65,7 @@ my_instance_id = MU::Cloud::AWS.getAWSMetaData("instance-id")
 
 if MU::Cloud::Google.hosted?
   instance = MU.myCloudDescriptor
-  admin_sg_name = "mu-master-"+MU.myInstanceId+"-ingress-allow"
+  admin_sg_name = MU.myInstanceId+"-"+MU.myVPC+"-ingress-allow"
   if !instance.tags.items or !instance.tags.items.include?(admin_sg_name)
     newitems = instance.tags.items ? instance.tags.items.dup : []
     newitems << admin_sg_name
@@ -87,9 +88,10 @@ end
 # Create a security group, or manipulate an existing one, so that we have all
 # of the appropriate network holes.
 if $opts[:sg]
-  open_ports = [80, 443, 2260, 7443, 8443, 9443, 8200]
+  open_ports = [80, 443, MU.mommaCatPort, 7443, 8443, 9443, 8200]
 
   found = MU::MommaCat.findStray("Google", "firewall_rule", dummy_ok: true, cloud_id: admin_sg_name)
+  found.reject! { |v| v.cloud_desc.network != MU.myVPC }
   admin_sg = found.first if !found.nil? and found.size > 0
 
   rules = []
@@ -108,21 +110,27 @@ if $opts[:sg]
   }
   cfg = {
     "name" => admin_sg_name,
+    "scrub_mu_isms" => true,
     "cloud" => "Google",
     "rules" => rules,
     "project" => MU::Cloud::Google.myProject,
     "target_tags" => [admin_sg_name],
     "vpc" => {
-      "vpc_id" => instance.network_interfaces.first.network
+      "id" => MU.myVPC
     }
   }
 
   if !admin_sg
-    admin_sg = MU::Cloud::FirewallRule.new(kitten_cfg: cfg, mu_name: "mu-master-"+MU.myInstanceId)
-    admin_sg.create
-    admin_sg.groom
+    admin_sg = MU::Cloud::FirewallRule.new(kitten_cfg: cfg, mu_name: admin_sg_name)
+    begin
+      admin_sg.create
+    rescue ::Google::Apis::ClientError => e
+      raise e if !e.message.match(/alreadyExists: /)
+    ensure
+      admin_sg.groom
+    end
   else
-# TODO Make sure the rules and tags are up to date
+    admin_sg.groom
   end
 
 end
@@ -208,7 +216,7 @@ end
 if $opts[:uploadlogs]
   today = Time.new.strftime("%Y%m%d").to_s
   ["master.log", "nodes.log"].each { |log|
-    if File.exists?("/Mu_Logs/#{log}-#{today}")
+    if File.exist?("/Mu_Logs/#{log}-#{today}")
       MU.log "Uploading /Mu_Logs/#{log}-#{today} to bucket #{$bucketname}"
       MU::Cloud::AWS.s3.put_object(
           bucket: $bucketname,

data/bin/mu-gen-docs

@@ -20,10 +20,9 @@ end
 
 require 'rubygems'
 require 'bundler/setup'
-require 'json'
 require 'erb'
-require 'optimist'
-require 'json-schema'
+require 'tempfile'
+require 'fileutils'
 require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
 require 'mu'
 
@@ -43,19 +42,104 @@ rescue
   yard = File.dirname(Gem.ruby)+'/yard'
 end
 
-
+docdir = Process.uid == 0 ? "/var/www/html/docs" : MU.dataDir+"/docs"
+if !Dir.exist?(docdir)
+  FileUtils.mkdir_p(docdir, mode: 0755)
+end
 
 MU::Config.emitSchemaAsRuby
-MU.log "Generating YARD documentation in /var/www/html/docs (see http://#{$MU_CFG['public_address']}/docs/frames.html)"
+if Process.uid == 0
+  MU.log "Generating YARD documentation in #{docdir} (see http://#{$MU_CFG['public_address']}/docs/frames.html)"
+else
+  MU.log "Generating YARD documentation in #{docdir}"
+end
 
 File.umask 0022
 
 Dir.chdir(MU.myRoot) do
-  #exec "env -i PATH=#{ENV['PATH']} HOME=#{ENV['HOME']} #{yard} doc modules -m markdown -o /var/www/html/docs" THIS DOESNT WORK IN CICD
+  imgs = %w{alpha.png beta.png release.png}
+  FileUtils.cp(imgs.map { |f| "extras/"+f }, docdir)
+  FileUtils.chmod(0644, imgs.map { |f| docdir+"/"+f })
+
+  readme = File.read("README.md")
+  readme += <<EOF
+
+# Supported resources
+EOF
 
-  system(%Q{#{yard} doc modules -m markdown -o /var/www/html/docs})
+  impl_counts = {}
+  cloudlist = MU::Cloud.supportedClouds.sort { |a, b|
+    counts = {
+      a => 0,
+      b => 0
+    }
+    MU::Cloud.resource_types.each_pair { |type, cfg|
+      impl_counts[type] ||= 0
+      [a, b].each { |cloud|
+        begin
+          myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
+          case myclass.quality
+          when MU::Cloud::RELEASE
+            counts[cloud] += 4
+            impl_counts[type] += 4
+          when MU::Cloud::BETA
+            counts[cloud] += 2
+            impl_counts[type] += 2
+          when MU::Cloud::ALPHA
+            counts[cloud] += 1
+            impl_counts[type] += 1
+          end
+        rescue MU::Cloud::MuCloudResourceNotImplemented
+        end
+      }
+    }
+    counts[b] <=> counts[a]
+  }
 
-  system(%Q{chcon -R -h -t httpd_sys_script_exec_t /var/www/html/})
+  readme += "\n\n<table><tr><th></th>"
+  cloudlist.each { |cloud|
+    readme += "<th>"+cloud+"</th>"
+  }
+  readme += "</tr>\n"
 
+  icon_style = 'height:2.2em;width:2.2em;padding:0px;'
+  MU::Cloud.resource_types.keys.sort { |a, b| impl_counts[b] <=> impl_counts[a] }.each { |type|
+    readme += "<tr><td><strong>{MU::Config::BasketofKittens::#{MU::Cloud.resource_types[type][:cfg_plural]} #{type.to_s}}</strong></td>"
+    cloudlist.each { |cloud|
+      readme += "<td><center>"
+      begin
+        myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
+        case myclass.quality
+        when MU::Cloud::RELEASE
+          readme += "<img src='release.png' style='#{icon_style}' title='Release Quality' alt='[Release Quality]'>"
+        when MU::Cloud::BETA
+          readme += "<img src='beta.png' style='#{icon_style}' title='Beta Quality' alt='[Beta Quality]'>"
+        when MU::Cloud::ALPHA
+          readme += "<img src='alpha.png' style='#{icon_style}' title='Alpha Quality' alt='[Alpha Quality]'>"
+        else
+          readme += "?"
+        end
+      rescue MU::Cloud::MuCloudResourceNotImplemented
+        readme += "-"
+      end
+      readme += "</center></td>"
+    }
+    readme += "</tr>\n"
+  }
+  readme += "</table>\n\n"
+
+  idx = Tempfile.new('mu-gen-docs-index', MU.myRoot)
+  idx.write(readme)
+  idx.rewind
+  idx.close
+  idx_short = idx.path.gsub(/.*?\/([^\/]+)$/, '\1')
+
+  system(%Q{#{yard} doc modules --readme #{idx_short} --markup markdown --output-dir #{docdir}})
+  # --exclude seems to just... not work
+  FileUtils.rm(docdir+"/file."+idx_short+".html")
+
+  if Process.uid == 0
+    system(%Q{chcon -R -h -t httpd_sys_script_exec_t /var/www/html/})
+  end
   system(%Q{#{yard} stats --list-undoc modules})
 end

data/bin/mu-load-config.rb

@@ -64,6 +64,7 @@ end
 # into the Ruby $LOAD_PATH.
 # @return [Hash]
 def loadMuConfig(default_cfg_overrides = nil)
+
   # Start with sane defaults
   default_cfg = {
     "installdir" => "/opt/mu",
@@ -109,9 +110,25 @@ def loadMuConfig(default_cfg_overrides = nil)
       "dcs" => ["127.0.0.1"]
     }
   }
+
+  in_gem = (Gem.paths and Gem.paths.home and File.dirname(__FILE__).match(/^#{Gem.paths.home}/))
+
+  if in_gem
+    default_cfg.delete("ldap")
+    default_cfg.delete("ssl")
+    default_cfg.delete("scratchpad")
+    default_cfg.delete("libdir")
+    default_cfg.delete("installdir")
+  else
+    if File.exist?("/opt/mu/etc/mu.yaml")
+      default_cfg.merge!(YAML.load(File.read("/opt/mu/etc/mu.yaml")))
+      default_cfg["config_files"] = ["/opt/mu/etc/mu.yaml"]
+    end
+  end
+
   default_cfg.merge!(default_cfg_overrides) if default_cfg_overrides
 
-  if !File.exists?(cfgPath) and Process.uid == 0
+  if !File.exist?(cfgPath) and Process.uid == 0
     puts "**** Master config #{cfgPath} does not exist, initializing *****"
     File.open(cfgPath, File::CREAT|File::TRUNC|File::RDWR, 0644){ |f|
       f.puts default_cfg.to_yaml
@@ -119,7 +136,7 @@ def loadMuConfig(default_cfg_overrides = nil)
   end
 
   global_cfg = { "config_files" => [] }
-  if File.exists?(cfgPath)
+  if File.exist?(cfgPath)
     global_cfg = YAML.load(File.read(cfgPath))
     global_cfg["config_files"] = [cfgPath]
   end
@@ -127,33 +144,39 @@ def loadMuConfig(default_cfg_overrides = nil)
   home = Etc.getpwuid(Process.uid).dir
   username = Etc.getpwuid(Process.uid).name
   if File.readable?("#{home}/.mu.yaml") and cfgPath != "#{home}/.mu.yaml"
-    global_cfg.merge!(YAML.load(File.read("#{home}/.mu.yaml")))
-    global_cfg["config_files"] << "#{home}/.mu.yaml"
+    localfile = YAML.load(File.read("#{home}/.mu.yaml"))
+    if localfile
+      global_cfg.merge!(localfile)
+      global_cfg["config_files"] << "#{home}/.mu.yaml"
+    end
   end
   if !global_cfg.has_key?("installdir")
     if ENV['MU_INSTALLDIR']
       global_cfg["installdir"] = ENV['MU_INSTALLDIR']
-    elsif Gem.paths and Gem.paths.home and !Dir.exists?("/opt/mu/lib")
-      global_cfg["installdir"] = File.realpath(File.expand_path(File.dirname(Gem.paths.home))+"/../../../")
-    else
+    elsif !in_gem
       global_cfg["installdir"] = "/opt/mu"
     end
   end
   if !global_cfg.has_key?("libdir")
     if ENV['MU_INSTALLDIR']
       global_cfg["libdir"] = ENV['MU_INSTALLDIR']+"/lib"
-    else
+    elsif !in_gem
       global_cfg["libdir"] = File.realpath(File.expand_path(File.dirname(__FILE__))+"/..")
     end
   end
   if !global_cfg.has_key?("datadir")
     if username != "root"
-      global_cfg["datadir"] = home+"/.mu"
+      global_cfg["datadir"] = home+"/.mu/var"
     elsif global_cfg.has_key?("installdir")
       global_cfg["datadir"] = global_cfg["installdir"]+"/var"
     else
       global_cfg["datadir"] = "/opt/mu/var"
     end
+    default_cfg["ssl"] = {
+      "cert" => global_cfg["datadir"]+"/ssl/mommacat.crt",
+      "key" => global_cfg["datadir"]+"/ssl/mommacat.key",
+      "chain" => global_cfg["datadir"]+"/ssl/Mu_CA.pem"
+    }
   end
 
   exit 1 if !validateClouds(global_cfg)
@@ -164,12 +187,29 @@ end
 
 # Shorthand for locating the path to mu.yaml
 def cfgPath
+  in_gem = false
+  gemwhich = %x{gem which mu 2>&1}.chomp
+  gemwhich = nil if $?.exitstatus != 0
+  mypath = File.realpath(File.expand_path(File.dirname(__FILE__)))
+  if !mypath.match(/^\/opt\/mu/)
+    if Gem.paths and Gem.paths.home and
+       (mypath.match(/^#{Gem.paths.home}/) or gemwhich.match(/^#{Gem.paths.home}/))
+      in_gem = true
+    elsif $?.exitstatus == 0 and gemwhich and !gemwhich.empty?
+      $LOAD_PATH.each { |path|
+        if path.match(/\/cloud-mu-[^\/]+\/modules/) or
+           path.match(/#{Regexp.quote(gemwhich)}/)
+          in_gem = true
+        end
+      }
+    end
+  end
   home = Etc.getpwuid(Process.uid).dir
   username = Etc.getpwuid(Process.uid).name
-  if Process.uid == 0
+  if Process.uid == 0 and !in_gem
     if ENV.include?('MU_INSTALLDIR')
       ENV['MU_INSTALLDIR']+"/etc/mu.yaml"
-    elsif Dir.exists?("/opt/mu")
+    elsif Dir.exist?("/opt/mu")
       "/opt/mu/etc/mu.yaml"
     else
       "#{home}/.mu.yaml"
@@ -180,7 +220,7 @@ def cfgPath
 end
 
 def cfgExists?
-  File.exists?(cfgPath)
+  File.exist?(cfgPath)
 end
 
 # Output an in-memory configuration hash to the standard config file location,

data/bin/mu-momma-cat

@@ -0,0 +1,36 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require File.expand_path(File.dirname(__FILE__))+"/mu-load-config.rb"
+require 'rubygems'
+require 'bundler/setup'
+require 'mu'
+
+if ARGV.size > 1 or !["start", "stop", "restart"].include?(ARGV[0])
+  MU.log "Expecting start, stop, or restart argument", MU::ERR
+  exit 1
+end
+
+
+case ARGV[0]
+when "start"
+  MU::MommaCat.start
+when "stop"
+  MU::MommaCat.stop
+when "restart"
+  MU::MommaCat.restart
+when "status"
+  MU::MommaCat.status
+end

data/bin/mu-node-manage

@@ -182,36 +182,38 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
   deploys.each { |muid|
     mommacat = MU::MommaCat.new(muid)
     next if mommacat.kittens.nil? or mommacat.kittens['servers'].nil?
-    mommacat.kittens['servers'].each_pair { |nodeclass, servers|
-      servers.each_pair { |mu_name, server|
-        next if nodes.size > 0 and !nodes.include?(mu_name)
-        count = count + 1
-        child = Process.fork {
-          begin
-            type = "server"
-            type = "server_pool" if server.config.has_key?("basis")
-            if vaults_only
-              next if !server.config.has_key?("vault_access")
-              server.config["vault_access"].each { |v|
-                MU::Groomer::Chef.grantSecretAccess(mu_name, v['vault'], v['item'])
-              }
-            else
-              mommacat.groomNode(server.cloud_id, nodeclass, type, mu_name: mu_name)
+    mommacat.kittens['servers'].each_pair { |habitat, nodeclasses|
+      nodeclasses.each_pair { |nodeclass, servers|
+        servers.each_pair { |mu_name, server|
+          next if nodes.size > 0 and !nodes.include?(mu_name)
+          count = count + 1
+          child = Process.fork {
+            begin
+              type = "server"
+              type = "server_pool" if server.config.has_key?("basis")
+              if vaults_only
+                next if !server.config.has_key?("vault_access")
+                server.config["vault_access"].each { |v|
+                  MU::Groomer::Chef.grantSecretAccess(mu_name, v['vault'], v['item'])
+                }
+              else
+                mommacat.groomNode(server.cloud_id, nodeclass, type, mu_name: mu_name)
+              end
+            rescue Exception => e
+              MU.log e.inspect, MU::ERR, details: e.backtrace
+              exit 1
             end
-          rescue Exception => e
-            MU.log e.inspect, MU::ERR, details: e.backtrace
-            exit 1
-          end
+          }
+          $children[child] = mu_name
         }
-        $children[child] = mu_name
-      }
-      while $children.size >= $opts[:concurrent]-1
-        child = Process.wait
-        if !$?.success?
-          badnodes << $children[child]
+        while $children.size >= $opts[:concurrent]-1
+          child = Process.wait
+          if !$?.success?
+            badnodes << $children[child]
+          end
+          $children.delete(child)
         end
-        $children.delete(child)
-      end
+      }
     }
   }
   Process.waitall.each { |child|
@@ -240,6 +242,10 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
         done = false
         begin
           serverobj = mommacat.findLitterMate(type: "server", mu_name: nodename)
+          if !serverobj
+            MU.log "Failed to load server object for #{nodename}", MU::ERR
+            next
+          end
 
           # Generate the command if attemting a chef run
           if chefrun
@@ -419,6 +425,7 @@ def updateAWSMetaData(deploys = MU::MommaCat.listDeploys, nodes = [])
               "deploySSHKey" => mommacat.ssh_public_key,
               "muID" => muid,
               "muUser" => MU.chef_user,
+              "mommaCatPort" => MU.mommaCatPort,
               "publicIP" => MU.mu_public_ip,
               "resourceName" => svr_class,
               "windowsAdminName" => server['windows_admin_username'],