cloud-mu 3.2.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd9a15a0e94a578919c0cb22bf5c95ee86b5b4a03fcc0ea1c35da2c161e8cf31
-  data.tar.gz: 04cdc9d9a70de97fdaf12021166b707921ec981d7ab2dc92cd1ebce491a9e130
+  metadata.gz: 210514240124518b016fcafc33868f4efcbd890764614b535a3460fd7736961e
+  data.tar.gz: 22982dbf157d8c94ecb38c9ca7ee9209dcdef516158ecd1835461da255b7a3df
 SHA512:
-  metadata.gz: adfef4f231a946a3929b7f8026a9e338658dadfef3482942a46e8fdb9b10c4ca7d68aa457d135815a3878882d88a103ec1ebacea4bcb1ae4c45b070d85ceee06
-  data.tar.gz: 3a7b04d7d2486b05e8d413b78029b610bb7ba6bd8a33a1c24cf7b8bdf63dabaf9e0a1baab8408a66f995d05df719261248cbf27ee5b230a350109053ddb0420a
+  metadata.gz: 412580d1d702cf61dcd3671bee157b787c0a4ffd79b2c57d845b29bb93cf71da092910209b1ba0a7ff7f98d5a37b21f0b542a99cc31e479b8d4a45b1fa778622
+  data.tar.gz: 55fe915449c29467c2731736b357c35a5aef4af243b1c57e06a986273e27eed1726769ce0982d0e6202888161df0664fd10b8179697b13b73c90b3e02f4bcf45

data/Dockerfile

@@ -8,7 +8,7 @@ RUN df -h
 
 RUN apt-get update
 
-RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl openssh-client
 
 RUN apt-get upgrade -y
 

data/ansible/roles/mu-nat/tasks/main.yml

@@ -9,6 +9,9 @@
     name: iptables-services
     state: present
 
+- name: modprobe br_netfilter
+  command: /sbin/modprobe br_netfilter
+
 - name: Enable ip_forward
   sysctl:
     name: net.ipv4.ip_forward

data/bin/mu-adopt

@@ -45,6 +45,7 @@ $opt = Optimist::options do
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
   opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
   opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
+  opt :pattern, "Only adopt resources whose resource name would match this pattern. Must be a valid regular expression. Alphabetical characters will be treated case-insensitively.", :required => false, :type => :string
 end
 
 ok = true
@@ -60,6 +61,16 @@ if $opt[:diff]
   $opt[:savedeploys] = false
 end
 
+pattern = nil
+if $opt[:pattern]
+  begin
+    pattern = Regexp.new($opt[:pattern], true)
+  rescue RegexpError => e
+    MU.log "Invalid --pattern option: #{e.message}", MU::ERR
+    exit 1
+  end
+end
+
 types = []
 $opt[:types].each { |t|
   t_name = t.gsub(/-/, "_")
@@ -98,7 +109,7 @@ if !ok
   exit 1
 end
 
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes])
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes], pattern: pattern)
 found = adoption.scrapeClouds
 if found.nil? or found.empty?
   MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }

data/bin/mu-aws-setup

@@ -44,6 +44,7 @@ Usage:
   opt :dns, "Ensure the presence of a private DNS Zone called for internal amongst Mu resources.", :require => false, :default => false, :type => :boolean
   opt :uploadlogs, "Push today's log files to the S3 bucket created by the -l option.", :require => false, :default => false, :type => :boolean
   opt :ephemeral, "Make sure all of our instance store (ephemeral) block devices are mapped and available.", :require => false, :default => false, :type => :boolean
+  opt :optdisk, "Create an EBS volume for /opt and slide our installation onto it", :require => false, :default => false, :type => :boolean
 end
 
 if MU::Cloud::AWS.hosted? and !$MU_CFG['aws']
@@ -70,7 +71,9 @@ preferred_ip = MU.mu_public_ip
 
 if $opts[:ephemeral] and !MU::Cloud::AWS.isGovCloud?
   instancetypes = MU::Cloud::AWS.listInstanceTypes
-  if instancetypes[MU.myRegion][instance.instance_type]["storage"] == "EBS only"
+  if !instancetypes or !instancetypes[MU::Cloud::AWS.myRegion] or !instancetypes[MU::Cloud::AWS.myRegion][instance.instance_type]
+    MU.log "Failed to load instance type mappings from Pricing API for #{instance.instance_type} in #{MU::Cloud::AWS.myRegion}", MU::ERR
+  elsif instancetypes[MU::Cloud::AWS.myRegion][instance.instance_type]["storage"] == "EBS only"
     MU.log "#{instance.instance_type} instance types do not have ephemeral volumes, skipping ephemeral device setup", MU::NOTICE
   else
 #    instance.block_device_mappings.each { |dev|
@@ -101,7 +104,7 @@ if $opts[:sg]
   admin_sg = nil
   if instance.security_groups.size > 0
     instance.security_groups.each { |sg|
-      found = MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, cloud_id: sg.group_id)
+      found = MU::MommaCat.findStray("AWS", "firewall_rule", region: MU::Cloud::AWS.myRegion, dummy_ok: true, cloud_id: sg.group_id)
       if found.size > 0 and
          !found.first.cloud_desc.group_name.match(/^Mu Client Rules for /)
         admin_sg = found.first
@@ -118,7 +121,7 @@ if $opts[:sg]
          !ranges.include?(range.cidr_ip) and rule.to_port != 80 and
          !(rule.to_port == 22 and range.cidr_ip == "#{preferred_ip}/32")
         MU.log "Revoking old Mu Master service access rule for #{range.cidr_ip} port #{rule.to_port.to_s}", MU::NOTICE
-        MU::Cloud::AWS.ec2(region: MU.myRegion, credentials: admin_sg.credentials).revoke_security_group_ingress(
+        MU::Cloud::AWS.ec2(region: MU::Cloud::AWS.myRegion, credentials: admin_sg.credentials).revoke_security_group_ingress(
           group_id: admin_sg.cloud_desc.group_id,
           ip_permissions: [
             {
@@ -173,7 +176,7 @@ if $opts[:sg]
     cfg = {
       "name" => "Mu Master",
       "cloud" => "AWS",
-      "region" => MU.myRegion,
+      "region" => MU::Cloud::AWS.myRegion,
       "rules" => rules
     }
 
@@ -233,7 +236,36 @@ elsif $opts[:ip]
   MU.log "Currently assigned IP address is #{instance.public_ip_address}"
 end
 
-$bucketname = MU.adminBucketName
+if $opts[:optdisk] and !File.open("/etc/mtab").read.match(/ \/opt[\s\/]/)
+  wd = Dir.getwd
+  Dir.chdir("/")
+  if File.exists?("/opt/opscode/bin/chef-server-ctl")
+    system("/opt/opscode/bin/chef-server-ctl stop")
+  end
+  if !File.exists?("/sbin/mkfs.xfs")
+    system("/usr/bin/yum -y install xfsprogs")
+  end
+  MU::Master.disk("/dev/xvdj", "/opt_tmp", 30)
+  uuid = MU::Master.diskUUID("/dev/xvdj")
+  if !uuid or uuid.empty?
+    MU.log "Failed to retrieve UUID of block device xvdj", MU::ERR, details: MU::Cloud::AWS.realDevicePath("/dev/xvdj")
+    exit 1
+  end
+  MU.log "Moving contents of /opt to /opt_tmp", MU::NOTICE
+  system("/bin/mv /opt/* /opt_tmp/")
+  exit 1 if $?.exitstatus != 0
+  MU.log "Remounting /opt_tmp /opt", MU::NOTICE
+  system("/bin/umount /opt_tmp")
+  exit 1 if $?.exitstatus != 0
+  system("echo '#{uuid} /opt xfs defaults 0 0' >> /etc/fstab")
+  system("/bin/mount -a")
+  exit 1 if $?.exitstatus != 0
+  if File.exists?("/opt/opscode/bin/chef-server-ctl")
+    system("/opt/opscode/bin/chef-server-ctl start")
+  end
+  Dir.chdir(wd)
+end
+
 
 if $opts[:logs]
   MU::Cloud::AWS.listCredentials.each { |credset|
@@ -385,6 +417,7 @@ if $opts[:logs]
 end
 
 if $opts[:dns] and !MU::Cloud::AWS.isGovCloud?
+  $bucketname ||= MU.adminBucketName
   if instance.vpc_id.nil? or instance.vpc_id.empty?
     MU.log "This Mu master appears to be in EC2 Classic. Route53 private DNS zones are not supported. Falling back to old /etc/hosts chicanery.", MU::ERR
   else
@@ -394,7 +427,7 @@ if $opts[:dns] and !MU::Cloud::AWS.isGovCloud?
       params = {
           :name => "platform-mu",
           :vpc => {
-              :vpc_region => MU.myRegion,
+              :vpc_region => MU::Cloud::AWS.myRegion,
               :vpc_id => instance.vpc_id
           },
           :hosted_zone_config => {
@@ -417,7 +450,7 @@ if $opts[:dns] and !MU::Cloud::AWS.isGovCloud?
         MU::Cloud::AWS.route53.associate_vpc_with_hosted_zone(
             hosted_zone_id: ext_zone.id,
             vpc: {
-                vpc_region: MU.myRegion,
+                vpc_region: MU::Cloud::AWS.myRegion,
                 vpc_id: instance.vpc_id
             }
         )
@@ -436,6 +469,7 @@ if $opts[:dns] and !MU::Cloud::AWS.isGovCloud?
 end
 
 if $opts[:uploadlogs]
+  $bucketname ||= MU.adminBucketName
   today = Time.new.strftime("%Y%m%d").to_s
   ["master.log", "nodes.log"].each { |log|
     if File.exist?("/Mu_Logs/#{log}-#{today}")

data/bin/mu-azure-setup

@@ -44,6 +44,7 @@ Usage:
   opt :logs, "Ensure the presence of an Cloud Storage bucket prefixed with 'Mu_Logs' for use with CloudTrails, syslog, etc.", :require => false, :default => false, :type => :boolean
 #  opt :dns, "Ensure the presence of a private DNS Zone called for internal amongst Mu resources.", :require => false, :default => false, :type => :boolean
   opt :uploadlogs, "Push today's log files to the Cloud Storage bucket created by the -l option.", :require => false, :default => false, :type => :boolean
+  opt :optdisk, "Create a block volume for /opt and slide our installation onto it", :require => false, :default => false, :type => :boolean
 end
 
 if MU::Cloud::Azure.hosted? and !$MU_CFG['google']
@@ -245,6 +246,39 @@ if $opts[:logs]
 
 end
 
+if $opts[:optdisk] and !File.open("/etc/mtab").read.match(/ \/opt[\s\/]/)
+  puts "PLACEHOLDER"
+#  myname = MU::Cloud::Google.getGoogleMetaData("instance/name")
+#  wd = Dir.getwd
+#  Dir.chdir("/")
+#  if File.exists?("/opt/opscode/bin/chef-server-ctl")
+#    system("/opt/opscode/bin/chef-server-ctl stop")
+#  end
+#  if !File.exists?("/sbin/mkfs.xfs")
+#    system("/usr/bin/yum -y install xfsprogs")
+#  end
+#  MU::Master.disk(myname+"-mu-opt", "/opt_tmp", 30)
+#  uuid = MU::Master.diskUUID(myname+"-mu-opt")
+#  if !uuid or uuid.empty?
+#    MU.log "Failed to retrieve UUID of block device #{myname}-mu-opt", MU::ERR, details: MU::Cloud::AWS.realDevicePath(myname+"-mu-opt")
+#    exit 1
+#  end
+#  MU.log "Moving contents of /opt to /opt_tmp", MU::NOTICE
+#  system("/bin/mv /opt/* /opt_tmp/")
+#  exit 1 if $?.exitstatus != 0
+#  MU.log "Remounting /opt_tmp /opt", MU::NOTICE
+#  system("/bin/umount /opt_tmp")
+#  exit 1 if $?.exitstatus != 0
+#  system("echo '#{uuid} /opt xfs defaults 0 0' >> /etc/fstab")
+#  system("/bin/mount -a")
+#  exit 1 if $?.exitstatus != 0
+#  if File.exists?("/opt/opscode/bin/chef-server-ctl")
+#    system("/opt/opscode/bin/chef-server-ctl start")
+#  end
+#  Dir.chdir(wd)
+end
+
+
 if $opts[:dns]
 end
 

data/bin/mu-configure

@@ -20,7 +20,6 @@ require 'open-uri'
 require 'colorize'
 require 'timeout'
 require 'etc'
-require 'aws-sdk-core'
 require 'json'
 require 'pp'
 require 'readline'
@@ -28,6 +27,30 @@ require 'fileutils'
 require 'erb'
 require 'tmpdir'
 
+AMROOT = Process.uid == 0
+HOMEDIR = Etc.getpwuid(Process.uid).dir
+CLEAN_ENV={
+  "PATH" => "/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/ruby-current/bin",
+  "HOME" => HOMEDIR
+}
+CLEAN_ENV_STR = CLEAN_ENV.keys.map { |k|
+  k+"=\""+CLEAN_ENV[k]+"\""
+}.join(" ")
+CHEF_CLIENT="/opt/chef/bin/chef-client"
+CHEF_CTL="/opt/opscode/bin/chef-server-ctl"
+GIT_PATTERN = /(((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?))?([\w\.@\:\/\-~]+)(\.git)?(\/)?/
+
+
+#def _x(cmd)
+#  puts "#{CLEAN_ENV} #{cmd}".bold
+#  %x{#{CLEAN_ENV} #{cmd}}
+#end
+
+def _system(cmd)
+  puts cmd.bold
+  system(CLEAN_ENV, cmd)
+end
+
 $IN_GEM = false
 gemwhich = %x{gem which mu 2>&1}.chomp
 gemwhich = nil if $?.exitstatus != 0
@@ -46,6 +69,36 @@ if !mypath.match(/^\/opt\/mu/)
   end
 end
 
+if !$NOOP
+  $IN_AWS = false
+  begin
+    Timeout.timeout(2) do
+      instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
+      $IN_AWS = true if !instance_id.nil? and instance_id.size > 0
+    end
+  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH
+  end
+  $IN_GOOGLE = false
+  begin
+    Timeout.timeout(2) do
+      instance_id = URI.open(
+        "http://metadata.google.internal/computeMetadata/v1/instance/name",
+        "Metadata-Flavor" => "Google"
+      ).read
+      $IN_GOOGLE = true if !instance_id.nil? and instance_id.size > 0
+    end
+  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH
+  end
+  $IN_AZURE = false
+  begin
+    Timeout.timeout(2) do
+      instance = URI.open("http://169.254.169.254/metadata/instance/compute?api-version=2017-08-01","Metadata"=>"true").read
+      $IN_AZURE = true if !instance.nil? and instance.size > 0
+    end
+  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Errno::EHOSTUNREACH
+  end
+end
+
 $possible_addresses = []
 $impossible_addresses = ['127.0.0.1', 'localhost']
 begin
@@ -62,18 +115,36 @@ Socket.getifaddrs.each { |iface|
   if iface.addr and iface.addr.ipv4?
     $possible_addresses << iface.addr.ip_address
     begin
-      addrinfo = Socket.gethostbyaddr(iface.addr.ip_address)
+      addrinfo = Socket.gethostbyaddr(iface.addr.ip_address.split(/\./).map { |o| o.to_i }.pack("CCCC"))
       $possible_addresses << addrinfo.first if !addrinfo.first.nil?
     rescue SocketError
       # usually no name to look up; that's ok
     end
   end
 }
-$possible_addresses.uniq!
-$possible_addresses.reject! { |i| i.match(/^(0\.0\.0\.0$|169\.254\.|127\.0\.)/)}
 
-GIT_PATTERN = /(((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?))?([\w\.@\:\/\-~]+)(\.git)?(\/)?/
+if $IN_AWS
+  ["local-ipv4", "public-ipv4"].each { |addr|
+    ip = URI.open("http://169.254.169.254/latest/meta-data/#{addr}").read.chomp
+    $possible_addresses.unshift(ip) if ip and ip =~ /^\d+\.\d+\.\d+\.\d+/
+  }
+elsif $IN_GOOGLE
+  ["ip", "access-configs/0/external-ip"].each { |addr|
+    ip = URI.open(
+      "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/#{addr}",
+      "Metadata-Flavor" => "Google"
+    ).read.chomp
+    $possible_addresses.unshift(ip) if ip and ip =~ /^\d+\.\d+\.\d+\.\d+/
+  }
+elsif $IN_AZURE
+  ["privateIpAddress", "publicIpAddress"].each { |addr|
+    ip = URI.open("http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/#{addr}?api-version=2017-08-01&format=text","Metadata"=>"true").read
+    $possible_addresses.unshift(ip) if ip and ip =~ /^\d+\.\d+\.\d+\.\d+/
+  }
+end
 
+$possible_addresses.uniq!
+$possible_addresses.reject! { |i| i.match(/^(0\.0\.0\.0$|169\.254\.|127\.0\.)/)}
 
 # Top-level keys in $MU_CFG for which we'll provide interactive, menu-driven
 # configuration.
@@ -404,8 +475,6 @@ def importCurrentValues
 end
 
 if !$NOOP
-  AMROOT = Process.uid == 0
-  HOMEDIR = Etc.getpwuid(Process.uid).dir
 
   $opts = Optimist::options do
     banner <<-EOS
@@ -476,66 +545,65 @@ if !$NOOP
     end
   end
 
-  $IN_AWS = false
-  begin
-    Timeout.timeout(2) do
-      instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
-      $IN_AWS = true if !instance_id.nil? and instance_id.size > 0
-    end
-  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH
-  end
-  $IN_GOOGLE = false
-  begin
-    Timeout.timeout(2) do
-      instance_id = open(
-        "http://metadata.google.internal/computeMetadata/v1/instance/name",
-        "Metadata-Flavor" => "Google"
-      ).read
-      $IN_GOOGLE = true if !instance_id.nil? and instance_id.size > 0
-    end
-  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH
-  end
-  $IN_AZURE = false
-  begin
-    Timeout.timeout(2) do
-      instance = open("http://169.254.169.254/metadata/instance/compute?api-version=2017-08-01","Metadata"=>"true").read
-      $IN_AZURE = true if !instance.nil? and instance.size > 0
+
+  if AMROOT and !$IN_GEM
+    Dir.chdir("/")
+    if $IN_AWS
+      _system("#{MU_BASE}/lib/bin/mu-aws-setup --optdisk")
+    elsif $IN_GOOGLE
+      _system("#{MU_BASE}/lib/bin/mu-gcp-setup --optdisk")
+    elsif $IN_AZURE
+      _system("#{MU_BASE}/lib/bin/mu-azure-setup --optdisk")
     end
-  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Errno::EHOSTUNREACH
+    exit 1 if $?.exitstatus != 0
   end
+  _system("cd #{MU_BASE}/lib/modules && umask 0022 && /usr/local/ruby-current/bin/bundle install")
+  _system("cd #{MU_BASE}/lib/modules && umask 0022 && /opt/chef/embedded/bin/bundle install")
 
   KNIFE_TEMPLATE = "log_level                :info
-  log_location             STDOUT
-  node_name                '<%= chefuser %>'
-  client_key               '<%= MU_BASE %>/var/users/<%= user %>/<%= chefuser %>.user.key'
-  validation_client_name   'mu-validator'
-  validation_key           '<%= MU_BASE %>/var/orgs/<%= user %>/<%= chefuser %>.org.key'
-  chef_server_url 'https://<%= MU.mu_public_addr %>:7443/organizations/<%= chefuser %>'
-  chef_server_root 'https://<%= MU.mu_public_addr %>:7443/organizations/<%= chefuser %>'
-  syntax_check_cache_path  '<%= HOMEDIR %>/.chef/syntax_check_cache'
-  cookbook_path [ '<%= HOMEDIR %>/.chef/cookbooks', '<%= HOMEDIR %>/.chef/site_cookbooks' ]
-  <% if $MU_CFG.has_key?('ssl') and $MU_CFG['ssl'].has_key?('chain') %>
-  ssl_ca_path '<%= File.dirname($MU_CFG['ssl']['chain']) %>'
-  ssl_ca_file '<%= File.basename($MU_CFG['ssl']['chain']) %>'
-  <% end %>
-  knife[:vault_mode] = 'client'
-  knife[:vault_admins] = ['<%= chefuser %>']"
+log_location             STDOUT
+node_name                '<%= chefuser %>'
+client_key               '<%= MU_BASE %>/var/users/<%= user %>/<%= chefuser %>.user.key'
+validation_client_name   'mu-validator'
+validation_key           '<%= MU_BASE %>/var/orgs/<%= user %>/<%= chefuser %>.org.key'
+chef_server_url 'https://<%= MU.mu_public_addr %>:7443/organizations/<%= chefuser %>'
+chef_server_root 'https://<%= MU.mu_public_addr %>:7443/organizations/<%= chefuser %>'
+syntax_check_cache_path  '<%= HOMEDIR %>/.chef/syntax_check_cache'
+cookbook_path [ '<%= HOMEDIR %>/.chef/cookbooks', '<%= HOMEDIR %>/.chef/site_cookbooks' ]
+<% if $MU_CFG.has_key?('ssl') and $MU_CFG['ssl'].has_key?('chain') %>
+ssl_ca_path '<%= File.dirname($MU_CFG['ssl']['chain']) %>'
+ssl_ca_file '<%= File.basename($MU_CFG['ssl']['chain']) %>'
+<% end %>
+knife[:vault_mode] = 'client'
+knife[:vault_admins] = ['<%= chefuser %>']"
 
   CLIENT_TEMPLATE = "chef_server_url  'https://<%= MU.mu_public_addr %>:7443/organizations/<%= user %>'
-  validation_client_name 'mu-validator'
-  log_location   STDOUT
-  node_name 'MU-MASTER'
-  verify_api_cert false
-  ssl_verify_mode :verify_none
-  "
+validation_client_name 'mu-validator'
+log_location   STDOUT
+node_name 'MU-MASTER'
+chef_license 'accept'
+verify_api_cert false
+ssl_verify_mode :verify_none
+"
+
+#chef_server_url  "https://127.0.0.1:7443/organizations/mu"
+#validation_client_name "mu-validator"
+#chef_license "accept"
+#log_location   STDOUT
+#node_name "MU-MASTER"
+#verify_api_cert false
+#ssl_verify_mode :verify_none
+#trusted_certs_dir "/etc/chef/trusted_certs"
+#file_cache_path "/var/chef/cache"
+#file_backup_path "/var/chef/backup"
 
   PIVOTAL_TEMPLATE = "node_name 'pivotal'
-  chef_server_url 'https://<%= MU.mu_public_addr %>:7443'
-  chef_server_root 'https://<%= MU.mu_public_addr %>:7443'
-  no_proxy '<%= MU.mu_public_addr %>'
-  client_key '/etc/opscode/pivotal.pem'
-  ssl_verify_mode :verify_none
-  "
+chef_server_url 'https://<%= MU.mu_public_addr %>:7443'
+chef_server_root 'https://<%= MU.mu_public_addr %>:7443'
+no_proxy '<%= MU.mu_public_addr %>'
+client_key '/etc/opscode/pivotal.pem'
+ssl_verify_mode :verify_none
+"
 
   $CHANGES = []
 
@@ -612,7 +680,7 @@ if !$NOOP
         keynamestr = repo.gsub(/[^a-z0-9\-]/i, "-") + Process.pid.to_s
         keypath = "#{HOMEDIR}/.ssh/#{keynamestr}"
         puts "Paste a complete SSH private key for #{ssh_user.bold}@#{ssh_host.bold} below, then ^D"
-        system("cat > #{keypath}")
+        _system("cat > #{keypath}")
         File.chmod(0600, keypath)
         puts "Key saved to "+keypath.bold
         deletekey = true
@@ -724,7 +792,7 @@ if !$NOOP
       ["public-ipv4", "local-ipv4"].each { |addr|
         begin
         Timeout.timeout(2) do
-          ip = open("http://169.254.169.254/latest/meta-data/#{addr}").read
+          ip = URI.open("http://169.254.169.254/latest/meta-data/#{addr}").read
           ips << ip if !ip.nil? and ip.size > 0
         end
         rescue OpenURI::HTTPError, Timeout::Error, SocketError
@@ -736,9 +804,9 @@ if !$NOOP
       begin
         Timeout.timeout(2) do
   # TODO iterate across multiple interfaces/access-configs
-          ip = open("#{base_url}/instance/network-interfaces/0/ip", "Metadata-Flavor" => "Google").read
+          ip = URI.open("#{base_url}/instance/network-interfaces/0/ip", "Metadata-Flavor" => "Google").read
           ips << ip if !ip.nil? and ip.size > 0
-          ip = open("#{base_url}/instance/network-interfaces/0/access-configs/0/external-ip", "Metadata-Flavor" => "Google").read
+          ip = URI.open("#{base_url}/instance/network-interfaces/0/access-configs/0/external-ip", "Metadata-Flavor" => "Google").read
           ips << ip if !ip.nil? and ip.size > 0
         end
       rescue OpenURI::HTTPError, Timeout::Error, SocketError => e
@@ -753,10 +821,10 @@ if !$NOOP
     $CONFIGURABLES["banner"]["default"] = "Mu Master at #{$CONFIGURABLES["public_address"]["default"]}"
     if $IN_AWS
   # XXX move this crap to a callback hook for puttering around in the AWS submenu
-      aws = JSON.parse(open("http://169.254.169.254/latest/dynamic/instance-identity/document").read)
+      aws = JSON.parse(URI.open("http://169.254.169.254/latest/dynamic/instance-identity/document").read)
       iam = nil
       begin
-        iam = open("http://169.254.169.254/latest/meta-data/iam/security-credentials").read
+        iam = URI.open("http://169.254.169.254/latest/meta-data/iam/security-credentials").read
       rescue OpenURI::HTTPError, SocketError
       end
   #    $CONFIGURABLES["aws"]["subtree"]["account_number"]["default"] = aws["accountId"]
@@ -914,7 +982,7 @@ if !$NOOP
         if data["named_subentries"]
           if data['subtree']['#entries']
             data['subtree']['#entries'].each_pair { |nameentry, subdata|
-              next if nameentry.match(/^#/)
+              next if nameentry.nil? or nameentry.match(/^#/)
               puts "  "+subdata["#menu"].bold+". "+nameentry.green.on_black
             }
           end
@@ -1218,13 +1286,13 @@ if !$NOOP
     pref_chef_version = File.read("#{MU_BASE}/var/mu-chef-client-version").chomp
     if (cur_chef_version != pref_chef_version and cur_chef_version.sub(/\-\d+$/, "") != pref_chef_version) or cur_chef_version.match(/is not installed/)
       puts "Updating MU-MASTER's Chef Client to '#{pref_chef_version}' from '#{cur_chef_version}'"
-      chef_installer = open("https://omnitruck.chef.io/install.sh").read
+      chef_installer = URI.open("https://omnitruck.chef.io/install.sh").read
       File.open("#{HOMEDIR}/chef-install.sh", File::CREAT|File::TRUNC|File::RDWR, 0644){ |f|
         f.puts chef_installer
       }
-      system("/bin/rm -rf /opt/chef ; sh #{HOMEDIR}/chef-install.sh -v #{pref_chef_version}");
+      _system("/bin/rm -rf /opt/chef ; sh #{HOMEDIR}/chef-install.sh -v #{pref_chef_version}");
       # This will go fix gems, permissions, etc
-      system("/opt/chef/bin/chef-apply #{MU_BASE}/lib/cookbooks/mu-master/recipes/init.rb");
+      _system("/opt/chef/bin/chef-apply #{MU_BASE}/lib/cookbooks/mu-master/recipes/init.rb");
     end
   end
 
@@ -1247,12 +1315,17 @@ if !$NOOP
   end
   begin
     require 'mu'
+  rescue LoadError, Gem::MissingSpecError
+    _system("cd #{MU_BASE}/lib/modules && umask 0022 && /usr/local/ruby-current/bin/bundle install")
+    require 'bundler'
+    pwd = Dir.pwd
+    Dir.chdir(MU_BASE+"/lib/modules")
+    Bundler.setup
+    require 'mu'
+    Dir.chdir(pwd)
   rescue MU::MuError => e
     puts "Correct the above error before proceeding. To retry, run:\n\n#{$0.bold} #{ARGV.join(" ").bold}"
     exit 1
-  rescue LoadError
-    system("cd #{MU_BASE}/lib/modules && umask 0022 && /usr/local/ruby-current/bin/bundle install")
-    require 'mu'
   end
 
   if $IN_GEM
@@ -1268,15 +1341,45 @@ if !$NOOP
   end
 
   if AMROOT and ($INITIALIZE or $CHANGES.include?("hostname"))
-    system("/bin/hostname #{$MU_CFG['hostname']}")
+    _system("/bin/hostname #{$MU_CFG['hostname']}")
+  end
+
+  def updateChefRbs
+    user = AMROOT ? "mu" : Etc.getpwuid(Process.uid).name
+    chefuser = user.gsub(/\./, "")
+    templates = { HOMEDIR+"/.chef/knife.rb" => KNIFE_TEMPLATE }
+    Dir.mkdir(HOMEDIR+"/.chef") if !Dir.exist?(HOMEDIR+"/.chef")
+    if AMROOT
+      templates["/etc/chef/client.rb"] = CLIENT_TEMPLATE
+      templates["/etc/opscode/pivotal.rb"] = PIVOTAL_TEMPLATE
+    end
+    templates.each_pair { |file, template|
+      erb = ERB.new(template)
+      processed = erb.result(binding)
+      tmpfile = file+".tmp."+Process.pid.to_s
+      File.open(tmpfile, File::CREAT|File::TRUNC|File::RDWR, 0644){ |f|
+        f.puts processed
+      }
+      if !File.size?(file) or File.read(tmpfile) != File.read(file)
+        File.rename(tmpfile, file)
+        MU.log "Updated #{file}", MU::NOTICE
+        $CHANGES << "chefcerts"
+      else
+        File.unlink(tmpfile)
+      end
+    }
   end
 
+
   # Do some more basic-but-Chef-dependent configuration *before* we meddle with
   # the Chef Server configuration, which depends on some of this (SSL certs and
   # local firewall ports).
   if AMROOT and ($INITIALIZE or $CHANGES.include?("chefartifacts"))
     MU.log "Purging and re-uploading all Chef artifacts", MU::NOTICE
     %x{/sbin/service iptables stop} if $INITIALIZE
+    if File.exists?("#{CHEF_CTL}")
+      _system("#{CHEF_CTL} start")
+    end
     output = %x{MU_INSTALLDIR=#{MU_BASE} MU_LIBDIR=#{MU_BASE}/lib MU_DATADIR=#{MU_BASE}/var #{MU_BASE}/lib/bin/mu-upload-chef-artifacts}
     if $?.exitstatus != 0
       puts output
@@ -1287,47 +1390,28 @@ if !$NOOP
     %x{/sbin/service iptables start} if !$INITIALIZE
   end
 
+  Dir.chdir(Dir.home)
+
   if $INITIALIZE and AMROOT
     MU.log "Force open key firewall holes", MU::NOTICE
-    system("chef-client -o 'recipe[mu-master::firewall-holes]'")
+    _system("#{CHEF_CLIENT} -o 'recipe[mu-master::firewall-holes]'")
   end
 
   if AMROOT
     MU.log "Checking internal SSL signing authority and certificates", MU::NOTICE
-    if !system("chef-client -o 'recipe[mu-master::ssl-certs]'") and $INITIALIZE
+    if !_system("#{CHEF_CLIENT} -o 'recipe[mu-master::ssl-certs]'") and $INITIALIZE
       MU.log "Got bad exit code trying to run recipe[mu-master::ssl-certs]', aborting", MU::ERR
       exit 1
     end
-  end
-
-  def updateChefRbs
-    user = AMROOT ? "mu" : Etc.getpwuid(Process.uid).name
-    chefuser = user.gsub(/\./, "")
-    templates = { HOMEDIR+"/.chef/knife.rb" => KNIFE_TEMPLATE }
-    Dir.mkdir(HOMEDIR+"/.chef") if !Dir.exist?(HOMEDIR+"/.chef")
-    if AMROOT
-      templates["/etc/chef/client.rb"] = CLIENT_TEMPLATE
-      templates["/etc/opscode/pivotal.rb"] = PIVOTAL_TEMPLATE
+    if !File.size?("#{$MU_CFG['datadir']}/ssl/mommacat.crt")
+      MU.log "I just ran recipe[mu-master::ssl-certs]', but #{$MU_CFG['datadir']}/ssl/mommacat.crt} is still missing. Bailing.", MU::ERR
+      exit 1
     end
-    templates.each_pair { |file, template|
-      erb = ERB.new(template)
-      processed = erb.result(binding)
-      tmpfile = file+".tmp."+Process.pid.to_s
-      File.open(tmpfile, File::CREAT|File::TRUNC|File::RDWR, 0644){ |f|
-        f.puts processed
-      }
-      if !File.size?(file) or File.read(tmpfile) != File.read(file)
-        File.rename(tmpfile, file)
-        MU.log "Updated #{file}", MU::NOTICE
-        $CHANGES << "chefcerts"
-      else
-        File.unlink(tmpfile)
-      end
-    }
   end
 
 
   if AMROOT
+    updateChefRbs if !$INITIALIZE
     erb = ERB.new(File.read("#{MU_BASE}/lib/cookbooks/mu-master/templates/default/chef-server.rb.erb"))
     updated_server_cfg = erb.result(binding)
     cfgpath = "/etc/opscode/chef-server.rb"
@@ -1335,7 +1419,7 @@ if !$NOOP
     File.open(tmpfile, File::CREAT|File::TRUNC|File::RDWR, 0644){ |f|
       f.puts updated_server_cfg
     }
-    if !File.size?(cfgpath) or File.read(tmpfile) != File.read(cfgpath)
+    if $INITIALIZE or !File.size?(cfgpath) or File.read(tmpfile) != File.read(cfgpath)
       File.rename(tmpfile, cfgpath)
       # Opscode can't seem to get things right with their postgres socket
       Dir.mkdir("/var/run/postgresql", 0755) if !Dir.exist?("/var/run/postgresql")
@@ -1344,12 +1428,15 @@ if !$NOOP
       elsif !File.exist?("/tmp/.s.PGSQL.5432") and File.exist?("/var/run/postgresql/.s.PGSQL.5432")
         File.symlink("/var/run/postgresql/.s.PGSQL.5432", "/tmp/.s.PGSQL.5432")
       end
-      MU.log "Chef Server config was modified, reconfiguring...", MU::NOTICE
+      MU.log "Chef Server config was modified, reconfiguring...", MU::NOTICE, details: updated_server_cfg
       # XXX Some undocumented port Chef needs only on startup is being blocked by
       # iptables. Something rabbitmq-related. Dopey workaround.
       %x{/sbin/service iptables stop}
-      system("/opt/opscode/bin/chef-server-ctl reconfigure")
-      system("/opt/opscode/bin/chef-server-ctl restart")
+      _system("#{CHEF_CTL} stop")
+      MU.retrier(wait: 10, max: 6, loop_if: Proc.new { $?.exitstatus != 0 }, loop_msg: "Trying to get chef-server-ctl reconfigure to work") {
+        _system("#{CHEF_CTL} reconfigure")
+      }
+      _system("#{CHEF_CTL} start")
       %x{/sbin/service iptables start} if !$INITIALIZE
       updateChefRbs
       $CHANGES << "chefcerts"
@@ -1361,21 +1448,21 @@ if !$NOOP
     updateChefRbs
   end
 
-  if $IN_AWS and AMROOT
-    system("#{MU_BASE}/lib/bin/mu-aws-setup --dns --sg --logs --ephemeral")
+  if $IN_AWS and AMROOT# and $IN_GEM
+    _system("#{MU_BASE}/lib/bin/mu-aws-setup --dns --sg --logs --ephemeral")
   # XXX --ip? Do we really care?
   end
   if $IN_GOOGLE and AMROOT
-    system("#{MU_BASE}/lib/bin/mu-gcp-setup --sg --logs")
+    _system("#{MU_BASE}/lib/bin/mu-gcp-setup --sg --logs")
   end
   if $IN_AZURE and AMROOT
-    system("#{MU_BASE}/lib/bin/mu-azure-setup --sg")
+    _system("#{MU_BASE}/lib/bin/mu-azure-setup --sg")
   end
 
   if $INITIALIZE or $CHANGES.include?("chefcerts")
-    system("rm -f #{HOMEDIR}/.chef/trusted_certs/* ; knife ssl fetch -c #{HOMEDIR}/.chef/knife.rb")
+    _system("rm -f #{HOMEDIR}/.chef/trusted_certs/* ; knife ssl fetch -c #{HOMEDIR}/.chef/knife.rb")
     if AMROOT
-      system("rm -f /etc/chef/trusted_certs/* ; knife ssl fetch -c /etc/chef/client.rb")
+      _system("rm -f /etc/chef/trusted_certs/* ; knife ssl fetch -c /etc/chef/client.rb")
     end
   end
 
@@ -1400,7 +1487,7 @@ if !$NOOP
       if !Dir.exist?(repodir)
         MU.log "Cloning #{repo} into #{repodir}", MU::NOTICE
         Dir.chdir(MU.dataDir)
-        system("/usr/bin/git clone #{repo}")
+        _system("/usr/bin/git clone #{repo}")
         $CHANGES << "chefartifacts"
       end
     }
@@ -1411,7 +1498,15 @@ if !$NOOP
   end
 
   begin
+    if File.exists?("#{CHEF_CTL}")
+      _system("#{CHEF_CTL} start")
+    end
     MU::Groomer::Chef.getSecret(vault: "secrets", item: "consul")
+  rescue OpenSSL::SSL::SSLError => e
+    if !$INITIALIZE
+      raise e
+    end
+    MU.log "Got SSL error connecting to Chef for vault secrets, this is normal during initial install", MU::NOTICE, details: e.message
   rescue MU::Groomer::MuNoSuchSecret
     data = {
       "private_key" => File.read("#{MU_BASE}/var/ssl/consul.key"),
@@ -1427,9 +1522,10 @@ if !$NOOP
   end
   if $INITIALIZE or $CHANGES.include?("vault")
     MU.log "Setting up Hashicorp Vault", MU::NOTICE
-    system("chef-client -o 'recipe[mu-master::vault]'")
+    _system("#{CHEF_CLIENT} -o 'recipe[mu-master::vault]'")
   end
 
+  set389DSCreds
   if $MU_CFG['ldap']['type'] == "389 Directory Services"
     begin
       MU::Master::LDAP.listUsers
@@ -1439,11 +1535,10 @@ if !$NOOP
     if $INITIALIZE or $CHANGES.include?("389ds")
       File.unlink("/root/389ds.tmp/389-directory-setup.inf") if File.exist?("/root/389ds.tmp/389-directory-setup.inf")
       MU.log "Configuring 389 Directory Services", MU::NOTICE
-      set389DSCreds
-      system("chef-client -o 'recipe[mu-master::389ds]'")
+      _system("#{CHEF_CLIENT} -o 'recipe[mu-master::389ds]'")
       exit 1 if $? != 0
       MU::Master::LDAP.initLocalLDAP
-      system("chef-client -o 'recipe[mu-master::sssd]'")
+      _system("#{CHEF_CLIENT} -o 'recipe[mu-master::sssd]'")
       exit 1 if $? != 0
     end
   end
@@ -1474,12 +1569,12 @@ if !$NOOP
     MU::Config.emitSchemaAsRuby
     MU.log "Generating YARD documentation in /var/www/html/docs (see http://#{$MU_CFG['public_address']}/docs/frames.html)"
     File.umask(0022)
-    system("cd #{MU.myRoot} && umask 0022 && env -i PATH=#{ENV['PATH']} HOME=#{HOMEDIR} /usr/local/ruby-current/bin/yard doc modules -m markdown -o /var/www/html/docs && chcon -R -h -t httpd_sys_script_exec_t /var/www/html/")
+    _system("cd #{MU.myRoot} && umask 0022 && /usr/local/ruby-current/bin/yard doc modules -m markdown -o /var/www/html/docs && chcon -R -h -t httpd_sys_script_exec_t /var/www/html/")
   end
 
 
   MU.log "Running chef-client on MU-MASTER", MU::NOTICE
-  system("chef-client -o '#{run_list.join(",")}'")
+  _system("#{CHEF_CLIENT} -o '#{run_list.join(",")}'")
 
 
   if !File.exist?("#{MU_BASE}/var/users/mu/email") or !File.exist?("#{MU_BASE}/var/users/mu/realname")
@@ -1507,11 +1602,11 @@ if !$NOOP
   end
 
   MU.log "Regenerating documentation in /var/www/html/docs"
-  %x{#{MU_BASE}/lib/bin/mu-gen-docs}
+  %x{#{CLEAN_ENV_STR} #{MU_BASE}/lib/bin/mu-gen-docs}
 
   if $INITIALIZE
     MU.log "Setting initial password for admin user 'mu', for logging into Nagios and other built-in services.", MU::NOTICE
-    puts %x{#{MU_BASE}/lib/bin/mu-user-manage -g mu -n "#{$MU_CFG['mu_admin_name']}"}
+    puts %x{#{CLEAN_ENV_STR} #{MU_BASE}/lib/bin/mu-user-manage -g mu -n "#{$MU_CFG['mu_admin_name']}"}
     MU.log "If Scratchpad web interface is not accessible, try the following:", MU::NOTICE
     puts "#{MU_BASE}/lib/bin/mu-user-manage -g --no-scratchpad mu".bold
   end