cloud-mu 2.1.0beta → 3.0.0beta

data/bin/mu-aws-setup

@@ -86,7 +86,7 @@ end
 # Create a security group, or manipulate an existing one, so that we have all
 # of the appropriate network holes.
 if $opts[:sg]
-  open_ports = [443, 2260, 7443, 8443, 9443, 8200]
+  open_ports = [443, MU.mommaCatPort, 7443, 8443, 9443, 8200]
   ranges = if $MU_CFG and $MU_CFG['my_networks'] and $MU_CFG['my_networks'].size > 0
     $MU_CFG['my_networks'].map { |r|
       r = r+"/32" if r.match(/^\d+\.\d+\.\d+\.\d+$/)
@@ -245,7 +245,7 @@ if $opts[:logs]
 
     resp = MU::Cloud::AWS.s3(credentials: credset).list_buckets
     resp.buckets.each { |bucket|
-      exists = true if bucket['name'] == bucketname
+      exists = true if bucket.name == bucketname
     }
     if !exists
       MU.log "Creating #{bucketname} bucket"
@@ -273,7 +273,7 @@ if $opts[:logs]
         body: "#{key}"
       )
     end
-    if File.exists?("#{MU.mySSLDir}/Mu_CA.pem")
+    if File.exist?("#{MU.mySSLDir}/Mu_CA.pem")
       MU.log "Putting the Mu Master's public SSL certificate into #{bucketname}/Mu_CA.pem"
       MU::Cloud::AWS.s3(credentials: credset).put_object(
         bucket: bucketname,
@@ -438,7 +438,7 @@ end
 if $opts[:uploadlogs]
   today = Time.new.strftime("%Y%m%d").to_s
   ["master.log", "nodes.log"].each { |log|
-    if File.exists?("/Mu_Logs/#{log}-#{today}")
+    if File.exist?("/Mu_Logs/#{log}-#{today}")
       MU.log "Uploading /Mu_Logs/#{log}-#{today} to bucket #{$bucketname}"
       MU::Cloud::AWS.s3.put_object(
           bucket: $bucketname,

data/bin/mu-azure-setup

@@ -0,0 +1,265 @@
+#!/usr/local/ruby-current/bin/ruby
+#
+# Copyright:: Copyright (c) 2017 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Perform initial Mu setup tasks:
+# 1. Set up an appropriate Security Group
+# 2. Associate a specific Elastic IP address to this MU server, if required.
+# 3. Create an S3 bucket for Mu logs.
+
+require 'etc'
+require 'securerandom'
+
+require File.expand_path(File.dirname(__FILE__))+"/mu-load-config.rb"
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require 'mu'
+require 'mu/master/ssl'
+Dir.chdir(MU.installDir)
+
+$opts = Optimist::options do
+  banner <<-EOS
+Usage:
+#{$0} [-i] [-s] [-l] [-u] [-d]
+  EOS
+#  opt :ip, "Attempt to configure the IP requested in the CHEF_PUBLIC_IP environment variable, or if none is set, to associate an arbitrary Elastic IP.", :require => false, :default => false, :type => :boolean
+  opt :sg, "Attempt to configure a Security Group with appropriate permissions.", :require => false, :default => false, :type => :boolean
+  opt :logs, "Ensure the presence of an Cloud Storage bucket prefixed with 'Mu_Logs' for use with CloudTrails, syslog, etc.", :require => false, :default => false, :type => :boolean
+#  opt :dns, "Ensure the presence of a private DNS Zone called for internal amongst Mu resources.", :require => false, :default => false, :type => :boolean
+  opt :uploadlogs, "Push today's log files to the Cloud Storage bucket created by the -l option.", :require => false, :default => false, :type => :boolean
+end
+
+if MU::Cloud::Azure.hosted? and !$MU_CFG['google']
+  new_cfg = $MU_CFG.dup
+  cfg_blob = MU::Cloud::Azure.hosted_config
+  if cfg_blob
+    cfg_blob['log_bucket_name'] ||= $MU_CFG['hostname']
+    new_cfg["google"] = { "default" => cfg_blob }
+    MU.log "Adding auto-detected Azure stanza to #{cfgPath}", MU::NOTICE
+    if new_cfg != $MU_CFG or !cfgExists?
+      MU.log "Generating #{cfgPath}"
+      saveMuConfig(new_cfg)
+      $MU_CFG = new_cfg
+    end
+  end
+end
+
+sgs_to_ifaces = {}
+ifaces_to_sgs = {}
+sgs = []
+if MU::Cloud::Azure.hosted?
+  instance = MU.myCloudDescriptor
+  # Azure VMs can have exactly one security group per network interface, so if
+  # there's already one, we use it.
+  iface_num = 0
+  instance.network_profile.network_interfaces.each { |iface|
+    iface_id = MU::Cloud::Azure::Id.new(iface.id)
+    ifaces_to_sgs[iface_id] = false
+    iface_desc = MU::Cloud::Azure.network.network_interfaces.get(MU.myInstanceId.resource_group, iface_id.to_s)
+    if iface_desc.network_security_group
+      sg_id = MU::Cloud::Azure::Id.new(iface_desc.network_security_group.id)
+      sgs << sg_id
+      sgs_to_ifaces[sg_id] = iface_id
+      ifaces_to_sgs[iface_id] = sg_id
+    else
+      ifaces_to_sgs[iface_id] = "mu-master-"+MU.myInstanceId.name
+      ifaces_to_sgs[iface_id] += "-"+iface_num.to_s if iface_num > 0
+    end
+    if iface_desc.ip_configurations
+      iface_desc.ip_configurations.each { |ipcfg|
+        ipcfg.subnet.id.match(/resourceGroups\/([^\/]+)\/providers\/Microsoft.Network\/virtualNetworks\/([^\/]+)\/subnets\/(.*)/)
+        rg = Regexp.last_match[1]
+        vpc_id = Regexp.last_match[2]
+        subnet_id = Regexp.last_match[3]
+        subnet = MU::Cloud::Azure.network.subnets.get(
+          rg,
+          vpc_id,
+          subnet_id
+        )
+        if subnet.network_security_group
+          sg_id = MU::Cloud::Azure::Id.new(subnet.network_security_group.id)
+          sgs << sg_id
+        end
+      }
+    end
+    iface_num += 1
+  }
+  sgs.uniq!
+
+#  if !instance.tags.items or !instance.tags.items.include?(admin_sg_name)
+#    newitems = instance.tags.items ? instance.tags.items.dup : []
+#    newitems << admin_sg_name
+#    MU.log "Setting my instance tags", MU::NOTICE, details: newitems
+#    newtags = MU::Cloud::Azure.compute(:Tags).new(
+#      fingerprint: instance.tags.fingerprint,
+#      items: newitems
+#    )
+#    MU::Cloud::Azure.compute.set_instance_tags(
+#      MU::Cloud::Azure.myProject,
+#      MU.myAZ,
+#      MU.myInstanceId,
+#      newtags
+#    )
+#    instance = MU.myCloudDescriptor
+#  end
+  preferred_ip = MU.mu_public_ip
+end
+
+# Create a security group, or manipulate an existing one, so that we have all
+# of the appropriate network holes.
+if $opts[:sg]
+  open_ports = [80, 443, MU.mommaCatPort, 7443, 8443, 9443, 8200]
+
+  sgs.each { |sg_id|
+    admin_sg_name = sg_id.is_a?(String) ? sg_id : sg_id.name
+
+    found = MU::MommaCat.findStray("Azure", "firewall_rule", dummy_ok: true, cloud_id: admin_sg_name, region: instance.location)
+    admin_sg = found.first if !found.nil? and found.size > 0
+
+    rules = []
+    open_ports.each { |port|
+      rules << {
+        "proto" => "tcp",
+        "port" => port.to_s
+      }
+    }
+
+    rules << {
+      "proto" => "tcp",
+      "port" => 22
+#      "hosts" => ["#{preferred_ip}/32"]
+    }
+    cfg = {
+      "name" => admin_sg_name,
+      "scrub_mu_isms" => true,
+      "cloud" => "Azure",
+      "rules" => rules,
+      "region" => instance.location,
+      "target_tags" => [admin_sg_name],
+      "vpc" => {
+        "vpc_id" => MU::Cloud::Azure::Id.new(instance.network_profile.network_interfaces.first.id)
+      }
+    }
+
+    if !admin_sg
+      admin_sg = MU::Cloud::FirewallRule.new(kitten_cfg: cfg, mu_name: admin_sg_name)
+      admin_sg.create
+      admin_sg.groom
+    else
+      rules.each { |rule|
+        admin_sg.addRule(rule["hosts"], proto: rule["proto"], port: rule["port"].to_i)
+      }
+    end
+  }
+end
+
+$bucketname = MU::Cloud::Azure.adminBucketName
+
+if $opts[:logs]
+  MU::Cloud::Azure.listCredentials.each { |credset|
+    bucketname = MU::Cloud::Azure.adminBucketName(credset)
+    exists = false
+
+    MU.log "Configuring log and secret Azure Cloud Storage bucket '#{bucketname}'"
+
+    bucket = nil
+    begin
+      bucket = MU::Cloud::Azure.storage(credentials: credset).get_bucket(bucketname)
+    rescue ::Azure::Apis::ClientError => e
+      if e.message.match(/notFound:/)
+        MU.log "Creating #{bucketname} bucket"
+        bucketobj = MU::Cloud::Azure.storage(:Bucket).new(
+          name: bucketname,
+          location: "US", # XXX why is this needed?
+          versioning: MU::Cloud::Azure.storage(:Bucket)::Versioning.new(
+            enabled: true
+          ),
+          lifecycle: MU::Cloud::Azure.storage(:Bucket)::Lifecycle.new(
+            rule: [ MU::Cloud::Azure.storage(:Bucket)::Lifecycle::Rule.new(
+              action: MU::Cloud::Azure.storage(:Bucket)::Lifecycle::Rule::Action.new(
+                type: "SetStorageClass",
+                storage_class: "DURABLE_REDUCED_AVAILABILITY"
+              ),
+              condition: MU::Cloud::Azure.storage(:Bucket)::Lifecycle::Rule::Condition.new(
+                age: 180
+              )
+            )]
+          )
+        )
+        bucket = MU::Cloud::Azure.storage(credentials: credset).insert_bucket(
+          MU::Cloud::Azure.defaultProject(credset),
+          bucketobj
+        )
+      else
+        pp e.backtrace
+        raise MU::MuError, e.inspect
+      end
+    end
+
+    ebs_key = nil
+
+    begin
+      ebs_key = MU::Cloud::Azure.storage(credentials: credset).get_object(bucketname, "log_vol_ebs_key")
+    rescue ::Azure::Apis::ClientError => e
+      if e.message.match(/notFound:/)
+        # XXX this may not be useful outside of AWS
+        MU.log "Creating new key for encrypted log volume"
+        key = SecureRandom.random_bytes(32)
+        f = Tempfile.new("logvolkey") # XXX this is insecure and stupid
+        f.write key
+        f.close
+        objectobj = MU::Cloud::Azure.storage(:Object).new(
+          bucket: bucketname,
+          name: "log_vol_ebs_key"
+        )
+        ebs_key = MU::Cloud::Azure.storage(credentials: credset).insert_object(
+          bucketname,
+          objectobj,
+          upload_source: f.path
+        )
+        f.unlink
+      else
+        raise MuError, e.inspect
+      end
+    end
+# XXX stop doing this per-bucket, chowderhead
+    MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
+  }
+
+end
+
+if $opts[:dns]
+end
+
+if $opts[:uploadlogs]
+  today = Time.new.strftime("%Y%m%d").to_s
+  ["master.log", "nodes.log"].each { |log|
+    if File.exist?("/Mu_Logs/#{log}-#{today}")
+      MU.log "Uploading /Mu_Logs/#{log}-#{today} to bucket #{$bucketname}"
+      MU::Cloud::AWS.s3.put_object(
+          bucket: $bucketname,
+          key: "#{log}/#{today}",
+          body: File.read("/Mu_Logs/#{log}-#{today}")
+      )
+    else
+      MU.log "No log /Mu_Logs/#{log}-#{today} was found", MU::WARN
+    end
+  }
+end

data/bin/mu-azure-tests

@@ -0,0 +1,43 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+#pp MU::Cloud::Azure.listRegions
+#pp MU::Cloud::Azure::Habitat.testcalls
+#pp MU::Cloud::Azure::VPC.find(cloud_id: MU::Cloud::Azure::Id.new(resource_group: "mu", name: "mu-vnet"))
+#pp MU::Cloud::Azure.authorization.role_assignments.list_for_resource_group("AKS-DEV-2019062015-KA-EASTUS")
+#pp MU::Cloud::Azure::Role.find(role_name: "Azure Kubernetes Service Cluster Admin Role")
+#puts MU::Cloud::Azure.default_subscription
+#pp MU::Cloud::Azure.fetchPublicIP("MYVPC-DEV-2019061911-XI-EASTUS", "ip-addr-thingy")
+#pp MU::Cloud::Azure.ensureProvider("egtazure", "Microsoft.ContainerService", force: true)
+pp MU::Cloud::Azure::Server.find(cloud_id: "mu")
+exit
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/6")
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/8")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/6")
+pp MU::Cloud::Azure::Server.fetchImage("Debian/debian-10/10")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2012-R2-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2016-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2019-Datacenter")

data/bin/mu-cleanup

@@ -23,6 +23,14 @@ require 'optimist'
 require 'mu'
 Dir.chdir(MU.installDir)
 
+credentials = []
+MU::Cloud.supportedClouds.each { |cloud|
+  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+  next if cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
+  credentials.concat(cloudclass.listCredentials)
+}
+credentials.uniq!
+
 $opts = Optimist::options do
   banner <<-EOS
 Usage:
@@ -35,6 +43,8 @@ Usage:
   opt :skipcloud, "Only purge Mu master deployment metadata, and skip all cloud resources.", :require => false, :default => false, :type => :boolean
   opt :web, "Generate web-friendly (HTML) output.", :require => false, :default => false, :type => :boolean
   opt :verbose, "Display debugging output.", :require => false, :default => false, :type => :boolean
+  opt :credentials, "Restrict to operating on a subset of available credential sets, instead of all that we know about.", :require => false, :default => credentials, :type => :strings
+  opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
   opt :quiet, "Display minimal output.", :require => false, :default => false, :type => :boolean
 end
 verbosity = MU::Logger::NORMAL
@@ -57,12 +67,14 @@ end
 
 
 MU::Cleanup.run(
-    $opts[:deploy],
-    noop: $opts[:noop],
-    skipsnapshots: $opts[:skipsnapshots],
-    onlycloud: $opts[:onlycloud],
-    verbosity: verbosity,
-    web: $opts[:web],
-    skipcloud: $opts[:skipcloud],
-    ignoremaster: $opts[:ignoremaster]
+  $opts[:deploy],
+  noop: $opts[:noop],
+  skipsnapshots: $opts[:skipsnapshots],
+  onlycloud: $opts[:onlycloud],
+  verbosity: verbosity,
+  web: $opts[:web],
+  skipcloud: $opts[:skipcloud],
+  ignoremaster: $opts[:ignoremaster],
+  credsets: $opts[:credentials],
+  regions: $opts[:regions]
 )

data/bin/mu-configure

@@ -29,12 +29,52 @@ require 'erb'
 require 'tmpdir'
 
 $IN_GEM = false
-if Gem.paths and Gem.paths.home and File.dirname(__FILE__).match(/^#{Gem.paths.home}/)
-  $IN_GEM = true
+gemwhich = %x{gem which mu 2>&1}.chomp
+gemwhich = nil if $?.exitstatus != 0
+mypath = File.realpath(File.expand_path(File.dirname(__FILE__)))
+if !mypath.match(/^\/opt\/mu/)
+  if Gem.paths and Gem.paths.home and
+     (mypath.match(/^#{Gem.paths.home}/) or gemwhich.match(/^#{Gem.paths.home}/))
+    $IN_GEM = true
+  elsif $?.exitstatus == 0 and gemwhich and !gemwhich.empty?
+    $LOAD_PATH.each { |path|
+      if path.match(/\/cloud-mu-[^\/]+\/modules/) or
+         path.match(/#{Regexp.quote(gemwhich)}/)
+        $IN_GEM = true
+      end
+    }
+  end
+end
+
+$possible_addresses = []
+$impossible_addresses = ['127.0.0.1', 'localhost']
+begin
+  sys_name = Socket.gethostname
+  official, aliases = Socket.gethostbyname(sys_name)
+  $possible_addresses << sys_name
+  $possible_addresses << official
+  $possible_addresses.concat(aliases)
+rescue SocketError
+  # don't let them use the default hostname if it doesn't resolve
+  $impossible_addresses << sys_name
 end
+Socket.getifaddrs.each { |iface|
+  if iface.addr and iface.addr.ipv4?
+    $possible_addresses << iface.addr.ip_address
+    begin
+      addrinfo = Socket.gethostbyaddr(iface.addr.ip_address)
+      $possible_addresses << addrinfo.first if !addrinfo.first.nil?
+    rescue SocketError
+      # usually no name to look up; that's ok
+    end
+  end
+}
+$possible_addresses.uniq!
+$possible_addresses.reject! { |i| i.match(/^(0\.0\.0\.0$|169\.254\.|127\.0\.)/)}
 
 GIT_PATTERN = /(((git|ssh|http(s)?)|(git@[\w\.]+))(:(\/\/)?))?([\w\.@\:\/\-~]+)(\.git)?(\/)?/
 
+
 # Top-level keys in $MU_CFG for which we'll provide interactive, menu-driven
 # configuration.
 $CONFIGURABLES = {
@@ -42,8 +82,7 @@ $CONFIGURABLES = {
     "title" => "Public Address",
     "desc" => "IP address or hostname",
     "required" => true,
-    "rootonly" => true,
-    "pattern" => /^(localhost|127\.0\.0\.1|#{Socket.gethostname})$/,
+    "pattern" => /^(#{$impossible_addresses.map { |a| Regexp.quote(a) }.join("|") })$/,
     "negate_pattern" => true,
     "changes" => ["389ds", "chef-server", "chefrun", "chefcerts"]
   },
@@ -52,14 +91,12 @@ $CONFIGURABLES = {
     "desc" => "Administative contact email",
     "pattern" => /\A([\w+\-].?)+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i,
     "required" => true,
-    "rootonly" => true,
     "changes" => ["mu-user", "chefrun"]
   },
   "mu_admin_name" => {
     "title" => "Admin Name",
     "desc" => "Administative contact's full name",
     "default" => "Mu Administrator",
-    "rootonly" => true,
     "changes" => ["mu-user", "chefrun"]
   },
   "hostname" => {
@@ -70,10 +107,17 @@ $CONFIGURABLES = {
     "desc" => "The local system's value for HOSTNAME",
     "changes" => ["chefrun", "hostname"]
   },
+  "mommacat_port" => {
+    "title" => "Momma Cat Listen Port",
+    "pattern" => /^[0-9]+$/i,
+    "default" => 2260,
+    "required" => $IN_GEM,
+    "desc" => "Listen port for the Momma Cat grooming daemon",
+    "changes" => ["chefrun"]
+  },
   "banner" => {
     "title" => "Banner",
     "desc" => "Login banner, displayed in various locations",
-    "rootonly" => true,
     "changes" => ["chefrun"]
   },
   "mu_repository" => {
@@ -105,6 +149,11 @@ $CONFIGURABLES = {
     "desc" => "If set to true, Mu will be allowed to modify routing and peering behavior of VPCs which it did not create, but for which it has permissions.",
     "boolean" => true
   },
+  "ansible_dir" => {
+    "title" => "Ansible directory",
+    "desc" => "Intended for use with minimal installs which use Ansible as a groomer and which do not store Ansible artifacts in a dedicated git repository. This allows simply pointing to a local directory.",
+    "required" => false
+  },
   "aws" => {
     "title" => "Amazon Web Services",
     "named_subentries" => true,
@@ -166,6 +215,10 @@ $CONFIGURABLES = {
         "title" => "Credentials File",
         "desc" => "JSON-formatted Service Account credentials for our GCP account, stored in plain text in a file. Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON and point this argument to the file. This is less secure than using 'credentials' to store in a vault."
       },
+      "credentials_encoded" => {
+        "title" => "Base64-Encoded Credentials",
+        "desc" => "JSON-formatted Service Account credentials for our GCP account, b64-encoded and dropped directly into mu.yaml. Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON and point this argument to the file. This is less secure than using 'credentials' to store in a vault."
+      },
       "region" => {
         "title" => "Default Region",
         "desc" => "Default Google Cloud Platform region in which we operate and deploy",
@@ -182,6 +235,11 @@ $CONFIGURABLES = {
         "required" => false,
         "desc" => "For Google Cloud projects which are attached to a GSuite domain. GCP service accounts cannot view or manage GSuite resources (groups, users, etc) directly, but must instead masquerade as a GSuite user which has delegated authority to the service account. See also: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority"
       },
+      "customer_id" => {
+        "title" => "GSuite Customer ID",
+        "required" => false,
+        "desc" => "For Google Cloud projects which are attached to a GSuite domain. Some API calls (groups, users, etc) require this identifier. From admin.google.com, choose Security, the Single Sign On, and look for the Entity ID field. The value after idpid= in the URL there should be the customer ID."
+      },
       "default" => {
         "title" => "Is Default Account",
         "default" => false,
@@ -194,28 +252,40 @@ $CONFIGURABLES = {
     "title" => "Microsoft Azure Cloud Computing Platform & Services",
     "named_subentries" => true,
     "subtree" => {
+      "directory_id" => {
+        "title" => "Directory ID",
+        "desc" => "AKA Tenant ID; the default Microsoft Azure Directory project in which we operate and deploy, from https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview"
+      },
+      "client_id" => {
+        "title" => "Client ID",
+        "desc" => "App client id used to authenticate to our subscription. From https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview"
+      },
+      "client_secret" => {
+        "title" => "Client Secret",
+        "desc" => "App client secret used to authenticate to our subscription. From https://portal.azure.com/#blade/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/RegisteredAppsPreview under the 'Certificates & secrets' tab, 'Client secrets.' This can only be retrieved upon initial secret creation."
+      },
       "subscription" => {
         "title" => "Default Subscription",
-        "desc" => "Default Microsoft Azure Directory project in which we operate and deploy."
-      },
-      "credentials" => {
-        "title" => "Credentials Vault:Item",
-        "desc" => "A secure Chef vault and item from which to retrieve the JSON-formatted Service Account credentials for our Azure account, in the format vault:itemname (e.g. 'secrets:google'). Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON, and import that key to the vault specified here. Import example: knife vault create secrets google -J my-google-service-account.json "
+        "desc" => "Default Microsoft Azure Subscription we will use to deploy, from https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade"
       },
+      # "credentials" => {
+      #   "title" => "Credentials Vault:Item",
+      #   "desc" => "A secure Chef vault and item from which to retrieve the JSON-formatted Service Account credentials for our Azure account, in the format vault:itemname (e.g. 'secrets:google'). Generate a service account at: https://console.cloud.google.com/iam-admin/serviceaccounts/project, making sure the account has sufficient privileges to manage cloud resources. Download the private key as JSON, and import that key to the vault specified here. Import example: knife vault create secrets google -J my-google-service-account.json "
+      # },
       "credentials_file" => {
         "title" => "Credentials File",
-        "desc" => "JSON-formatted Service Account credentials for our Azure account, stored in plain text in a file."
+        "desc" => "JSON file which contains a hash of directory_id, client_id, client_secret, and subscription values. If found, these will be override values entered directly in mu-configure."
       },
       "region" => {
         "title" => "Default Region",
         "desc" => "Default Microsoft Azure region in which we operate and deploy",
         "default" => "eastus"
       },
-      "log_bucket_name" => {
-        "title" => "Log and Secret Bucket Name",
-        "desc" => "Cloud Storage bucket into which we'll synchronize deploy secrets, and if we're hosted in Azure, collected system logs",
-        "changes" => ["chefrun"]
-      },
+      # "log_bucket_name" => {
+      #   "title" => "Log and Secret Bucket Name",
+      #   "desc" => "Cloud Storage bucket into which we'll synchronize deploy secrets, and if we're hosted in Azure, collected system logs",
+      #   "changes" => ["chefrun"]
+      # },
       "default" => {
         "title" => "Is Default Account",
         "default" => false,
@@ -226,6 +296,55 @@ $CONFIGURABLES = {
   }
 }
 
+def cloneHash(hash)
+  new = {}
+  hash.each_pair { |k,v|
+    if v.is_a?(Hash)
+      new[k] = cloneHash(v)
+    elsif !v.nil?
+      new[k] = v.dup
+    end
+  }
+  new
+end
+
+# Load values from our existing configuration into the $CONFIGURABLES hash
+def importCurrentValues
+  require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+  $CONFIGURABLES.each_key { |key|
+    next if !$MU_CFG.has_key?(key)
+    if $CONFIGURABLES[key].has_key?("subtree")
+      # It's a sub-tree. I'm too lazy to write a recursive thing for this, just
+      # cover the simple case that we actually care about for now.
+      if $CONFIGURABLES[key]["named_subentries"]
+        $CONFIGURABLES[key]['subtree']["#title"] = $CONFIGURABLES[key]['title']
+        $MU_CFG[key].each_pair { |nameentry, subtree|
+          $CONFIGURABLES[key]['subtree']["#entries"] ||= {}
+          $CONFIGURABLES[key]['subtree']["#entries"][nameentry] = cloneHash($CONFIGURABLES[key]['subtree'])
+          $CONFIGURABLES[key]['subtree']["#entries"][nameentry].delete("#entries")
+          $CONFIGURABLES[key]["subtree"]["#entries"][nameentry]["name"] = {
+            "title" => "Name",
+            "desc" => "A name/alias for this account.",
+            "required" => true,
+            "value" => nameentry
+          }
+          $CONFIGURABLES[key]["subtree"].keys.each { |subkey|
+            next if !subtree.has_key?(subkey)
+            $CONFIGURABLES[key]["subtree"]["#entries"][nameentry][subkey]["value"] = subtree[subkey]
+          }
+        }
+      else
+        $CONFIGURABLES[key]["subtree"].keys.each { |subkey|
+          next if !$MU_CFG[key].has_key?(subkey)
+          $CONFIGURABLES[key]["subtree"][subkey]["value"] = $MU_CFG[key][subkey]
+        }
+      end
+    else
+      $CONFIGURABLES[key]["value"] = $MU_CFG[key]
+    end
+  }
+end
+
 AMROOT = Process.uid == 0
 HOMEDIR = Etc.getpwuid(Process.uid).dir
 
@@ -240,16 +359,16 @@ $opts = Optimist::options do
       data["subtree"].each_pair { |subkey, subdata|
         next if !AMROOT and subdata['rootonly']
         subdata['cli-opt'] = (key+"-"+subkey).gsub(/_/, "-")
-        opt (key+"-"+subkey).to_sym, subdata["desc"], :require => false, :type => (subdata["boolean"] ? :boolean : :string)
+        opt subdata['cli-opt'].to_sym, subdata["desc"], :require => false, :type => (subdata["boolean"] ? :boolean : :string)
         required << subdata['cli-opt'] if subdata['required']
       }
     elsif data["array"]
       data['cli-opt'] = key.gsub(/_/, "-")
-      opt key.to_sym, data["desc"], :require => false, :type => (data["boolean"] ? :booleans : :strings)
+      opt data['cli-opt'].to_sym, data["desc"], :require => false, :type => (data["boolean"] ? :booleans : :strings)
       required << data['cli-opt'] if data['required']
     else
       data['cli-opt'] = key.gsub(/_/, "-")
-      opt key.to_sym, data["desc"], :require => false, :type => (data["boolean"] ? :boolean : :string)
+      opt data['cli-opt'].to_sym, data["desc"], :require => false, :type => (data["boolean"] ? :boolean : :string)
       required << data['cli-opt'] if data['required']
     end
   }
@@ -264,16 +383,38 @@ else
   MU_BASE = "/opt/mu"
 end
 
-$INITIALIZE = (!File.size?("#{MU_BASE}/etc/mu.yaml") or $opts[:force])
+def cfgPath
+  home = Etc.getpwuid(Process.uid).dir
+  username = Etc.getpwuid(Process.uid).name
+  if Process.uid == 0
+    if ENV.include?('MU_INSTALLDIR')
+      ENV['MU_INSTALLDIR']+"/etc/mu.yaml"
+    elsif Dir.exist?("/opt/mu")
+      "/opt/mu/etc/mu.yaml"
+    else
+      "#{home}/.mu.yaml"
+    end
+  else
+    "#{home}/.mu.yaml"
+  end
+end
+
+$INITIALIZE = (!File.size?(cfgPath) or $opts[:force])
+
 $HAVE_GLOBAL_CONFIG = File.size?("#{MU_BASE}/etc/mu.yaml")
-if !AMROOT and ($INITIALIZE or !$HAVE_GLOBAL_CONFIG) and !$IN_GEM
+if !AMROOT and !$HAVE_GLOBAL_CONFIG and !$IN_GEM and Dir.exist?("/opt/mu/lib")
   puts "Global configuration has not been initialized or is missing. Must run as root to correct."
   exit 1
 end
 
-if !$HAVE_GLOBAL_CONFIG and $opts[:noninteractive] and (!$opts[:public_address] or !$opts[:mu_admin_email])
-  puts "Specify --public-address and --mu-admin-email on new non-interactive configs"
-  exit 1
+if !$HAVE_GLOBAL_CONFIG and $opts[:noninteractive] and (!$opts[:"public-address"] or !$opts[:"mu-admin-email"])
+  if $IN_GEM
+    importCurrentValues # maybe we're in local-only mode
+  end
+  if !$MU_CFG or !$MU_CFG['mu_admin_email'] or !$MU_CFG['mu_admin_name']
+    puts "Specify --public-address and --mu-admin-email on new non-interactive configs"
+    exit 1
+  end
 end
 
 $IN_AWS = false
@@ -298,13 +439,12 @@ end
 $IN_AZURE = false
 begin
   Timeout.timeout(2) do
-    instance_id = open("http://169.254.169.254/metadata/instance/compute").read
-    $IN_AWS = true if !instance_id.nil? and instance_id.size > 0
+    instance = open("http://169.254.169.254/metadata/instance/compute?api-version=2017-08-01","Metadata"=>"true").read
+    $IN_AZURE = true if !instance.nil? and instance.size > 0
   end
 rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Errno::EHOSTUNREACH
 end
 
-
 KNIFE_TEMPLATE = "log_level                :info
 log_location             STDOUT
 node_name                '<%= chefuser %>'
@@ -340,17 +480,6 @@ ssl_verify_mode :verify_none
 
 $CHANGES = []
 
-def cloneHash(hash)
-  new = {}
-  hash.each_pair { |k,v|
-    if v.is_a?(Hash)
-      new[k] = cloneHash(v)
-    elsif !v.nil?
-      new[k] = v.dup
-    end
-  }
-  new
-end
 
 $MENU_MAP = {}
 def assignMenuEntries(tree = $CONFIGURABLES, map = $MENU_MAP)
@@ -420,7 +549,7 @@ def trySSHKeyWithGit(repo, keypath = nil)
       response = Readline.readline("Y/N> ".bold, false)
     end while !response and !response.match(/^(y|n)$/i)
     if response == "y" or response == "Y"
-      Dir.mkdir("#{HOMEDIR}/.ssh", 0700) if !Dir.exists?("#{HOMEDIR}/.ssh")
+      Dir.mkdir("#{HOMEDIR}/.ssh", 0700) if !Dir.exist?("#{HOMEDIR}/.ssh")
       keynamestr = repo.gsub(/[^a-z0-9\-]/i, "-") + Process.pid.to_s
       keypath = "#{HOMEDIR}/.ssh/#{keynamestr}"
       puts "Paste a complete SSH private key for #{ssh_user.bold}@#{ssh_host.bold} below, then ^D"
@@ -433,7 +562,7 @@ def trySSHKeyWithGit(repo, keypath = nil)
     end
   end
 
-  if File.exists?("#{HOMEDIR}/.ssh/config")
+  if File.exist?("#{HOMEDIR}/.ssh/config")
     FileUtils.cp("#{HOMEDIR}/.ssh/config", "#{HOMEDIR}/.ssh/config.bak.#{Process.pid.to_s}")
     cfgbackup = "#{HOMEDIR}/.ssh/config.bak.#{Process.pid.to_s}"
   end
@@ -482,8 +611,8 @@ def cloneGitRepo(repo)
     elsif $?.exitstatus != 0 and output.match(/permission denied/i)
       puts ""
       puts output.red.on_black
-      if $opts[:ssh_keys_given]
-        $opts[:ssh_keys].each { |keypath|
+      if $opts[:"ssh-keys-given"]
+        $opts[:"ssh-keys"].each { |keypath|
           if trySSHKeyWithGit(fullrepo, keypath)
             Dir.chdir(cwd)
             return fullrepo
@@ -558,10 +687,9 @@ def setDefaults
     end
   end
 
-  ips.concat(Socket.ip_address_list.delete_if { |i| !i.ipv4? or i.ip_address.match(/^(0\.0\.0\.0$|169\.254\.|127\.0\.)/) }.map { |a| a.ip_address })
 
   $CONFIGURABLES["allow_invade_foreign_vpcs"]["default"] = false
-  $CONFIGURABLES["public_address"]["default"] = ips.first
+  $CONFIGURABLES["public_address"]["default"] = $possible_addresses.first
   $CONFIGURABLES["hostname"]["default"] = Socket.gethostname
   $CONFIGURABLES["banner"]["default"] = "Mu Master at #{$CONFIGURABLES["public_address"]["default"]}"
   if $IN_AWS
@@ -605,19 +733,42 @@ def importCLIValues
   $CONFIGURABLES.each_pair { |key, data|
     next if !AMROOT and data['rootonly']
     if data.has_key?("subtree")
+
       if !data['named_subentries']
         data["subtree"].each_pair { |subkey, subdata|
           next if !AMROOT and subdata['rootonly']
-          if $opts[(subdata['cli-opt'].gsub(/-/, "_")+"_given").to_sym]
-            newval = runValueCallback(subdata, $opts[subdata['cli-opt'].gsub(/-/, "_").to_sym])
+          if $opts[(subdata['cli-opt'].+"_given").to_sym]
+            newval = runValueCallback(subdata, $opts[subdata['cli-opt'].to_sym])
             subdata["value"] = newval if !newval.nil?
             $CHANGES.concat(subdata['changes']) if subdata['changes']
           end
         }
+      # Honor CLI adds for named trees (credentials, etc) if there are no
+      # entries in them yet.
+      elsif data["#entries"].nil? or data["#entries"].empty?
+        newvals = false
+        data["subtree"].each_pair { |subkey, subdata|
+          next if !AMROOT and subdata['rootonly']
+          next if !subdata['cli-opt']
+          if $opts[(subdata['cli-opt']+"_given").to_sym]
+            newval = runValueCallback(subdata, $opts[subdata['cli-opt'].to_sym])
+            if !newval.nil?
+              subdata["value"] = newval
+              newvals = true
+            end
+          end
+        }
+        if newvals
+          newtree = data["subtree"].dup
+          newtree['default']['value'] = true if newtree['default']
+          data['subtree']['#entries'] = {
+            "default" => newtree
+          }
+        end
       end
     else
-      if $opts[(data['cli-opt'].gsub(/-/, "_")+"_given").to_sym]
-        newval = runValueCallback(data, $opts[data['cli-opt'].gsub(/-/, "_").to_sym])
+      if $opts[(data['cli-opt']+"_given").to_sym]
+        newval = runValueCallback(data, $opts[data['cli-opt'].to_sym])
         data["value"] = newval if !newval.nil?
         $CHANGES.concat(data['changes']) if data['changes']
       end
@@ -625,42 +776,6 @@ def importCLIValues
   }
 end
 
-# Load values from our existing configuration into the $CONFIGURABLES hash
-def importCurrentValues
-  require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
-  $CONFIGURABLES.each_key { |key|
-    next if !$MU_CFG.has_key?(key)
-    if $CONFIGURABLES[key].has_key?("subtree")
-      # It's a sub-tree. I'm too lazy to write a recursive thing for this, just
-      # cover the simple case that we actually care about for now.
-      if $CONFIGURABLES[key]["named_subentries"]
-        $CONFIGURABLES[key]['subtree']["#title"] = $CONFIGURABLES[key]['title']
-        $MU_CFG[key].each_pair { |nameentry, subtree|
-          $CONFIGURABLES[key]['subtree']["#entries"] ||= {}
-          $CONFIGURABLES[key]['subtree']["#entries"][nameentry] = cloneHash($CONFIGURABLES[key]['subtree'])
-          $CONFIGURABLES[key]['subtree']["#entries"][nameentry].delete("#entries")
-          $CONFIGURABLES[key]["subtree"]["#entries"][nameentry]["name"] = {
-            "title" => "Name",
-            "desc" => "A name/alias for this account.",
-            "required" => true,
-            "value" => nameentry
-          }
-          $CONFIGURABLES[key]["subtree"].keys.each { |subkey|
-            next if !subtree.has_key?(subkey)
-            $CONFIGURABLES[key]["subtree"]["#entries"][nameentry][subkey]["value"] = subtree[subkey]
-          }
-        }
-      else
-        $CONFIGURABLES[key]["subtree"].keys.each { |subkey|
-          next if !$MU_CFG[key].has_key?(subkey)
-          $CONFIGURABLES[key]["subtree"][subkey]["value"] = $MU_CFG[key][subkey]
-        }
-      end
-    else
-      $CONFIGURABLES[key]["value"] = $MU_CFG[key]
-    end
-  }
-end
 
 def printVal(data)
   valid = true
@@ -969,6 +1084,7 @@ if !$opts[:noninteractive]
   $CONFIGURABLES, $MENU_MAP = menu
   $MU_CFG = setConfigTree
 else
+  $MU_CFG = setConfigTree
   if !entireConfigValid?
     puts "Configuration had validation errors, exiting.\nRe-invoke #{$0} to correct."
     exit 1
@@ -1012,7 +1128,7 @@ def set389DSCreds
         data = MU::Groomer::Chef.getSecret(vault: "mu_ldap", item: creds)
         MU::Groomer::Chef.grantSecretAccess("MU-MASTER", "mu_ldap", creds)
       end
-    rescue MU::Groomer::Chef::MuNoSuchSecret
+    rescue MU::Groomer::MuNoSuchSecret
       user = cfg["user"]
       pw = Password.pronounceable(14..16)
       if $MU_CFG["ldap"].has_key?(creds)
@@ -1038,7 +1154,7 @@ def set389DSCreds
   }
 end
 
-if AMROOT
+if AMROOT and !$IN_GEM
   cur_chef_version = `/bin/rpm -q chef`.sub(/^chef-(\d+\.\d+\.\d+-\d+)\..*/, '\1').chomp
   pref_chef_version = File.read("#{MU_BASE}/var/mu-chef-client-version").chomp
   if (cur_chef_version != pref_chef_version and cur_chef_version.sub(/\-\d+$/, "") != pref_chef_version) or cur_chef_version.match(/is not installed/)
@@ -1054,11 +1170,12 @@ if AMROOT
 end
 
 if $INITIALIZE
-  if AMROOT
+  if AMROOT and !$IN_GEM
     %x{/sbin/service iptables stop} # Chef run will set up correct rules later
   end
   $MU_SET_DEFAULTS = setConfigTree
   require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+  saveMuConfig($MU_SET_DEFAULTS)
 else
   if AMROOT
     $NEW_CFG = $MU_CFG.merge(setConfigTree)
@@ -1078,10 +1195,16 @@ rescue LoadError
   system("cd #{MU_BASE}/lib/modules && umask 0022 && /usr/local/ruby-current/bin/bundle install")
   require 'mu'
 end
-
+exit
 if $IN_GEM
+  if $INITIALIZE
+    $MU_CFG = MU.detectCloudProviders
+  end
+  require 'mu/master/ssl'
+  MU::Master::SSL.bootstrap
   puts $MU_CFG.to_yaml
   saveMuConfig($MU_CFG)
+  MU::MommaCat.restart
   exit
 end
 
@@ -1122,7 +1245,7 @@ def updateChefRbs
   user = AMROOT ? "mu" : Etc.getpwuid(Process.uid).name
   chefuser = user.gsub(/\./, "")
   templates = { HOMEDIR+"/.chef/knife.rb" => KNIFE_TEMPLATE }
-  Dir.mkdir(HOMEDIR+"/.chef") if !Dir.exists?(HOMEDIR+"/.chef")
+  Dir.mkdir(HOMEDIR+"/.chef") if !Dir.exist?(HOMEDIR+"/.chef")
   if AMROOT
     templates["/etc/chef/client.rb"] = CLIENT_TEMPLATE
     templates["/etc/opscode/pivotal.rb"] = PIVOTAL_TEMPLATE
@@ -1156,10 +1279,10 @@ if AMROOT
   if !File.size?(cfgpath) or File.read(tmpfile) != File.read(cfgpath)
     File.rename(tmpfile, cfgpath)
     # Opscode can't seem to get things right with their postgres socket
-    Dir.mkdir("/var/run/postgresql", 0755) if !Dir.exists?("/var/run/postgresql")
-    if File.exists?("/tmp/.s.PGSQL.5432") and !File.exists?("/var/run/postgresql/.s.PGSQL.5432")
+    Dir.mkdir("/var/run/postgresql", 0755) if !Dir.exist?("/var/run/postgresql")
+    if File.exist?("/tmp/.s.PGSQL.5432") and !File.exist?("/var/run/postgresql/.s.PGSQL.5432")
       File.symlink("/tmp/.s.PGSQL.5432", "/var/run/postgresql/.s.PGSQL.5432")
-    elsif !File.exists?("/tmp/.s.PGSQL.5432") and File.exists?("/var/run/postgresql/.s.PGSQL.5432")
+    elsif !File.exist?("/tmp/.s.PGSQL.5432") and File.exist?("/var/run/postgresql/.s.PGSQL.5432")
       File.symlink("/var/run/postgresql/.s.PGSQL.5432", "/tmp/.s.PGSQL.5432")
     end
     MU.log "Chef Server config was modified, reconfiguring...", MU::NOTICE
@@ -1186,6 +1309,9 @@ end
 if $IN_GOOGLE and AMROOT
   system("#{MU_BASE}/lib/bin/mu-gcp-setup --sg --logs")
 end
+if $IN_AZURE and AMROOT
+  system("#{MU_BASE}/lib/bin/mu-azure-setup --sg")
+end
 
 if $INITIALIZE or $CHANGES.include?("chefcerts")
   system("rm -f #{HOMEDIR}/.chef/trusted_certs/* ; knife ssl fetch -c #{HOMEDIR}/.chef/knife.rb")
@@ -1212,7 +1338,7 @@ if $MU_CFG['repos'] and $MU_CFG['repos'].size > 0
     repo.match(/\/([^\/]+?)(\.git)?$/)
     shortname = Regexp.last_match(1)
     repodir = MU.dataDir + "/" + shortname
-    if !Dir.exists?(repodir)
+    if !Dir.exist?(repodir)
       MU.log "Cloning #{repo} into #{repodir}", MU::NOTICE
       Dir.chdir(MU.dataDir)
       system("/usr/bin/git clone #{repo}")
@@ -1227,7 +1353,7 @@ end
 
 begin
   MU::Groomer::Chef.getSecret(vault: "secrets", item: "consul")
-rescue MU::Groomer::Chef::MuNoSuchSecret
+rescue MU::Groomer::MuNoSuchSecret
   data = {
     "private_key" => File.read("#{MU_BASE}/var/ssl/consul.key"),
     "certificate" => File.read("#{MU_BASE}/var/ssl/consul.crt"),
@@ -1252,7 +1378,7 @@ if $MU_CFG['ldap']['type'] == "389 Directory Services"
     $CHANGES << "389ds"
   end
   if $INITIALIZE or $CHANGES.include?("389ds")
-    File.unlink("/root/389ds.tmp/389-directory-setup.inf") if File.exists?("/root/389ds.tmp/389-directory-setup.inf")
+    File.unlink("/root/389ds.tmp/389-directory-setup.inf") if File.exist?("/root/389ds.tmp/389-directory-setup.inf")
     MU.log "Configuring 389 Directory Services", MU::NOTICE
     set389DSCreds
     system("chef-client -o 'recipe[mu-master::389ds]'")
@@ -1297,7 +1423,7 @@ MU.log "Running chef-client on MU-MASTER", MU::NOTICE
 system("chef-client -o '#{run_list.join(",")}'")
 
 
-if !File.exists?("#{MU_BASE}/var/users/mu/email") or !File.exists?("#{MU_BASE}/var/users/mu/realname")
+if !File.exist?("#{MU_BASE}/var/users/mu/email") or !File.exist?("#{MU_BASE}/var/users/mu/realname")
   MU.log "Finalizing the 'mu' Chef/LDAP account", MU::NOTICE
   MU.setLogging(MU::Logger::SILENT)
   MU::Master.manageUser(