cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/bin/mu-gcp-setup

@@ -45,7 +45,24 @@ Usage:
   opt :uploadlogs, "Push today's log files to the Cloud Storage bucket created by the -l option.", :require => false, :default => false, :type => :boolean
 end
 
-if MU::Cloud::Google.hosted
+if MU::Cloud::Google.hosted? and !$MU_CFG['google']
+  new_cfg = $MU_CFG.dup
+  cfg_blob = MU::Cloud::Google.hosted_config
+  if cfg_blob
+    cfg_blob['log_bucket_name'] ||= $MU_CFG['hostname']
+    new_cfg["google"] = { "default" => cfg_blob }
+    MU.log "Adding auto-detected Google stanza to #{cfgPath}", MU::NOTICE
+    if new_cfg != $MU_CFG or !cfgExists?
+      MU.log "Generating #{cfgPath}"
+      saveMuConfig(new_cfg)
+      $MU_CFG = new_cfg
+    end
+  end
+end
+
+my_instance_id = MU::Cloud::AWS.getAWSMetaData("instance-id")
+
+if MU::Cloud::Google.hosted?
   instance = MU.myCloudDescriptor
   admin_sg_name = "mu-master-"+MU.myInstanceId+"-ingress-allow"
   if !instance.tags.items or !instance.tags.items.include?(admin_sg_name)
@@ -110,74 +127,79 @@ if $opts[:sg]
 
 end
 
-$bucketname = $MU_CFG['google']['log_bucket_name']
+$bucketname = MU::Cloud::Google.adminBucketName
 
 if $opts[:logs]
-  exists = false
-
-  MU.log "Configuring log and secret Google Cloud Storage bucket '#{$bucketname}'"
-
-  bucket = nil
-  begin
-    bucket = MU::Cloud::Google.storage.get_bucket($bucketname)
-  rescue ::Google::Apis::ClientError => e
-    if e.message.match(/notFound:/)
-      MU.log "Creating #{$bucketname} bucket"
-      bucketobj = MU::Cloud::Google.storage(:Bucket).new(
-        name: $bucketname,
-        location: "US", # XXX why is this needed?
-        versioning: MU::Cloud::Google.storage(:Bucket)::Versioning.new(
-          enabled: true
-        ),
-        lifecycle: MU::Cloud::Google.storage(:Bucket)::Lifecycle.new(
-          rule: [ MU::Cloud::Google.storage(:Bucket)::Lifecycle::Rule.new(
-            action: MU::Cloud::Google.storage(:Bucket)::Lifecycle::Rule::Action.new(
-              type: "SetStorageClass",
-              storage_class: "DURABLE_REDUCED_AVAILABILITY"
-            ),
-            condition: MU::Cloud::Google.storage(:Bucket)::Lifecycle::Rule::Condition.new(
-              age: 180
-            )
-          )]
+  MU::Cloud::Google.listCredentials.each { |credset|
+    bucketname = MU::Cloud::Google.adminBucketName(credset)
+    exists = false
+
+    MU.log "Configuring log and secret Google Cloud Storage bucket '#{bucketname}'"
+
+    bucket = nil
+    begin
+      bucket = MU::Cloud::Google.storage(credentials: credset).get_bucket(bucketname)
+    rescue ::Google::Apis::ClientError => e
+      if e.message.match(/notFound:/)
+        MU.log "Creating #{bucketname} bucket"
+        bucketobj = MU::Cloud::Google.storage(:Bucket).new(
+          name: bucketname,
+          location: "US", # XXX why is this needed?
+          versioning: MU::Cloud::Google.storage(:Bucket)::Versioning.new(
+            enabled: true
+          ),
+          lifecycle: MU::Cloud::Google.storage(:Bucket)::Lifecycle.new(
+            rule: [ MU::Cloud::Google.storage(:Bucket)::Lifecycle::Rule.new(
+              action: MU::Cloud::Google.storage(:Bucket)::Lifecycle::Rule::Action.new(
+                type: "SetStorageClass",
+                storage_class: "DURABLE_REDUCED_AVAILABILITY"
+              ),
+              condition: MU::Cloud::Google.storage(:Bucket)::Lifecycle::Rule::Condition.new(
+                age: 180
+              )
+            )]
+          )
         )
-      )
-      bucket = MU::Cloud::Google.storage.insert_bucket(
-        MU::Cloud::Google.defaultProject,
-        bucketobj
-      )
-    else
-      raise MuError, e.inspect
+        bucket = MU::Cloud::Google.storage(credentials: credset).insert_bucket(
+          MU::Cloud::Google.defaultProject(credset),
+          bucketobj
+        )
+      else
+        pp e.backtrace
+        raise MU::MuError, e.inspect
+      end
     end
-  end
 
-  ebs_key = nil
-
-  begin
-    ebs_key = MU::Cloud::Google.storage.get_object($bucketname, "log_vol_ebs_key")
-  rescue ::Google::Apis::ClientError => e
-    if e.message.match(/notFound:/)
-      # XXX this may not be useful outside of AWS
-      MU.log "Creating new key for encrypted log volume"
-      key = SecureRandom.random_bytes(32)
-      f = Tempfile.new("logvolkey") # XXX this is insecure and stupid
-      f.write key
-      f.close
-      objectobj = MU::Cloud::Google.storage(:Object).new(
-        bucket: $bucketname,
-        name: "log_vol_ebs_key"
-      )
-      ebs_key = MU::Cloud::Google.storage.insert_object(
-        $bucketname,
-        objectobj,
-        upload_source: f.path
-      )
-      f.unlink
-    else
-      raise MuError, e.inspect
+    ebs_key = nil
+
+    begin
+      ebs_key = MU::Cloud::Google.storage(credentials: credset).get_object(bucketname, "log_vol_ebs_key")
+    rescue ::Google::Apis::ClientError => e
+      if e.message.match(/notFound:/)
+        # XXX this may not be useful outside of AWS
+        MU.log "Creating new key for encrypted log volume"
+        key = SecureRandom.random_bytes(32)
+        f = Tempfile.new("logvolkey") # XXX this is insecure and stupid
+        f.write key
+        f.close
+        objectobj = MU::Cloud::Google.storage(:Object).new(
+          bucket: bucketname,
+          name: "log_vol_ebs_key"
+        )
+        ebs_key = MU::Cloud::Google.storage(credentials: credset).insert_object(
+          bucketname,
+          objectobj,
+          upload_source: f.path
+        )
+        f.unlink
+      else
+        raise MuError, e.inspect
+      end
     end
-  end
+# XXX stop doing this per-bucket, chowderhead
+    MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
+  }
 
-  MU::Master.disk("/dev/xvdl", "/Mu_Logs", 50, "log_vol_ebs_key", "ram7")
 end
 
 if $opts[:dns]

data/bin/mu-load-config.rb

@@ -18,6 +18,46 @@ require 'json'
 require 'erubis'
 require 'socket'
 
+# Make sure things make sense in our various cloud subsections, which are more
+# complicated than the rest. May alter the hash it's passed.
+def validateClouds(cfg)
+  ok = true
+
+  ['aws', 'google', 'azure'].each { |cloud|
+    if cfg[cloud]
+      found_default = false
+      # Muddle up old-style single-account cloud configs into an array of
+      # named accounts, which is what we're expecting to see nowadays.
+      if cfg[cloud] and cfg[cloud].values.any? { |h| !h.is_a?(Hash) }
+        puts "Converting single #{cloud} #{cfgPath} account entry to default alias"
+        cfg[cloud] = {
+          "default" => cfg[cloud]
+        }
+        cfg[cloud]["default"]["default"] = true
+        found_default = true
+      else
+        missing_alias = false
+        cfg[cloud].each_pair { |acctalias, acct|
+          if acct["default"]
+            if found_default
+              puts "Multiple accounts have 'default' set in #{cloud}"
+              ok = false
+            end
+            found_default = true
+          end
+        }
+      end
+      if !found_default
+        first = cfg[cloud].keys.first
+        puts "No default #{cloud} credentials specified in #{cfgPath}, arbitrarily designating '#{first}'"
+        cfg[cloud][first]["default"] = true
+      end
+    end
+  }
+
+  ok
+end
+
 # Locate and load the Mu Master's configuration, typically stored in
 # /opt/mu/etc/mu.yaml. If ~/.mu.yaml exists, load that too and allow it to
 # override values from the global config. Also puts Mu's /modules directory
@@ -94,7 +134,7 @@ def loadMuConfig(default_cfg_overrides = nil)
   if !global_cfg.has_key?("installdir")
     if ENV['MU_INSTALLDIR']
       global_cfg["installdir"] = ENV['MU_INSTALLDIR']
-    elsif Gem.paths and Gem.paths.home
+    elsif Gem.paths and Gem.paths.home and !Dir.exists?("/opt/mu/lib")
       global_cfg["installdir"] = File.realpath(File.expand_path(File.dirname(Gem.paths.home))+"/../../../")
     else
       global_cfg["installdir"] = "/opt/mu"
@@ -117,10 +157,13 @@ def loadMuConfig(default_cfg_overrides = nil)
     end
   end
 
+  exit 1 if !validateClouds(global_cfg)
+
   $LOAD_PATH << "#{global_cfg["libdir"]}/modules"
   return default_cfg.merge(global_cfg).freeze
 end
 
+# Shorthand for locating the path to mu.yaml
 def cfgPath
   home = Etc.getpwuid(Process.uid).dir
   username = Etc.getpwuid(Process.uid).name
@@ -128,7 +171,7 @@ def cfgPath
     if ENV.include?('MU_INSTALLDIR')
       ENV['MU_INSTALLDIR']+"/etc/mu.yaml"
     elsif Dir.exists?("/opt/mu")
-      File.realpath(File.expand_path(File.dirname(__FILE__)+"/../../etc"))
+      "/opt/mu/etc/mu.yaml"
     else
       "#{home}/.mu.yaml"
     end
@@ -146,6 +189,7 @@ end
 # @param cfg [Hash]: The configuration to dump
 # @param comment [Hash]: A configuration blob that will be appended as a commented block
 def saveMuConfig(cfg, comment = nil)
+  exit 1 if !validateClouds(cfg)
   puts "**** Saving master config to #{cfgPath} *****"
   File.open(cfgPath, File::CREAT|File::TRUNC|File::RDWR, 0644){ |f|
     f.puts cfg.to_yaml

data/bin/mu-self-update

@@ -168,15 +168,6 @@ elif [ "`diff -r $MU_LIBDIR/Berksfile.lock $MU_DATADIR/tmp/berks_changes.$$/Berk
 	set -- "-u" "$@"
 fi
 
-/bin/rm -rf $MU_DATADIR/tmp/cookbook_changes.$$
-/bin/rm -rf $MU_DATADIR/tmp/berks_changes.$$
-
-if [ "$rebuild_chef_artifacts" == "1" ];then
-  /bin/rm -rf /root/.berkshelf/cookbooks
-  $bindir/mu-upload-chef-artifacts -p
-else
-  $bindir/mu-upload-chef-artifacts -r mu
-fi
 
 set +e
 /sbin/service nagios stop
@@ -203,7 +194,18 @@ if [ "$chef_major" == "12" ];then
 fi
 
 /opt/chef/bin/chef-apply $MU_LIBDIR/cookbooks/mu-master/recipes/init.rb
+
+/bin/rm -rf $MU_DATADIR/tmp/cookbook_changes.$$
+/bin/rm -rf $MU_DATADIR/tmp/berks_changes.$$
+
+if [ "$rebuild_chef_artifacts" == "1" ];then
+  /bin/rm -rf /root/.berkshelf/cookbooks
+  $bindir/mu-upload-chef-artifacts -p
+else
+  $bindir/mu-upload-chef-artifacts -r mu
+fi
 $bindir/mu-configure -n
+
 for dir in $MU_LIBDIR /opt/chef/embedded /opt/opscode/embedded /usr/local/ruby-current/;do
   echo "${GREEN}Sanitizing permissions in ${BOLD}$dir${NORM}${GREEN}${NORM}"
   for tree in `ls -1 $dir/`;do

data/bin/mu-upload-chef-artifacts

@@ -44,10 +44,10 @@ real_datadir="$MU_DATADIR"
 
 
 source $MU_INSTALLDIR/lib/install/deprecated-bash-library.sh
-if [ -x $MU_INSTALLDIR/lib/bin/mu-gen-env ];then
-  $MU_INSTALLDIR/lib/bin/mu-gen-env > $HOMEDIR/.murc
-  source $HOMEDIR/.murc
-fi
+# if [ -x $MU_INSTALLDIR/lib/bin/mu-gen-env ];then
+#   $MU_INSTALLDIR/lib/bin/mu-gen-env > $HOMEDIR/.murc
+#   source $HOMEDIR/.murc
+# fi
 export MU_DATADIR="$real_datadir"
 
 cd $MU_CHEF_CACHE

data/{mu.gemspec → cloud-mu.gemspec}

@@ -17,7 +17,7 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '1.9.0-beta'
+  s.version     = '2.0.0-alpha'
   s.date        = '2018-12-11'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
@@ -34,7 +34,7 @@ EOF
   s.authors     = ["John Stange", "Robert Patt-Corner", "Ryan Bolyard", "Clara Bridges", "Zach Rowe"]
   s.email       = 'eGTLabs@eglobaltech.com'
   s.files       = build_file_list(whereami)
-  s.executables = Dir.entries(whereami+"/bin")
+  s.executables = Dir.entries(whereami+"/bin").reject { |f| File.directory?(f) }
   s.homepage    =
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'

data/cookbooks/awscli/Berksfile

@@ -0,0 +1,8 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+
+# Supermarket Cookbooks

data/cookbooks/mu-activedirectory/Berksfile

@@ -0,0 +1,11 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+
+# Supermarket Cookbooks
+cookbook "windows", '~> 5.1.1'
+cookbook "chef-vault", '~> 3.1.1'
+cookbook "yum-epel", '~> 3.2.0'

data/cookbooks/mu-firewall/Berksfile

@@ -0,0 +1,9 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+
+# Supermarket Cookbooks
+cookbook	'firewall', '~> 2.6.5'

data/cookbooks/mu-firewall/metadata.rb

@@ -13,4 +13,4 @@ version          '0.1.0'
 	supports os
 end
 
-depends	'firewall', '~> 2.6.5'
+depends	'firewall', '~> 2.7.0'

data/cookbooks/mu-glusterfs/Berksfile

@@ -0,0 +1,10 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook 'mu-firewall'
+
+# Supermarket Cookbooks
+cookbook 'yum', '~> 5.1.0'

data/cookbooks/mu-jenkins/Berksfile

@@ -0,0 +1,14 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook 'mu-master'
+cookbook 'mu-utility'
+cookbook 'mu-tools'
+
+# Supermarket Cookbooks
+cookbook 'java', '~> 2.2.0'
+cookbook 'jenkins', '~> 6.2.0'
+cookbook 'chef-vault', '~> 3.1.1'

data/cookbooks/mu-master/Berksfile

@@ -0,0 +1,23 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook 'nagios'
+cookbook 'mu-utility'
+cookbook 'mu-tools'
+cookbook 'mu-firewall'
+cookbook 'mu-activedirectory'
+cookbook 's3fs'
+
+# Supermarket Cookbooks
+cookbook 'nrpe', '~> 2.0.3'
+cookbook 'postfix', '~> 5.3.1'
+cookbook 'bind', '~> 2.2.0'
+cookbook 'bind9-ng', '~> 0.1.0'
+cookbook 'vault-cluster', '~> 2.1.0'
+cookbook 'consul-cluster', '~> 2.0.0'
+cookbook 'hostsfile', '~> 3.0.1'
+cookbook 'chef-vault', '~> 3.1.1'
+cookbook 'apache2', '< 4.0'

data/cookbooks/mu-master/attributes/default.rb

@@ -77,7 +77,7 @@ default['nagios']['host_name_attribute'] = 'chef_node_name'
 default['application_attributes']['logs']['volume_size_gb'] = 50
 default['application_attributes']['logs']['mount_device'] = "/dev/xvdl"
 default['application_attributes']['logs']['label'] = "#{node['hostname']} /Mu_Logs"
-default['application_attributes']['logs']['secure_location'] = MU.adminBucketName
+#default['application_attributes']['logs']['secure_location'] = MU.adminBucketName
 default['application_attributes']['logs']['ebs_keyfile'] = "log_vol_ebs_key"
 default['application_attributes']['logs']['mount_directory'] = "/Mu_Logs"
 

data/cookbooks/mu-master/metadata.rb

@@ -7,7 +7,7 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 source_url 'https://github.com/cloudamatic/mu'
 issues_url 'https://github.com/cloudamatic/mu/issues'
 chef_version '>= 12.1' if respond_to?(:chef_version)
-version '0.9.0'
+version '0.9.2'
 
 %w( centos ).each do |os|
 	supports os
@@ -18,7 +18,7 @@ depends 'nrpe', '~> 2.0.3'
 depends 'mu-utility'
 depends 'mu-tools'
 depends 'mu-activedirectory'
-depends 's3fs', '~> 3.0.1'
+depends 's3fs'
 depends 'postfix', '~> 5.3.1'
 depends 'bind', '~> 2.2.0'
 depends 'bind9-ng', '~> 0.1.0'

data/cookbooks/mu-master/recipes/default.rb

@@ -27,7 +27,7 @@ include_recipe 'mu-master::firewall-holes'
 include_recipe 'mu-master::ssl-certs'
 include_recipe 'mu-master::vault'
 include_recipe 'mu-tools::gcloud'
-include_recipe 'mu-master::eks-kubectl'
+#include_recipe 'mu-master::eks-kubectl'
 
 master_ips = get_mu_master_ips
 master_ips << "127.0.0.1"