cloud-mu 3.1.4 → 3.1.5

data/modules/mu/clouds/google/vpc.rb

@@ -513,7 +513,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param target_subnets_key [String]: The subnet/subnets on the other side of the peered VPC.
         # @param instance_id [String]: The instance ID in the target subnet/subnets.
         # @return [Boolean]
-        def self.have_route_peered_vpc?(source_subnets_key, target_subnets_key, instance_id)
+        def self.can_route_to_master_peer?(source_subnets_key, target_subnets_key, instance_id)
         end
 
         # Retrieves the route tables of used by subnets

data/modules/mu/config.rb

@@ -180,8 +180,19 @@ module MU
       $file_format = MU::Config.guessFormat(path)
       $yaml_refs = {}
       erb = ERB.new(File.read(path), nil, "<>")
+      erb.filename = path
 
-      raw_text = erb.result(erb_binding)
+      begin
+        raw_text = erb.result(erb_binding)
+      rescue NameError => e
+        loc = e.backtrace[0].sub(/:(\d+):.*/, ':\1')
+        msg = if e.message.match(/wrong constant name Config.getTail PLACEHOLDER ([^\s]+) REDLOHECALP/)
+          "Variable '#{Regexp.last_match[1]}' referenced in config, but not defined. Missing required parameter?"
+        else
+          e.message
+        end
+        raise ValidationError, msg+" at "+loc
+      end
       raw_json = nil
 
       # If we're working in YAML, do some magic to make includes work better.

data/modules/mu/config/server.yml

@@ -7,6 +7,7 @@ vpc:
 platform: ubuntu
 ssh_user: ubuntu
 associate_public_ip: true
+add_private_ips: 3
 canned_iam_policies:
 - AmazonDynamoDBReadOnlyAccess
 - AmazonElastiCacheFullAccess

data/modules/mu/defaults/AWS.yaml

@@ -73,33 +73,56 @@ ubuntu14:
   ap-southeast-1: ami-2855964b
   ap-southeast-2: ami-d19fc4b2
 win2k12r2: &1
-  us-east-1: ami-055c10ae78f3a58a2
-  us-east-2: ami-fbbe929e
-  us-west-1: ami-ec91ac8c
-  us-west-2: ami-106ca068
-  eu-central-1: ami-59e15a36
-  eu-west-1: ami-65b16b1c
-  sa-east-1: ami-93d6afff
-  ap-northeast-1: ami-dcd375ba
-  ap-northeast-2: ami-fa2e8b94
-  ap-southeast-1: ami-b61657d5
-  ap-southeast-2: ami-9a7b97f8
-  ap-south-1: ami-99a8eaf6
-  ca-central-1: ami-608b3304
-win2k16:
-  us-east-1: ami-d2cb25a8
-  us-east-2: ami-2db59748
-  us-west-1: ami-2db59748
-  us-west-2: ami-3b47ba43
-  eu-central-1: ami-37d46558
-  eu-west-1: ami-06c5d662
-  sa-east-1: ami-53fd803f
-  ap-northeast-1: ami-ce8b42a8
-  ap-northeast-2: ami-c17ca7af
-  ap-southeast-1: ami-fe51279d
-  ap-southeast-2: ami-792bcd1b
-  ap-south-1: ami-448dcb2b
-  ca-central-1: ami-a39920c7
+  us-east-1: ami-00f7cf8d57d29a8a7
+  us-east-2: ami-0c14a2a9b1d88428d
+  ca-central-1: ami-0210e4efc4186f89d
+  us-west-2: ami-036681205605cba8c
+  us-west-1: ami-072d0f2b03f351e5c
+  eu-west-1: ami-061524b3efcc026da
+  eu-west-2: ami-0a7aeb2dae7c7154b
+  eu-west-3: ami-0b16adff6701f08bb
+  eu-north-1: ami-09bd34c6465aa914b
+  sa-east-1: ami-078221cae70b179c4
+  eu-central-1: ami-047d37ec58a8469fb
+  ap-northeast-1: ami-0ce23ffef990003d2
+  ap-south-1: ami-0106284f16a19651a
+  ap-northeast-2: ami-0518e43d0367f1a6d
+  ap-southeast-1: ami-0858019a829a5169d
+  ap-southeast-2: ami-0e0d7d3acb6427f53
+win2k16: &5
+  us-east-1: ami-090dd1749dc0be91d
+  us-east-2: ami-09c9eeb95291e63d7
+  ca-central-1: ami-0c63a53a15fca4238
+  us-west-2: ami-0b8540e9a207143eb
+  eu-west-1: ami-067b5d8d85d3c8cf8
+  us-west-1: ami-0a2dc7e2cf21ab3e9
+  eu-west-2: ami-0cece465ae4b18027
+  eu-west-3: ami-0cc5f29ed6e0e8a67
+  eu-central-1: ami-02d4da18299373531
+  sa-east-1: ami-021d5c906c1898430
+  ap-northeast-1: ami-01129c98f812a3be7
+  ap-south-1: ami-005c7910458e9541d
+  ap-northeast-2: ami-024840a881c449d78
+  ap-southeast-2: ami-070af17644a596301
+  ap-southeast-1: ami-0bc2e098a07140bc4
+  eu-north-1: ami-03dcc78d77d2ee027
+win2k19:
+  us-east-1: ami-09946f18cbcdce65c
+  us-east-2: ami-02ab72768678bb7d0
+  ca-central-1: ami-0fcf1b24169d88d7f
+  us-west-2: ami-025ca67c85e4e147d
+  eu-west-2: ami-08a0e09c469e6a557
+  us-west-1: ami-05960eb854f91cbb8
+  eu-west-1: ami-0f91d02c05561cf5b
+  eu-central-1: ami-0faafc220143c941c
+  eu-west-3: ami-0d2a0b6f21c7ce4a2
+  eu-north-1: ami-0fecae116e9e331bf
+  sa-east-1: ami-050693ece618acaa5
+  ap-northeast-2: ami-0e0ebd96765911d72
+  ap-northeast-1: ami-0729606d0a4051499
+  ap-southeast-1: ami-06ba249ee50ee9669
+  ap-southeast-2: ami-03a051a6d0f2b79d5
+  ap-south-1: ami-02c287a24c5e872f6
 amazon:
   us-east-1: ami-b73b63a0
   us-east-2: ami-58277d3d
@@ -114,7 +137,7 @@ amazon:
   ap-southeast-1: ami-b953f2da
   ap-southeast-2: ami-db704cb8
 win2k12: *1
-windows: *1
+windows: *5
 ubuntu: *2
 centos: *3
 rhel7: *4

data/modules/mu/groomers/ansible.rb

@@ -66,6 +66,7 @@ module MU
       # @param permissions [Boolean]: If true, save the secret under the current active deploy (if any), rather than in the global location for this user
       # @param deploy_dir [String]: If permissions is +true+, save the secret here
       def self.saveSecret(vault: nil, item: nil, data: nil, permissions: false, deploy_dir: nil)
+
         if vault.nil? or vault.empty? or item.nil? or item.empty?
           raise MuError, "Must call saveSecret with vault and item names"
         end
@@ -73,7 +74,6 @@ module MU
           raise MuError, "Ansible vault/item names cannot include forward slashes"
         end
         pwfile = vaultPasswordFile
-        
 
         dir = if permissions
           if deploy_dir
@@ -95,8 +95,9 @@ module MU
         if File.exist?(path)
           MU.log "Overwriting existing vault #{vault} item #{item}"
         end
+
         File.open(path, File::CREAT|File::RDWR|File::TRUNC, 0600) { |f|
-          f.write data
+          f.write data.to_yaml
         }
 
         cmd = %Q{#{ansibleExecDir}/ansible-vault encrypt #{path} --vault-password-file #{pwfile}}
@@ -115,14 +116,23 @@ module MU
       # @param item [String]: The item within the repository to retrieve
       # @param field [String]: OPTIONAL - A specific field within the item to return.
       # @return [Hash]
-      def self.getSecret(vault: nil, item: nil, field: nil)
+      def self.getSecret(vault: nil, item: nil, field: nil, deploy_dir: nil)
         if vault.nil? or vault.empty?
           raise MuError, "Must call getSecret with at least a vault name"
         end
-
         pwfile = vaultPasswordFile
-        dir = secret_dir+"/"+vault
-        if !Dir.exist?(dir)
+
+        dir = nil
+        try = [secret_dir+"/"+vault]
+        try << deploy_dir+"/ansible/vaults/"+vault if deploy_dir
+        try << MU.mommacat.deploy_dir+"/ansible/vaults/"+vault if MU.mommacat.deploy_dir
+        try.each { |maybe_dir|
+          if Dir.exist?(maybe_dir) and (item.nil? or File.exist?(maybe_dir+"/"+item))
+            dir = maybe_dir
+            break
+          end
+        }
+        if dir.nil?
           raise MuNoSuchSecret, "No such vault #{vault}"
         end
 
@@ -135,17 +145,23 @@ module MU
           cmd = %Q{#{ansibleExecDir}/ansible-vault view #{itempath} --vault-password-file #{pwfile}}
           MU.log cmd
           a = `#{cmd}`
-          # If we happen to have stored recognizeable JSON, return it as parsed,
-          # which is a behavior we're used to from Chef vault. Otherwise, return
-          # a String.
+          # If we happen to have stored recognizeable JSON or YAML, return it
+          # as parsed, which is a behavior we're used to from Chef vault.
+          # Otherwise, return a String.
           begin
             data = JSON.parse(a)
-            if field and data[field]
-              data = data[field]
-            end
           rescue JSON::ParserError
-            data = a
+            begin
+              data = YAML.load(a)
+            rescue Psych::SyntaxError => e
+              data = a
+            end
           end
+          [vault, item, field].each { |tier|
+            if data and data.is_a?(Hash) and tier and data[tier]
+              data = data[tier]
+            end
+          }
         else
           data = []
           Dir.foreach(dir) { |entry|
@@ -160,7 +176,7 @@ module MU
 
       # see {MU::Groomer::Ansible.getSecret}
       def getSecret(vault: nil, item: nil, field: nil)
-        self.class.getSecret(vault: vault, item: item, field: field)
+        self.class.getSecret(vault: vault, item: item, field: field, deploy_dir: @server.deploy.deploy_dir)
       end
 
       # Delete a Ansible data bag / Vault
@@ -215,7 +231,9 @@ module MU
           play = {
             "hosts" => @server.config['name']
           }
-          play["become"] = "yes" if @server.config['ssh_user'] != "root"
+          if !@server.windows? and @server.config['ssh_user'] != "root"
+            play["become"] = "yes"
+          end
           play["roles"] = override_runlist if @server.config['run_list'] and !@server.config['run_list'].empty?
           play["vars"] = @server.config['ansible_vars'] if @server.config['ansible_vars']
 
@@ -252,6 +270,7 @@ module MU
             sleep 30
             retries += 1
             MU.log "Failed Ansible run, will retry (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE, details: cmd
+
             retry
           else
             tmpfile.unlink if tmpfile
@@ -280,7 +299,7 @@ module MU
           "hosts" => @server.config['name']
         }
 
-        if @server.config['ssh_user'] != "root"
+        if !@server.windows? and @server.config['ssh_user'] != "root"
           play["become"] = "yes"
         end
 
@@ -292,9 +311,26 @@ module MU
           play["vars"] = @server.config['ansible_vars']
         end
 
+        if @server.windows?
+          play["vars"] ||= {}
+          play["vars"]["ansible_connection"] = "winrm"
+          play["vars"]["ansible_winrm_scheme"] = "https"
+          play["vars"]["ansible_winrm_transport"] = "ntlm"
+          play["vars"]["ansible_winrm_server_cert_validation"] = "ignore" # XXX this sucks; use Mu_CA.pem if we can get it to work
+#          play["vars"]["ansible_winrm_ca_trust_path"] = "#{MU.mySSLDir}/Mu_CA.pem"
+          play["vars"]["ansible_user"] = @server.config['windows_admin_username']
+          win_pw = @server.getWindowsAdminPassword
+
+          pwfile = MU::Groomer::Ansible.vaultPasswordFile
+          cmd = %Q{#{MU::Groomer::Ansible.ansibleExecDir}/ansible-vault}
+          output = %x{#{cmd} encrypt_string '#{win_pw.gsub(/'/, "\\\\'")}' --vault-password-file #{pwfile}}
+
+          play["vars"]["ansible_password"] = output
+        end
+
         File.open(@ansible_path+"/"+@server.config['name']+".yml", File::CREAT|File::RDWR|File::TRUNC, 0600) { |f|
           f.flock(File::LOCK_EX)
-          f.puts [play].to_yaml
+          f.puts [play].to_yaml.sub(/ansible_password: \|-?[\n\s]+/, 'ansible_password: ') # Ansible doesn't like this (legal) YAML
           f.flock(File::LOCK_UN)
         }
       end
@@ -388,6 +424,7 @@ module MU
         if !system(cmd, "encrypt_string", string, "--name", name, "--vault-password-file", pwfile)
           raise MuError, "Failed Ansible command: #{cmd} encrypt_string <redacted> --name #{name} --vault-password-file"
         end
+        output
       end
 
       # Hunt down and return a path for Ansible executables

data/modules/mu/groomers/chef.rb

@@ -329,7 +329,7 @@ module MU
             }
           else
             MU.log "Invoking Chef over WinRM on #{@server.mu_name}: #{purpose}"
-            winrm = @server.getWinRMSession(haveBootstrapped? ? 1 : max_retries)
+            winrm = @server.getWinRMSession(haveBootstrapped? ? 2 : max_retries)
             if @server.windows? and @server.windowsRebootPending?(winrm)
               # Windows frequently gets stuck here
               if retries > 5
@@ -415,9 +415,9 @@ module MU
           if retries < max_retries
             retries += 1
             MU.log "#{@server.mu_name}: Chef run '#{purpose}' failed after #{Time.new - runstart} seconds, retrying (#{retries}/#{max_retries})", MU::WARN, details: e.message.dup
-            if purpose != "Base Windows configuration"
-              windows_try_ssh = !windows_try_ssh
-            end
+#            if purpose != "Base Windows configuration"
+#              windows_try_ssh = !windows_try_ssh
+#            end
             if e.is_a?(WinRM::WinRMError)
               if @server.windows? and retries >= 3 and retries % 3 == 0
                 # Mix in a hard reboot if WinRM isn't answering
@@ -623,9 +623,15 @@ module MU
             kb.config[:winrm_port] = 5986
             kb.config[:session_timeout] = timeout
             kb.config[:operation_timeout] = timeout
-            kb.config[:winrm_authentication_protocol] = :cert
-            kb.config[:winrm_client_cert] = "#{MU.mySSLDir}/#{@server.mu_name}-winrm.crt"
-            kb.config[:winrm_client_key] = "#{MU.mySSLDir}/#{@server.mu_name}-winrm.key"
+            if retries % 2 == 0
+              kb.config[:winrm_authentication_protocol] = :basic
+              kb.config[:winrm_user] = @server.config['windows_admin_username']
+              kb.config[:winrm_password] = @server.getWindowsAdminPassword
+            else
+              kb.config[:winrm_authentication_protocol] = :cert
+              kb.config[:winrm_client_cert] = "#{MU.mySSLDir}/#{@server.mu_name}-winrm.crt"
+              kb.config[:winrm_client_key] = "#{MU.mySSLDir}/#{@server.mu_name}-winrm.key"
+            end
 #          kb.config[:ca_trust_file] = "#{MU.mySSLDir}/Mu_CA.pem"
             # XXX ca_trust_file doesn't work for some reason, so we have to set the below for now
             kb.config[:winrm_ssl_verify_mode] = :verify_none

data/modules/mu/master/ssl.rb

@@ -160,7 +160,6 @@ module MU
 
         key = getKey(name, for_user: for_user)
 
-puts cn_str
         cn = OpenSSL::X509::Name.parse(cn_str)
 
         # If we're generating our local CA, we're not really doing a CSR, but

data/modules/mu/mommacat.rb

@@ -976,8 +976,15 @@ module MU
                         nil
                       elsif !mu_name.nil?
                         mu_name
+                      # AWS-style tags
+                      elsif descriptor.respond_to?(:tags) and
+                            descriptor.tags.is_a?(Array) and
+                            descriptor.tags.first.respond_to?(:key) and
+                            descriptor.tags.map { |t| t.key }.include?("Name")
+                        descriptor.tags.select { |t| t.key == "Name" }.first.value
                       else
                         try = nil
+                        # Various GCP fields
                         [:display_name, :name, (resourceclass.cfg_name+"_name").to_sym].each { |field|
                           if descriptor.respond_to?(field) and descriptor.send(field).is_a?(String)
                             try = descriptor.send(field)
@@ -1568,6 +1575,7 @@ MESSAGE_END
         MU::Master::SSL.bootstrap
         sans = []
         sans << canonical_ip if canonical_ip
+        sans << resource.mu_name.downcase if resource.mu_name and resource.mu_name != cert_cn
         # XXX were there other names we wanted to include?
         key = MU::Master::SSL.getKey(cert_cn, keysize: keysize)
         cert, pfx_cert = MU::Master::SSL.getCert(cert_cn, "/CN=#{cert_cn}/O=Mu/C=US", sans: sans, pfx: is_windows)

data/modules/tests/ecs.yaml

@@ -0,0 +1,23 @@
+# Test ECS
+# clouds: AWS
+---
+appname: smoketest
+vpcs:
+- name: ecs
+container_clusters:
+- name: ecsplain
+  flavor: ECS
+  instance_type: t2.medium
+  vpc:
+    name: ecs
+  containers:
+  - name: nginx
+    image: "nginx:1.8"
+- name: ecsfargate
+  flavor: Fargate
+  instance_type: t2.medium
+  vpc:
+    name: ecs
+  containers:
+  - name: nginx
+    image: "nginx:1.8"

data/modules/tests/includes-and-params.yaml

@@ -7,7 +7,7 @@ appname: smoketest
 parameters:
 - name: instancesize
   prettyname: "Instance Size"
-  default: <%= $environment == "prod" ? "t3.large" : "t3.small" %>
+  default: <%= $environment == "prod" ? "m4.large" : "t2.small" %>
 <%= include("poolparams-include.inc") %>
 vpcs:
 - name: parsemess
@@ -18,6 +18,7 @@ server_pools:
 - name: svr
   cloud: AWS
   ssh_user: ec2-user
+  platform: amazon
   tags:
     - key: Env
       value: <%= env %>

data/modules/tests/server-with-scrub-muisms.yaml

@@ -10,6 +10,7 @@ servers:
 - name: <%= name %>
   groomer: Ansible
   platform: centos7
+  ssh_user: centos
   cloud: <%= cloud %>
 <% if cloud == "AWS" %>
   size: t2.medium

data/modules/tests/win2k12.yaml

@@ -0,0 +1,25 @@
+# Windows Server tests
+# clouds: AWS, Google
+---
+appname: smoketest
+us_only: true
+vpcs:
+- name: windows
+  cloud: <%= cloud %>
+servers:
+- name: win2k12
+  platform: win2k12
+  cloud: <%= cloud %>
+<% if cloud == "AWS" %>
+  size: m4.large
+<% elsif cloud == "Azure" %>
+  size: Standard_DS1_v2
+<% elsif cloud == "Google" %>
+  size: n1-standard-2
+<% end %>
+  vpc:
+    name: windows
+    subnet_pref: public
+  associate_public_ip: true
+  static_ip:
+    assign_ip: true

data/modules/tests/win2k16.yaml

@@ -0,0 +1,25 @@
+# Windows Server tests
+# clouds: AWS, Google
+---
+appname: smoketest
+us_only: true
+vpcs:
+- name: windows
+  cloud: <%= cloud %>
+servers:
+- name: win2k16
+  platform: win2k16
+  cloud: <%= cloud %>
+<% if cloud == "AWS" %>
+  size: m4.large
+<% elsif cloud == "Azure" %>
+  size: Standard_DS1_v2
+<% elsif cloud == "Google" %>
+  size: n1-standard-2
+<% end %>
+  vpc:
+    name: windows
+    subnet_pref: public
+  associate_public_ip: true
+  static_ip:
+    assign_ip: true